diff --git a/Dockerfile b/Dockerfile index 81d5956f0..10a44b68e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -16,12 +16,12 @@ ARG USER_GID=$USER_UID # Whether to download Provider as part of image creation ARG DOWNLOAD_PROVIDER=NO # TF Provider version -ARG TF_OCI_PROVIDER=7.8.0 +ARG TF_OCI_PROVIDER=7.19.0 ARG TF_NULL_PROVIDER=3.2.3 RUN microdnf install -y sudo && \ groupadd --gid $USER_GID $USERNAME && \ - useradd --uid $USER_UID --gid $USER_GID -d /$USERNAME -m $USERNAME && \ + useradd --uid $USER_UID --gid $USER_GID $USERNAME && \ echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \ chmod 0440 /etc/sudoers.d/$USERNAME && \ mkdir -p /cd3user/tenancies && \ diff --git a/OCIWorkVMStack/scripts/installToolkit.sh b/OCIWorkVMStack/scripts/installToolkit.sh index 66b3e2541..01b10f0f0 100644 --- a/OCIWorkVMStack/scripts/installToolkit.sh +++ b/OCIWorkVMStack/scripts/installToolkit.sh @@ -1,10 +1,13 @@ #!/bin/bash username=cd3user -sudo mkdir -p /$username/mount_path logfile="/$username/mount_path/installToolkit.log" toolkit_dir="/tmp/githubCode" -tenancyconfig_properties="$toolkit_dir/cd3_automation_toolkit/user-scripts/tenancyconfig.properties" +mount_dir="/$username/mount_path" +sudo mkdir -p /$mount_dir/tenancies +sudo mkdir -p /$mount_dir/oci_tools + +tenancyconfig_properties="/$mount_dir/oci_tools/cd3_automation_toolkit/user-scripts/tenancyconfig.properties" start=$(date +%s.%N) sudo sh -c "echo '########################################################################' >> /etc/motd" sudo sh -c "echo ' Welcome to CD3 Automation Toolkit WorkVM' >> /etc/motd" @@ -66,7 +69,10 @@ fi sudo podman --version >> $logfile 2>&1 || true echo "***Download Toolkit***" >> $logfile 2>&1 -sudo git clone https://github.com/oracle-devrel/cd3-automation-toolkit.git $toolkit_dir >> $logfile 2>&1 +sudo git clone https://github.com/oracle-devrel/cd3-automation-toolkit.git -b develop $toolkit_dir >> $logfile 2>&1 +cp -r $toolkit_dir/cd3_automation_toolkit /$mount_dir/oci_tools/ +cp -r $toolkit_dir/othertools /$mount_dir/oci_tools/ +sudo chown -R $username:$username /$mount_dir/oci_tools/ stop_exec curl -H "Authorization: Bearer Oracle" -L http://169.254.169.254/opc/v2/instance/ -o /tmp/metadata.json @@ -88,7 +94,7 @@ stop_exec sudo podman images >> $logfile 2>&1 echo "***Setting Up podman Container***" >> $logfile 2>&1 -sudo podman run --name cd3_toolkit -it -p 8443:8443 -d -v /cd3user/mount_path:/cd3user/tenancies cd3_toolkit bash >> $logfile 2>&1 +sudo podman run --name cd3_toolkit -it -p 8443:8443 -d -v /cd3user/mount_path:/cd3user cd3_toolkit bash >> $logfile 2>&1 stop_exec sudo podman ps -a >> $logfile 2>&1 echo "Connect to Container using command - sudo podman exec -it cd3_toolkit bash " >> $logfile 2>&1 @@ -99,4 +105,4 @@ duration_sec=$(echo "$(date +%s.%N) - $start" | bc) duration_min=$(echo "$duration_sec%3600/60" | bc) execution_time=`printf "%.2f seconds" $duration_sec` echo "Script Execution Time in Seconds: $execution_time" >> $logfile 2>&1 -echo "Script Execution Time in Minutes: approx $duration_min Minutes" >> $logfile 2>&1 +echo "Script Execution Time in Minutes: approx $duration_min Minutes" >> $logfile 2>&1 \ No newline at end of file diff --git a/README.md b/README.md index 9c905f465..b8243c7bf 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@
- [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2025.1.4)  • [Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/)  • [CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/) •  [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp)  • [Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/)  • [Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724)  • [Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries) + [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2025.2.0)  • [Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/)  • [CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/) •  [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp)  • [Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/)  • [Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724)  • [Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries)
@@ -39,16 +39,15 @@ Additionally, the toolkit also supports seamless resource management using OCI D DNS Management Load Balancers - - OCI Network Firewall - KMS - Policy Enforcement using OPA - - Compute Storage - Database + Oracle, MySQL Databases + + + OCI Network Firewall + KMS + Policy Enforcement using OPA OKE @@ -63,15 +62,14 @@ Additionally, the toolkit also supports seamless resource management using OCI D Cloud Guard SHOWOCI report - CIS Landing Zone
Compliance - + CIS Compliance Report - - +📌 The Toolkit also extends its support to ADB@Azure +
## Why CD3? diff --git a/cd3_automation_toolkit/CostManagement/__init__.py b/cd3_automation_toolkit/CostManagement/__init__.py deleted file mode 100644 index 2c187a105..000000000 --- a/cd3_automation_toolkit/CostManagement/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .Budget import * \ No newline at end of file diff --git a/cd3_automation_toolkit/DeveloperServices/__init__.py b/cd3_automation_toolkit/DeveloperServices/__init__.py deleted file mode 100644 index 006227503..000000000 --- a/cd3_automation_toolkit/DeveloperServices/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 - -from .ResourceManager import * -from .OKE import * - diff --git a/cd3_automation_toolkit/Governance/__init__.py b/cd3_automation_toolkit/Governance/__init__.py deleted file mode 100644 index b755e9d6c..000000000 --- a/cd3_automation_toolkit/Governance/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python3 - - -from .Tagging import * -from .Quota import * - diff --git a/cd3_automation_toolkit/Identity/__init__.py b/cd3_automation_toolkit/Identity/__init__.py deleted file mode 100644 index c9ed93a69..000000000 --- a/cd3_automation_toolkit/Identity/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 - -from .Compartments import create_terraform_compartments -from .Users import create_terraform_users -from .Groups import create_terraform_groups -from .Policies import create_terraform_policies -from .export_identity_nonGreenField import export_identity -from .NetworkSources import export_networkSources -from .NetworkSources import create_terraform_networkSources -from .Users import export_users diff --git a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/templates/notifications-subscriptions-template b/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/templates/notifications-subscriptions-template deleted file mode 100644 index 326111ac2..000000000 --- a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/templates/notifications-subscriptions-template +++ /dev/null @@ -1,57 +0,0 @@ -{% if skeleton %} -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. -# -############################ -# ManagementServices -# Notifications_Subscriptions - tfvars -# Allowed Values: -# topic_id can be ocid or the key of notifications_topics (map) -# compartment_id can be the ocid or the name of the compartment hierarchy delimited by double hiphens "--" -# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Security--Prod" where "Security" is the parent of "Prod" compartment -############################ - -notifications_subscriptions = { - ##Add New Subscriptions for {{ region|lower }} here## -} -{% else %} - {{ subscription_tf_name }} = { - subscription_name = "{{ subscription_tf_name }}" - compartment_id = "{{ compartment_tf_name }}" - endpoint = "{{ endpoint }}" - protocol = "{{ protocol }}" - topic_id = "{{ topic_tf_name }}" - - {# ##Do not modify below this line## #} - {# #} - {# ###Section for adding Defined and Freeform Tags### #} - {% if subscription_defined_tags and subscription_defined_tags != 'nan' and subscription_defined_tags != '' and subscription_defined_tags != [['nan']] %} - {% if subscription_defined_tags[0] %} - defined_tags = { - {% for tags in subscription_defined_tags %} - {% if not loop.last %} - "{{ tags[0] }}"= "{{ tags[1] }}" , - {% else %} - "{{ tags[0] }}"= "{{ tags[1] }}" - {% endif %} - {% endfor %} - } - {% endif %} - {% endif %} - {% if subscription_freeform_tags and subscription_freeform_tags != 'nan' and subscription_freeform_tags != '' and subscription_freeform_tags != [['nan']] %} - {% if subscription_freeform_tags[0] %} - freeform_tags = { - {% for tags in subscription_freeform_tags %} - {% if not loop.last %} - "{{ tags[0] }}"="{{ tags[1] }}", - {% else %} - "{{ tags[0] }}"="{{ tags[1] }}" - {% endif %} - {% endfor %} - } - {% endif %} - {% endif %} - {# ###Section for adding Defined and Freeform Tags ends here### #} - }, - -{% endif %} \ No newline at end of file diff --git a/cd3_automation_toolkit/ManagementServices/__init__.py b/cd3_automation_toolkit/ManagementServices/__init__.py deleted file mode 100755 index 320106094..000000000 --- a/cd3_automation_toolkit/ManagementServices/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .EventsAndNotifications import * -from .Monitoring import * -from .Logging import * -from .ServiceConnectorHub import * \ No newline at end of file diff --git a/cd3_automation_toolkit/Network/__init__.py b/cd3_automation_toolkit/Network/__init__.py deleted file mode 100644 index 6e0889176..000000000 --- a/cd3_automation_toolkit/Network/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python3 - -from .BaseNetwork import * -from .LoadBalancers import * -from .DNS import * -from .Global import * -#from .IPManagement import * \ No newline at end of file diff --git a/cd3_automation_toolkit/Release-Notes b/cd3_automation_toolkit/Release-Notes index 171cfd696..084fa59be 100644 --- a/cd3_automation_toolkit/Release-Notes +++ b/cd3_automation_toolkit/Release-Notes @@ -1,3 +1,14 @@ +------------------------------------- +CD3 Automation Toolkit Tag v2025.2.0 +Oct 10th, 2025 +------------------------------------- +1. MultiCloud Support - Create and Export ADB @Azure using CD3. Documentation Link - https://oracle-devrel.github.io/cd3-automation-toolkit/latest/connect-container-to-azure-subscription/ +2. Support full functionality of OCI Network Firewall using CD3. +3. Upgrade ADB Excel and Terraform to support all features. Password can be fetched from OCI vault too. +4. Performance tuning of Terraform for notifications and identity domain modules. +5. Support for IPv6 under networking. +6. Bug fix wrt SCH log re-ordering during terraform plan. + ------------------------------------- CD3 Automation Toolkit Tag v2025.1.4 Sep 17th, 2025 diff --git a/cd3_automation_toolkit/Security/__init__.py b/cd3_automation_toolkit/Security/__init__.py deleted file mode 100644 index ae781f9cf..000000000 --- a/cd3_automation_toolkit/Security/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python3 - -from .CloudGuard import * -from .KeyVault import * -from .Firewall import * - diff --git a/cd3_automation_toolkit/Storage/__init__.py b/cd3_automation_toolkit/Storage/__init__.py deleted file mode 100644 index d14dd28ed..000000000 --- a/cd3_automation_toolkit/Storage/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python3 - -from .BlockVolume import * -from .FileSystem import * -from .ObjectStorage import * - diff --git a/cd3_automation_toolkit/__init__.py b/cd3_automation_toolkit/__init__.py deleted file mode 100644 index 84c6c55a8..000000000 --- a/cd3_automation_toolkit/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 - -from .cd3Validator import validate_cd3 -from .cd3FirewallValidator import validate_firewall_cd3 - diff --git a/cd3_automation_toolkit/azurecloud/__init__.py b/cd3_automation_toolkit/azurecloud/__init__.py new file mode 100644 index 000000000..5f7ce86af --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/python/__init__.py b/cd3_automation_toolkit/azurecloud/python/__init__.py new file mode 100644 index 000000000..6332da756 --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/python/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 + +from .azrCommonTools import * +from .database import * \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/python/azrCommonTools.py b/cd3_automation_toolkit/azurecloud/python/azrCommonTools.py new file mode 100644 index 000000000..f5f40cd9b --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/python/azrCommonTools.py @@ -0,0 +1,84 @@ + + +from typing import Dict +from common.python.commonTools import * +from typing import Dict, Optional +import os +# Azure SDKs +try: + from azure.identity import ClientSecretCredential + from azure.core.exceptions import HttpResponseError + +except ImportError as e: + raise ImportError( + "Missing Azure SDK packages. Install with:\n" + " pip install azure-identity azure-mgmt-oracledatabase\n" + ) + + + +def _read_properties_file(filepath: str) -> Dict[str, str]: + """Read key=value pairs from setUpAzure.properties into a dict.""" + if not os.path.exists(filepath): + raise FileNotFoundError(f"Azure properties file not found: {filepath}") + props: Dict[str, str] = {} + with open(filepath, "r", encoding="utf-8") as f: + for raw in f: + line = raw.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + k, v = line.split("=", 1) + props[k.strip()] = v.strip() + required = ["subscription_id", "tenant_id", "client_id", "client_secret"] + missing = [k for k in required if not props.get(k)] + if missing: + return "Missing required Azure credentials in {filepath}: {', '.join(missing)}" + + return props + + +class azrCommonTools(): + tagColumns = {'common tags', 'common_tags'} + def authenticate(self,azure_properties_file): + # Azure credential & client + az = _read_properties_file(azure_properties_file) + if "Missing required" in az: + print("\nCannot run export workflow as authentication parameters are missing!!\n") + exit() + credential = ClientSecretCredential( + tenant_id=az["tenant_id"], + client_id=az["client_id"], + client_secret=az["client_secret"], + ) + credentials=[credential,az["subscription_id"]] + return credentials + + def split_tag_values(columnname, columnvalue, tempdict): + columnvalue = columnvalue.replace("\n", "") + if ";" in columnvalue: + # If there are more than one tag; split them by ";" and "=" + + columnname = commonTools.check_column_headers(columnname) + multivalues = columnvalue.split(";") + multivalues = [part.split("=") for part in multivalues if part] + + tempdict = {columnname: multivalues} + else: + # If there is only one tag; split them only by "="; each key-value pair is stored as a list + columnname = commonTools.check_column_headers(columnname) + multivalues = columnvalue.split("=") + multivalues = [str(part).strip() for part in multivalues if part] + + tempdict = {columnname: [multivalues]} + return tempdict + + def _flatten_tags(tags: Optional[Dict[str, str]]) -> str: + if not tags: + return "" + try: + return ";".join([f"{k}={v}" for k, v in tags.items() if v is not None]) + except Exception: + return "" + diff --git a/cd3_automation_toolkit/azurecloud/python/database/__init__.py b/cd3_automation_toolkit/azurecloud/python/database/__init__.py new file mode 100644 index 000000000..1952d113f --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/python/database/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from .create_terraform_adb_azure import create_terraform_adb_azure +from .export_adb_azure import export_adb_azure +from .create_terraform_exa_infra_azure import create_terraform_exa_infra_azure +from .create_terraform_exa_vmclusters_azure import create_terraform_exa_vmclusters_azure + diff --git a/cd3_automation_toolkit/azurecloud/python/database/create_terraform_adb_azure.py b/cd3_automation_toolkit/azurecloud/python/database/create_terraform_adb_azure.py new file mode 100644 index 000000000..38371a071 --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/python/database/create_terraform_adb_azure.py @@ -0,0 +1,134 @@ +#!/usr/bin/python3 +# Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. +# +# This script will produce a Terraform file that will be used to set up OCI Database +# Autonomous Database @Azure +# +# Author: Suruchi +# Oracle Consulting +# +import os +import sys +from jinja2 import Environment, FileSystemLoader +from pathlib import Path +sys.path.append(os.getcwd()+"/..") +from common.python.commonTools import * +import azurecloud.python.azrCommonTools as azrCommonTools + + +###### +# Required Inputs- CD3 excel file, prefix AND outdir +###### +# Execution of the code begins here +def create_terraform_adb_azure(inputfile, outdir, prefix): + + filename = inputfile + sheetName = "ADB-Azure" + resource=sheetName.lower() + auto_tfvars_filename = prefix + '_' + sheetName.lower() + '.auto.tfvars' + + # Load the template file + file_loader = FileSystemLoader(f'{Path(__file__).parent}/templates') + env = Environment(loader=file_loader, keep_trailing_newline=True, trim_blocks=True, lstrip_blocks=True) + template = env.get_template('adb-azure-template') + + tfStr='' + + + # Read cd3 using pandas dataframe + df, col_headers = commonTools.read_cd3(filename, sheetName) + #Remove empty rows + df = df.dropna(how='all') + df = df.reset_index(drop=True) + + # List of the column headers + dfcolumns = df.columns.values.tolist() + + # Iterate over rows + for i in df.index: + region = str(df.loc[i, 'Region']).strip() + # Encountered + if (region in commonTools.endNames): + break + + region=region.strip().lower() + + # temporary dictionary1 and dictionary2 + tempStr = {} + tempdict = {} + + # All columns ar mandatory except customer contacts and tags + if (str(df.loc[i, 'Region']).lower() == 'nan' or \ + str(df.loc[i, 'Resource Group Name']).lower() == 'nan' or \ + str(df.loc[i, 'Network Details']).lower() == 'nan' or \ + str(df.loc[i, 'DB Version']).lower() == 'nan'): + print("\nAll fields except Customer Contacts and Common Tags are mandatory. Please enter a value and try again !!") + print("\n** Exiting **") + exit(1) + + + for columnname in dfcolumns: + # Column value + columnvalue = str(df[columnname][i]).strip() + + # Check for boolean/null in column values + columnvalue = commonTools.check_columnvalue(columnvalue) + + # Check for multivalued columns + tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) + + + # Process Defined and Freeform Tags + if columnname.lower() in azrCommonTools.tagColumns: + tempdict = azrCommonTools.split_tag_values(columnname, columnvalue, tempdict) + + if columnname == "ADB Display Name": + display_tf_name = columnvalue.strip() + display_tf_name = commonTools.check_tf_variable(display_tf_name) + tempdict = {'display_tf_name': display_tf_name} + + if columnname == "Resource Group Name": + container_id = columnvalue.strip() + tempdict = {'container_id': container_id} + + + if columnname == 'Database Workload': + autonomous_value = columnvalue.strip().lower() + tempdict = {'autonomous_value': autonomous_value} + + if columnname == "Network Details": + if len(columnvalue.split("@")) == 2: + network_container_id = columnvalue.split("@")[0].strip() + vcn_subnet_name = columnvalue.split("@")[1].strip() + else: + network_container_id = container_id + vcn_subnet_name = columnvalue + + if ("::" not in vcn_subnet_name): + print("Invalid Network Details format specified for row " + str(i + 3) + ". Exiting!!!") + exit(1) + else: + vcn_name = vcn_subnet_name.split("::")[0].strip() + subnet_id = vcn_subnet_name.split("::")[1].strip() + + tempdict = {'network_container_id': network_container_id, 'vnet_name': vcn_name,'subnet_id': subnet_id} + + + columnname = commonTools.check_column_headers(columnname) + tempStr[columnname] = str(columnvalue).strip() + tempStr.update(tempdict) + + # Write all info to TF string + tfStr=tfStr + template.render(tempStr) + + + if(tfStr!=''): + outfile = outdir + "/" + auto_tfvars_filename + commonTools.backup_file(outdir, resource, auto_tfvars_filename) + src = "##Add New ADB @Azure here##" + tfStr= template.render(count=0).replace(src, tfStr + "\n" + src) + tfStr = "".join([s for s in tfStr.strip().splitlines(True) if s.strip("\r\n").strip()]) + oname=open(outfile,'w') + oname.write(tfStr) + oname.close() + print(outfile + " containing TF for ADB @Azure has been created") diff --git a/cd3_automation_toolkit/azurecloud/python/database/create_terraform_exa_infra_azure.py b/cd3_automation_toolkit/azurecloud/python/database/create_terraform_exa_infra_azure.py new file mode 100644 index 000000000..bdc3778de --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/python/database/create_terraform_exa_infra_azure.py @@ -0,0 +1,118 @@ +#!/usr/bin/python3 +# Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. +# +# This script will produce a Terraform file that will be used to set up OCI Database +# Database EXA +# +# Author: Suruchi +# Oracle Consulting +# + +import os +import sys +from jinja2 import Environment, FileSystemLoader +from pathlib import Path +sys.path.append(os.getcwd()+"/..") +from common.python.commonTools import * +import azurecloud.python.azrCommonTools as azrCommonTools + + +###### +# Required Inputs- CD3 excel file, prefix AND outdir +###### +# Execution of the code begins here +def create_terraform_exa_infra_azure(inputfile, outdir, prefix): + + filename = inputfile + sheetName = "EXA-Infra-Azure" + auto_tfvars_filename = prefix + '_' + sheetName.lower() + '.auto.tfvars' + resource = sheetName.lower() + + # Load the template file + file_loader = FileSystemLoader(f'{Path(__file__).parent}/templates') + env = Environment(loader=file_loader, keep_trailing_newline=True, trim_blocks=True, lstrip_blocks=True) + template = env.get_template('exa-infra-azure-template') + + # Read cd3 using pandas dataframe + df, col_headers = commonTools.read_cd3(filename, sheetName) + + #Remove empty rows + df = df.dropna(how='all') + df = df.reset_index(drop=True) + tfStr = '' + + # List of the column headers + dfcolumns = df.columns.values.tolist() + + # Iterate over rows + for i in df.index: + region = str(df.loc[i, 'Region']).strip() + + # Encountered + if (region in commonTools.endNames): + break + + if region.lower() == 'nan': + continue + + region=region.strip().lower() + + # temporary dictionary1 and dictionary2 + tempStr = {} + tempdict = {} + + # Check if values are entered for mandatory fields + if str(df.loc[i, 'Region']).lower() == 'nan' or \ + str(df.loc[i, 'Resource Group Name']).lower() == 'nan' or \ + str(df.loc[i, 'Exadata Infra Display Name']).lower() == 'nan' or \ + str(df.loc[i, 'Shape']).lower() == 'nan': + print("\nAll fields except Maintenane Window, Customer Contacts and Common Tags are mandatory. Please enter a value and try again !!") + + exit(1) + + #tempdict = {'oracle_db_software_edition' : 'ENTERPRISE_EDITION_EXTREME_PERFORMANCE'} + + for columnname in dfcolumns: + # Column value + columnvalue = str(df[columnname][i]).strip() + + # Check for boolean/null in column values + columnvalue = commonTools.check_columnvalue(columnvalue) + + # Check for multivalued columns + tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) + + + # Process Defined and Freeform Tags + if columnname.lower() in azrCommonTools.tagColumns: + tempdict = azrCommonTools.split_tag_values(columnname, columnvalue, tempdict) + + if columnname == "Exadata Infra Display Name": + display_name = columnvalue.strip() + display_tf_name = commonTools.check_tf_variable(display_name) + tempdict = {'display_name': display_name, 'display_tf_name': display_tf_name} + + if columnname == "Resource Group Name": + container_id = columnvalue.strip() + tempdict = {'container_id': container_id} + + + columnname = commonTools.check_column_headers(columnname) + tempStr[columnname] = str(columnvalue).strip() + tempStr.update(tempdict) + + + # Write all info to TF string + tfStr = tfStr + template.render(tempStr) + + # Write TF string to the file + if (tfStr != ''): + outfile = outdir + "/" + auto_tfvars_filename + commonTools.backup_file(outdir, resource, auto_tfvars_filename) + src = "##Add New Exa-Infra @Azure here##" + tfStr = template.render(count=0).replace(src, tfStr + "\n" + src) + tfStr = "".join([s for s in tfStr.strip().splitlines(True) if s.strip("\r\n").strip()]) + oname = open(outfile, 'w') + oname.write(tfStr) + oname.close() + print(outfile + " containing TF for Exa-Infra @Azure has been created") \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/python/database/create_terraform_exa_vmclusters_azure.py b/cd3_automation_toolkit/azurecloud/python/database/create_terraform_exa_vmclusters_azure.py new file mode 100644 index 000000000..ffd857a5b --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/python/database/create_terraform_exa_vmclusters_azure.py @@ -0,0 +1,153 @@ +#!/usr/bin/python3 +# Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. +# +# This script will produce a Terraform file that will be used to set up OCI Database +# Database EXA +# +# Author: Kartikey Rajput +# Oracle Consulting +# Modified (TF Upgrade): Kartikey Rajput +# +import os +import sys +from jinja2 import Environment, FileSystemLoader +from pathlib import Path +sys.path.append(os.getcwd()+"/..") +from common.python.commonTools import * +import azurecloud.python.azrCommonTools as azrCommonTools + + +###### +# Required Inputs- CD3 excel file, Config file, prefix AND outdir +###### +# Execution of the code begins here +def create_terraform_exa_vmclusters_azure(inputfile, outdir, prefix): + filename = inputfile + + sheetName = "EXA-VMClusters-Azure" + auto_tfvars_filename = prefix + '_' + sheetName.lower() + '.auto.tfvars' + resource = sheetName.lower() + + + # Load the template file + file_loader = FileSystemLoader(f'{Path(__file__).parent}/templates') + env = Environment(loader=file_loader, keep_trailing_newline=True, trim_blocks=True, lstrip_blocks=True) + template = env.get_template('exa-vmcluster-azure-template') + + # Read cd3 using pandas dataframe + df, col_headers = commonTools.read_cd3(filename, sheetName) + tfStr = '' + + # Remove empty rows + df = df.dropna(how='all') + df = df.reset_index(drop=True) + + # List of the column headers + dfcolumns = df.columns.values.tolist() + #subnets = parseSubnets(filename) + + + # Iterate over rows + for i in df.index: + region = str(df.loc[i, 'Region']).strip() + + # Encountered + if (region in commonTools.endNames): + break + + if region.lower() == 'nan': + continue + + region = region.strip().lower() + + + # temporary dictionary1 and dictionary2 + tempStr = {} + tempdict = {} + + + # Check if values are entered for mandatory fields + ''' + if str(df.loc[i, 'Region']).lower() == 'nan' or \ + str(df.loc[i, 'Compartment Name']).lower() == 'nan' or \ + str(df.loc[i, 'Exadata Infra Display Name']).lower() == 'nan' or \ + str(df.loc[i, 'VM Cluster Display Name']).lower() == 'nan' or \ + str(df.loc[i, 'Client Network Details']).lower() == 'nan' or \ + str(df.loc[i, 'Backup Network Details']).lower() == 'nan' or \ + str(df.loc[i, 'CPU Core Count']).lower() == 'nan' or \ + str(df.loc[i, 'SSH Key Var Name']).lower() == 'nan' or \ + str(df.loc[i, 'Hostname Prefix']).lower() == 'nan' or \ + str(df.loc[i, 'Oracle Grid Infrastructure Version']).lower() == 'nan': + print("\nRegion, Compartment Name, Exadata Infra Display Name, VM Cluster Display Name, Network Details, CPU Core Count, Hostname Prefix, Oracle Grid Infrastructure Version, SSH Key Var Name are mandatory fields. Please enter a value and try again.......Exiting!!") + exit(1) + ''' + + # tempdict = {'oracle_db_software_edition' : 'ENTERPRISE_EDITION_EXTREME_PERFORMANCE'} + + for columnname in dfcolumns: + # Column value + columnvalue = str(df[columnname][i]).strip() + + # Check for boolean/null in column values + columnvalue = commonTools.check_columnvalue(columnvalue) + + # Check for multivalued columns + tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) + + if columnname == "Resource Group Name": + container_id = columnvalue.strip() + tempdict = {'container_id': container_id} + + if columnname == "VM Cluster Display Name": + display_name = columnvalue.strip() + display_tf_name = commonTools.check_tf_variable(display_name) + tempdict = {'display_tf_name': display_tf_name, 'display_name': display_name} + + if columnname == "Network Details": + if len(columnvalue.split("@")) == 2: + network_container_id = columnvalue.split("@")[0].strip() + vcn_subnet_name = columnvalue.split("@")[1].strip() + else: + network_container_id = container_id + vcn_subnet_name = columnvalue + + if ("::" not in vcn_subnet_name): + print("Invalid Network Details format specified for row " + str(i + 3) + ". Exiting!!!") + exit(1) + else: + vcn_name = vcn_subnet_name.split("::")[0].strip() + subnet_id = vcn_subnet_name.split("::")[1].strip() + + tempdict = {'network_container_id': network_container_id, 'vnet_name': vcn_name,'subnet_id': subnet_id} + + + # Process Defined and Freeform Tags + if columnname.lower() in azrCommonTools.tagColumns: + tempdict = azrCommonTools.split_tag_values(columnname, columnvalue, tempdict) + + + if columnname == "Exadata Infra Display Name": + exadata_infrastructure_name = columnvalue.strip() + tempdict = {'exadata_infrastructure_name': exadata_infrastructure_name} + + + columnname = commonTools.check_column_headers(columnname) + tempStr[columnname] = str(columnvalue).strip() + tempStr.update(tempdict) + + + + # Write all info to TF string + tfStr = tfStr + template.render(tempStr) + + # Write TF string to the file + if (tfStr != ''): + outfile = outdir + "/" + auto_tfvars_filename + commonTools.backup_file(outdir, resource, auto_tfvars_filename) + src = "##Add New Exa-VMCluster @Azure here##" + tfStr = template.render(count=0).replace(src, tfStr + "\n" + src) + tfStr = "".join([s for s in tfStr.strip().splitlines(True) if s.strip("\r\n").strip()]) + oname = open(outfile, 'w') + oname.write(tfStr) + oname.close() + print(outfile + " containing TF for Exa-VMCluster @Azure has been created") \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/python/database/export_adb_azure.py b/cd3_automation_toolkit/azurecloud/python/database/export_adb_azure.py new file mode 100644 index 000000000..cc527154a --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/python/database/export_adb_azure.py @@ -0,0 +1,343 @@ +#!/usr/bin/python3 +# Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved. +# This script will Export ADB @Azure resources into CD3 (existing worksheet) and write Terraform/tofu import commands +# Author: Ulaganathan N +# Oracle Consulting +############################################################################### +import os +import sys +import subprocess as sp +sys.path.append(os.getcwd()+"/..") +from common.python.commonTools import * +import azurecloud.python.azrCommonTools as azrCommonTools +from typing import Dict, List, Optional +try: + from azure.mgmt.oracledatabase import OracleDatabaseMgmtClient as OracleDBClient +except ImportError: + from azure.mgmt.oracledatabase import OracleDatabaseManagementClient as OracleDBClient + + +# Global declaration +importCommands: Dict[str, str] = {} + + +def pick_first_not_none(*values): + """Return the first non-None value from a list of values.""" + for v in values: + if v is not None: + return v + return None + + +def normalize_enum_token(value): + """ + Return the enum/member token expected by tfvars as a string. + For Example: + 'WorkloadType.OLTP' -> 'OLTP' + """ + if value is None: + return "" + s = str(value) + return s.split(".", 1)[1] if "." in s else s + + +def _format_rg_vnet_subnet_from_id(net_id: str) -> str: + """Return 'resourceGroup@vnet::subnet' from a subnet ARM ID.""" + if not net_id: + return "" + parts = net_id.strip("/").split("/") + try: + rg = parts[parts.index("resourceGroups") + 1] + vnet = parts[parts.index("virtualNetworks") + 1] + subnet = parts[parts.index("subnets") + 1] + return f"{rg}@{vnet}::{subnet}" + except (ValueError, IndexError): + return "" + + +def _get_rg_from_id(resource_id: str) -> str: + # ARM ID format: /subscriptions//resourceGroups//providers/.../autonomousDatabases/ + try: + parts = resource_id.split("/") + idx = parts.index("resourceGroups") + return parts[idx + 1] + except Exception: + return "" + + +def _list_adbs(client: "OracleDBClient", resource_groups: Optional[List[str]] = None): + """Yield ADB @Azure resources across provided RGs or entire subscription.""" + if resource_groups: + for rg in resource_groups: + try: + for adb in client.autonomous_databases.list_by_resource_group(rg): + yield adb + except HttpResponseError as e: + print(f"[WARN] Failed listing ADBs in RG '{rg}': {e}") + else: + for adb in client.autonomous_databases.list_by_subscription(): + yield adb + + +def print_adbs_azure(adb, values_for_column: Dict[str, List], state: Dict, tf_or_tofu: str, ): + """Populate CD3 columns for a single ADB @Azure and queue Terraform import commands.""" + props = getattr(adb, "properties", None) or adb + + # Resource names and IDs + rg_name = _get_rg_from_id(getattr(adb, "id", "")) + adb_name = getattr(adb, "name", "") + adb_location = getattr(adb, "location", "") + + # Contacts (list of objects with .email) + contacts_csv = "" + try: + contacts = getattr(props, "customer_contacts", None) + if contacts: + emails = [] + for c in contacts: + email = getattr(c, "email", None) or getattr(c, "contact", None) + if email: + emails.append(email) + contacts_csv = ",".join(emails) + except Exception: + contacts_csv = "" + + # Tags (dict) — kept as metadata only if the sheet has 'Common Tags' column(not used yet) + common_tags = azrCommonTools._flatten_tags(getattr(adb, "tags", None)) + + # Prefer subnetId to derive rg@vnet::subnet; private endpoint IDs don't contain vnet/subnet names + subnet_id = pick_first_not_none( + getattr(props, "subnet_id", None), + getattr(props, "subnetId", None), + ) + formatted_net = _format_rg_vnet_subnet_from_id(subnet_id) + + # Optional: keep raw IDs if needed elsewhere + network_details_raw = pick_first_not_none( + getattr(props, "subnet_id", None), + getattr(props, "subnetId", None), + getattr(props, "private_endpoint_id", None), + getattr(props, "privateEndpointId", None), + "", + ) + + # Whitelisted IPs (array of strings) + ips_list = pick_first_not_none( + getattr(props, "whitelisted_ips", None), # snake_case (SDK model) + getattr(props, "whitelistedIps", None), # camelCase (REST casing) + ) + whitelisted_ips = ",".join(ips_list) if isinstance(ips_list, list) else "" + + # Compute details + compute_model_raw = (getattr(props, "compute_model", None) or getattr(props, "computeModel", None) or "") + compute_model = normalize_enum_token(compute_model_raw) + + compute_count = ( + getattr(props, "compute_count", None) + or getattr(props, "computeCount", None) + or "" + ) + ocpu_cores = ( + getattr(props, "cpu_core_count", None) + or getattr(props, "ocpuCoreCount", None) + or "" + ) + + db_version = getattr(props, "db_version", None) or getattr(props, "databaseVersion", None) or "" + db_edition_raw = getattr(props, "database_edition", None) or getattr(props, "databaseEdition", None) or "" + db_edition = normalize_enum_token(db_edition_raw) + storage_tbs = getattr(props, "data_storage_size_in_tbs", None) or getattr(props, "dataStorageSizeInTbs", None) or "" + workload_raw = getattr(props, "db_workload", None) or getattr(props, "databaseWorkload", None) or "" + workload = normalize_enum_token(workload_raw) + license_model_raw = getattr(props, "license_model", None) or getattr(props, "licenseModel", None) or "" + license_model = normalize_enum_token(license_model_raw) + backup_retention_days = \ + (getattr(props, "backup_retention_period_in_days", None) or getattr(props, "backupRetentionDays", + None) or "") + # Character sets + char_set = pick_first_not_none( + getattr(props, "character_set", None), + getattr(props, "characterSet", None), + ) + nchar_set = pick_first_not_none( + getattr(props, "ncharacter_set", None), + getattr(props, "ncharacterSet", None), + ) + + # Auto-scaling flags + auto_scaling_storage = pick_first_not_none( + getattr(props, "is_auto_scaling_for_storage_enabled", None), + getattr(props, "isAutoScalingForStorageEnabled", None), + ) + auto_scaling_enabled = pick_first_not_none( + getattr(props, "is_auto_scaling_enabled", None), + getattr(props, "isAutoScalingEnabled", None), + ) + + # mTLS requirement + mtls_required = pick_first_not_none( + getattr(props, "is_mtls_connection_required", None), + getattr(props, "isMtlsConnectionRequired", None), + ) + + module_name = "adb-azure" + resource_type = "azurerm_oracle_autonomous_database" + resource_name_in_module = "autonomous_database" # Need to change if tf module uses a different name + + adb_tf_name = commonTools.check_tf_variable(adb_name) + + # module.[""].. + tf_address = f'module.{module_name}["{adb_tf_name}"].{resource_type}.{resource_name_in_module}' + + # Avoid duplicate imports by checking current state addresses + if tf_address not in state.get("resources", []): + # Wrap ADDRESS in single quotes to avoid escaping the ["] in a POSIX shell + importCommands["global"] += f"\n{tf_or_tofu} import '{tf_address}' {getattr(adb, 'id', '')}" + + # Populate CD3 columns as per provided header list (write only if column exists) + for col_header in values_for_column: + if col_header in ("Resource Group", "Resource Group", "Resource Group Name"): + values_for_column[col_header].append(rg_name) + elif col_header == "Region": + # If the sheet still has Region, fill it from Azure location (no per-region scripting) + values_for_column[col_header].append(adb_location) + elif col_header == "ADB Display Name": + values_for_column[col_header].append(adb_name) + elif col_header == "Network Details": + # Write the formatted rg@vnet::subnet if available; else leave blank or fall back to raw + values_for_column[col_header].append(formatted_net or "") + elif col_header == "Whitelisted IP Addresses": + values_for_column[col_header].append(whitelisted_ips) + elif col_header == "DB Name": + values_for_column[col_header].append("") # Not exposed in Azure UI/API + elif col_header == "DB Version": + values_for_column[col_header].append(db_version) + elif col_header == "Database Edition": + values_for_column[col_header].append(db_edition) + elif col_header == "Admin Password": + values_for_column[col_header].append("Rand0mPaswd#123") # never retrievable + elif col_header == "Compute Model": + values_for_column[col_header].append(compute_model) + elif col_header == "Compute Count": + values_for_column[col_header].append(compute_count) + elif col_header in ("OCPU Core Count", "OCPU Core Count"): + values_for_column[col_header].append(ocpu_cores) + elif col_header in ("Data Storage Size in TBs", "Data Storage Size in TB"): + values_for_column[col_header].append(storage_tbs) + elif col_header == "Database Workload": + if workload == "DW": + workload = "adw" + elif workload == "AJD": + workload = "json" + elif workload == "OLTP": + workload = "atp" + elif workload == "APEX": + workload = "apex" + values_for_column[col_header].append(workload.upper()) + elif col_header == "License Model": + values_for_column[col_header].append(license_model) + elif col_header == "Backup Retention Period In Days": + values_for_column[col_header].append(backup_retention_days) + elif col_header == 'Character Set': + values_for_column[col_header].append(char_set) + elif col_header == 'nCharacter Set': + values_for_column[col_header].append(nchar_set) + elif col_header == "Auto Scaling for Storage Enabled": + values_for_column[col_header].append(auto_scaling_storage) + elif col_header == "Auto Scaling Enabled": + values_for_column[col_header].append(auto_scaling_enabled) + elif col_header == "MTLS Connection Required": + values_for_column[col_header].append(mtls_required) + elif col_header == "Customer Contacts": + values_for_column[col_header].append(contacts_csv) + elif col_header == "Common Tags": + values_for_column[col_header].append(common_tags) + else: + values_for_column[col_header].append("") + ''' + elif col_header.lower() in azrCommonTools.tagColumns: + try: + values_for_column = commonTools.export_tags(adb, col_header, values_for_column) + except Exception: + values_for_column[col_header].append("") + + else: + # Extra/custom columns via Excel_Columns mapping + try: + oci_objs = [adb] + values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, + values_for_column) + except Exception: + values_for_column[col_header].append("") + ''' + + + +def export_adb_azure(inputfile: str, outdir: str,credentials, + export_resource_groups: Optional[List[str]] = None): + """ + Export ADB @Azure resources into CD3 (existing worksheet) and write Terraform/tofu import commands. + - No region/service_dir/export_tags/compartment scoping. + - Single import script at outdir/azure folder. + """ + global importCommands, sheet_dict + + tf_or_tofu = "terraform" + + # Validate input Excel + cd3file = inputfile + if '.xls' not in cd3file: + print("\nAcceptable cd3 format: .xlsx") + sys.exit(1) + sheetName = "ADB-Azure" + # Read CD3 + df, values_for_column = commonTools.read_cd3(cd3file, sheetName) + + # Get dict for columns from Excel_Columns + + print("\nCD3 excel file should not be opened during export process!!!") + print("Tab- ADB-Azure will be overwritten during export process!!!\n") + + # Prepare a single import commands script at outdir (subscription scope) + + resource = 'import_' + sheetName.lower() + file_name = 'import_commands_' + sheetName.lower() + '.sh' + script_file = os.path.join(outdir, file_name) + if os.path.exists(script_file): + commonTools.backup_file(outdir, resource, file_name) + os.makedirs(outdir, exist_ok=True) + importCommands["global"] = "" + + + client = OracleDBClient(credential=credentials[0], subscription_id=credentials[1]) + + print("\nFetching details of ADB @Azure...") + + # Build state resources (to avoid duplicate import lines) at outdir + state = {'path': outdir, 'resources': []} + try: + tf_state_list = [tf_or_tofu, "state", "list"] + byteOutput = sp.check_output(tf_state_list, cwd=state["path"], stderr=sp.DEVNULL) + output = byteOutput.decode('UTF-8').rstrip() + for item in output.split('\n'): + state["resources"].append(item.replace("\"", "\\\"")) + except Exception: + pass + + # Iterate ADBs in requested RGs or entire subscription + rgs = export_resource_groups if export_resource_groups else None + for adb in _list_adbs(client, rgs): + print_adbs_azure(adb, values_for_column, state, tf_or_tofu) + + # Write back to CD3 + commonTools.write_to_cd3(values_for_column, cd3file, sheetName) + # Region count if present, else any main column (e.g., ADB Display Name) + count_col = "ADB Display Name" if "ADB Display Name" in values_for_column else next(iter(values_for_column.keys())) + print("{0} ADB @Azure exported into CD3.\n".format(len(values_for_column.get(count_col, [])))) + + # Write import script + init_commands = f'\n######### Writing import for ADB @Azure #########\n\n#!/bin/bash\n{tf_or_tofu} init' + if importCommands.get("global"): + importCommands["global"] += f'\n{tf_or_tofu} plan\n' + with open(script_file, 'a', encoding='utf-8') as importCommandsfile: + importCommandsfile.write(init_commands + importCommands["global"]) \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/python/database/templates/adb-azure-template b/cd3_automation_toolkit/azurecloud/python/database/templates/adb-azure-template new file mode 100644 index 000000000..7f8a67f92 --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/python/database/templates/adb-azure-template @@ -0,0 +1,94 @@ +{% if count == 0 %} +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +############################ +# ADB @Azure +# ADB @Azure - tfvars +############################ + +az_oci_adb = { + ##Add New ADB @Azure here## +} +{% else %} + + {{ display_tf_name }} = { + az_region = "{{ region }}" + resource_group_name = "{{ container_id }}" + + display_name = "{{ display_tf_name }}" + db_version = "{{ db_version }}" + license_model = "{{ license_model.split('_')|map('capitalize')|join }}" + admin_password = "{{ admin_password }}" + + network_resource_group_name = "{{ network_container_id }}" + virtual_network_id = "{{ vnet_name }}" + subnet_id = "{{ subnet_id }}" + {% if data_storage_size_in_tbs != "" and data_storage_size_in_tbs != "nan" %} + data_storage_size_in_tbs = {{ data_storage_size_in_tbs }} + {% endif %} + {% if autonomous_value == 'adw' %} + db_workload = "DW" + {% elif autonomous_value == 'atp' %} + db_workload = "OLTP" + {% elif autonomous_value == 'json' %} + db_workload = "AJD" + {% elif autonomous_value == 'apex' %} + db_workload = "APEX" + {% else %} + db_workload = "{{ autonomous_value }}" + {% endif %} + + compute_model = "ECPU" + compute_count = {{ compute_count }} + + {% if character_set != "" and character_set != "nan" %} + character_set = "{{ character_set }}" + {% endif %} + {% if ncharacter_set != "" and ncharacter_set != "nan" %} + ncharacter_set = "{{ ncharacter_set }}" + {% endif %} + + {% if customer_contacts and customer_contacts != "" and customer_contacts != "nan" %} + customer_contacts = {{ customer_contacts.split(",") | map('trim') | list | tojson }} + {% endif %} + + {% if backup_retention_period_in_days != "" and backup_retention_period_in_days != "nan" %} + backup_retention_period_in_days = {{ backup_retention_period_in_days }} + {% endif %} + + {% if auto_scaling_for_storage_enabled != "" and auto_scaling_for_storage_enabled != "nan" %} + auto_scaling_for_storage_enabled = {{ auto_scaling_for_storage_enabled }} + {% endif %} + + {% if auto_scaling_enabled != "" and auto_scaling_enabled != "nan" %} + auto_scaling_enabled = {{ auto_scaling_enabled }} + {% endif %} + + {% if mtls_connection_required != "" and mtls_connection_required != "nan" %} + mtls_connection_required = {{ mtls_connection_required }} + {% endif %} + + + {# ##Do not modify below this line## #} + {# #} + {# ###Section for adding Common Tags### #} + {% if common_tags and common_tags != 'nan' and common_tags != '' and common_tags != [['nan']] %} + {% if common_tags[0] %} + common_tags = { + {% for tags in common_tags %} + {% if not loop.last %} + "{{ tags[0] }}"= "{{ tags[1] }}" , + {% else %} + "{{ tags[0] }}"= "{{ tags[1] }}" + {% endif %} + {% endfor %} + } + {% endif %} + {% endif %} + + {# ###Section for adding Common Tags ends here### #} + }, + +{% endif %} + diff --git a/cd3_automation_toolkit/Database/templates/adb-template b/cd3_automation_toolkit/azurecloud/python/database/templates/adb-gcp-template similarity index 69% rename from cd3_automation_toolkit/Database/templates/adb-template rename to cd3_automation_toolkit/azurecloud/python/database/templates/adb-gcp-template index 18c55e8a2..bd168ed1f 100644 --- a/cd3_automation_toolkit/Database/templates/adb-template +++ b/cd3_automation_toolkit/azurecloud/python/database/templates/adb-gcp-template @@ -3,51 +3,32 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. # ############################ -# ADB -# ADB - tfvars +# ADB @Azure +# ADB @Azure - tfvars # Allowed Values: # compartment_id and network_compartment_id can be the ocid or the name of the compartment hierarchy delimited by double hiphens "--" # Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Database--Prod" where "Database" is the parent of "Prod" compartment ############################ -adb = { - ##Add New ADB for {{ region|lower }} here## +az_oci_adb = { + ##Add New ADB @Azure here## } {% else %} {{ display_tf_name }} = { + az_region = "{{ azure_region }}" + resource_group_name = "{{ container_id }}" - compartment_id = "{{ compartment_name }}" - db_name = "{{ db_name }}" - display_name = "{{ adb_display_name }}" - + display_name = "{{ display_tf_name }}" + db_version = "{{ db_version }}" + license_model = "{{ license_model | replace("_","") }}" admin_password = "{{ admin_password }}" - {% if character_set != "" and character_set != "nan" %} - character_set = "{{ character_set }}" - {% endif %} - - {% if ncharacter_set != "" and ncharacter_set != "nan" %} - ncharacter_set = "{{ ncharacter_set }}" - {% endif %} - - {% if customer_contacts and customer_contacts != "" and customer_contacts != "nan" %} - customer_contacts = {{ customer_contacts | replace("\'","\"") }} - {% endif %} - - cpu_core_count = {{ cpu_core_count }} - - {% if database_edition == "" %} - database_edition = null - {% else %} - database_edition = "{{ database_edition }}" #Only for BYOL license model - {% endif %} - - {% if data_storage_size_in_tb != "" and data_storage_size_in_tb != "nan" %} - data_storage_size_in_tbs = {{ data_storage_size_in_tb }} + virtual_network_id = "{{ virtual_network_id }}" + subnet_id = "{{ subnet_id }}" + {% if data_storage_size_in_tbs != "" and data_storage_size_in_tbs != "nan" %} + data_storage_size_in_tbs = {{ data_storage_size_in_tbs }} {% endif %} - db_version = "19c" - {% if autonomous_value == 'adw' %} db_workload = "DW" {% elif autonomous_value == 'atp' %} @@ -60,26 +41,37 @@ adb = { db_workload = "{{ autonomous_value }}" {% endif %} - license_model = "{{ license_model }}" - whitelisted_ips = [{{ whitelisted_ips }}] + compute_model = "ECPU" + compute_count = {{ compute_count }} - {% if network_compartment_id == "" %} - network_compartment_id = null - {% else %} - network_compartment_id = "{{ network_compartment_id }}" + {% if character_set != "" and character_set != "nan" %} + character_set = "{{ character_set }}" {% endif %} - nsg_ids = [{{ nsg_ids }}] - {% if subnet_id == "" %} - subnet_id = null - {% else %} - subnet_id = "{{ subnet_id }}" + {% if ncharacter_set != "" and ncharacter_set != "nan" %} + ncharacter_set = "{{ ncharacter_set }}" {% endif %} - {% if vcn_name == "" %} - vcn_name = null - {% else %} - vcn_name = "{{ vcn_name }}" + + {% if customer_contacts and customer_contacts != "" and customer_contacts != "nan" %} + customer_contacts = [ {{ customer_contacts | replace("\'","\"") }} ] {% endif %} + {% if backup_retention_period_in_days != "" and backup_retention_period_in_days != "nan" %} + backup_retention_period_in_days = {{ backup_retention_period_in_days }} + {% endif %} + + {% if auto_scaling_for_storage_enabled != "" and auto_scaling_for_storage_enabled != "nan" %} + auto_scaling_for_storage_enabled = {{ auto_scaling_for_storage_enabled }} + {% endif %} + + {% if auto_scaling_enabled != "" and auto_scaling_enabled != "nan" %} + auto_scaling_enabled = {{ auto_scaling_enabled }} + {% endif %} + + {% if mtls_connection_required != "" and mtls_connection_required != "nan" %} + mtls_connection_required = {{ mtls_connection_required }} + {% endif %} + + {# ##Do not modify below this line## #} {# #} {# ###Section for adding Defined and Freeform Tags### #} diff --git a/cd3_automation_toolkit/azurecloud/python/database/templates/exa-infra-azure-template b/cd3_automation_toolkit/azurecloud/python/database/templates/exa-infra-azure-template new file mode 100644 index 000000000..ed4463a08 --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/python/database/templates/exa-infra-azure-template @@ -0,0 +1,59 @@ +{% if count == 0 %} +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +############################ +# Exa Infra @Azure +# Exa Infra @Azure - tfvars +############################ + +az_oci_exa_infra = { + ##Add New Exa-Infra @Azure here## +} +{% else %} + + {{ display_tf_name }} = { + az_region = "{{ region }}" + az_zone = "{{ availability_zone}}" + resource_group_name = "{{ container_id }}" + + display_name = "{{ display_name }}" + compute_count = "{{ database_servers }}" + storage_count = "{{ storage_servers }}" + shape = "{{ shape }}" + + #database_server_type = "{{ database_server_type }}" + #storage_server_type = "{{ storage_server_type }}" + + maintenance_window = { + patching_mode="{{ maintenance_method }}" + preference="NoPreference" + } + + {% if customer_contacts and customer_contacts != "" and customer_contacts != "nan" %} + customer_contacts = {{ customer_contacts.split(",") | map('trim') | list | tojson }} + {% endif %} + + + {# ##Do not modify below this line## #} + {# #} + {# ###Section for adding Common Tags### #} + {% if common_tags and common_tags != 'nan' and common_tags != '' and common_tags != [['nan']] %} + {% if common_tags[0] %} + common_tags = { + {% for tags in common_tags %} + {% if not loop.last %} + "{{ tags[0] }}"= "{{ tags[1] }}" , + {% else %} + "{{ tags[0] }}"= "{{ tags[1] }}" + {% endif %} + {% endfor %} + } + {% endif %} + {% endif %} + + {# ###Section for adding Common Tags ends here### #} + }, + +{% endif %} + diff --git a/cd3_automation_toolkit/azurecloud/python/database/templates/exa-vmcluster-azure-template b/cd3_automation_toolkit/azurecloud/python/database/templates/exa-vmcluster-azure-template new file mode 100644 index 000000000..5bb8c4975 --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/python/database/templates/exa-vmcluster-azure-template @@ -0,0 +1,113 @@ +{% if count == 0 %} +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +############################ +# Exa VM Cluster @Azure +# Exa VM Cluster @Azure - tfvars +############################ + +az_oci_exa_vmclusters = { + ##Add New Exa-VMCluster @Azure here## +} +{% else %} + + {{ display_tf_name }} = { + az_region = "{{ region }}" + resource_group_name = "{{ container_id }}" + + network_resource_group_name = "{{ network_container_id }}" + virtual_network_id = "{{ vnet_name }}" + subnet_id = "{{ subnet_id }}" + + exadata_infrastructure_name = "{{ exadata_infrastructure_name }}" + + display_name = "{{ display_name }}" + hostname = "{{ hostname }}" + gi_version = "{{ gi_version }}" + cpu_core_count = "{{ cpu_core_count }}" + license_model = "{{ license_model.split('_')|map('capitalize')|join }}" + ssh_public_keys = ["{{ ssh_public_keys }}"] + + {% if domain %} + domain = "{{ domain }}" + {% endif %} + {% if oci_zone_id %} + oci_zone_id = "{{ oci_zone_id }}" + {% endif %} + {% if backup_subnet_cidr %} + backup_subnet_cidr = "{{ backup_subnet_cidr }}" + {% endif %} + {% if cluster_name %} + cluster_name = "{{ cluster_name }}" + {% endif %} + + {% if diagnostics_events_enabled %} + diagnostics_events_enabled = {{ diagnostics_events_enabled }} + {% endif %} + {% if health_monitoring_enabled %} + health_monitoring_enabled = {{ health_monitoring_enabled }} + {% endif %} + {% if incident_logs_enabled %} + incident_logs_enabled = {{ incident_logs_enabled }} + {% endif %} + {% if data_storage_percentage %} + data_storage_percentage = {{ data_storage_percentage }} + {% endif %} + {% if data_storage_size_in_tbs %} + data_storage_size_in_tbs = {{ data_storage_size_in_tbs }} + {% endif %} + {% if db_node_storage_size_in_gbs %} + db_node_storage_size_in_gbs = {{ db_node_storage_size_in_gbs }} + {% endif %} + {% if memory_size_in_gbs %} + memory_size_in_gbs = {{ memory_size_in_gbs }} + {% endif %} + {% if time_zone %} + time_zone = "{{ time_zone }}" + {% endif %} + {% if local_backup_enabled %} + local_backup_enabled = {{ local_backup_enabled }} + {% endif %} + {% if sparse_diskgroup_enabled %} + sparse_diskgroup_enabled = {{ sparse_diskgroup_enabled }} + {% endif %} + {% if scan_listener_port_tcp %} + scan_listener_port_tcp = {{ scan_listener_port_tcp }} + {% endif %} + {% if scan_listener_port_tcp_ssl %} + scan_listener_port_tcp_ssl = {{ scan_listener_port_tcp_ssl }} + {% endif %} + {% if system_version %} + system_version = "{{ system_version }}" + {% endif %} + {% if mount_point %} + mount_point = "{{ mount_point }}" + {% endif %} + {% if size_in_gb %} + size_in_gb = {{ size_in_gb }} + {% endif %} + + + {# ##Do not modify below this line## #} + {# #} + {# ###Section for adding Common Tags### #} + {% if common_tags and common_tags != 'nan' and common_tags != '' and common_tags != [['nan']] %} + {% if common_tags[0] %} + common_tags = { + {% for tags in common_tags %} + {% if not loop.last %} + "{{ tags[0] }}"= "{{ tags[1] }}" , + {% else %} + "{{ tags[0] }}"= "{{ tags[1] }}" + {% endif %} + {% endfor %} + } + {% endif %} + {% endif %} + + {# ###Section for adding Common Tags ends here### #} + }, + +{% endif %} + diff --git a/cd3_automation_toolkit/azurecloud/terraform/adb-azure.tf b/cd3_automation_toolkit/azurecloud/terraform/adb-azure.tf new file mode 100644 index 000000000..302ebd44a --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/adb-azure.tf @@ -0,0 +1,93 @@ +#data "azurerm_resource_group" "resource_group" { +# for_each = var.azurerm_oci_adb != null ? var.azurerm_oci_adb : {} +# name = each.value.resource_group_name +#} + +data "azurerm_virtual_network" "virtual_network" { + #depends_on = [module.avm_network] + for_each = var.az_oci_adb != null ? var.az_oci_adb : {} + name = each.value.virtual_network_id + resource_group_name = each.value.network_resource_group_name +} + +data "azurerm_subnet" "subnet" { + #depends_on = [module.avm_network] + for_each = var.az_oci_adb != null ? var.az_oci_adb : {} + name = each.value.subnet_id + virtual_network_name = each.value.virtual_network_id + resource_group_name = each.value.network_resource_group_name +} + +/* +output rg { +value = data.azurerm_resource_group.resource_group["demoadb"].id +} +*/ + +# Azure VNet with delegated subnet +/* +module "avm_network" { + for_each = var.azurerm_oci_adb != null ? var.azurerm_oci_adb : {} + #count = each.value.virtual_network_address_space != "" && each.value.subnet_address_prefix != "" ? 1 : 0 + + source = "Azure/avm-res-network-virtualnetwork/azurerm" + version = "0.5.0" + + # depends_on = [ module.azure-resource-grp ] + + tags = each.value.common_tags + resource_group_name = each.value.resource_group_name + location = each.value.az_region + name = each.value.virtual_network_id + address_space = each.value.virtual_network_address_space + + subnets = { + delegated = { + name = each.value.subnet_id + address_prefixes = each.value.subnet_address_prefix + + delegation = [{ + name = "Oracle.Database/networkAttachments" + service_delegation = { + name = "Oracle.Database/networkAttachments" + actions = ["Microsoft.Network/networkinterfaces/*", "Microsoft.Network/virtualNetworks/subnets/join/action"] + + } + }] + } + } +} +*/ + +# Oracle Autonomous Database@Azure +module "adb-azure" { + for_each = var.az_oci_adb != null ? var.az_oci_adb : {} + + # depends_on = [ module.azure-resource-grp ] + source = "./modules/azurerm-oci-adb" + name = each.value.display_name + #resource_group_name = data.azurerm_resource_group.resource_group[each.key].id + virtual_network_id = data.azurerm_virtual_network.virtual_network[each.key].id + subnet_id = data.azurerm_subnet.subnet[each.key].id + #network_resource_group_name = each.value.network_resource_group_name + + resource_group_name = each.value.resource_group_name + location = each.value.az_region + display_name = each.value.display_name + db_workload = each.value.db_workload + mtls_connection_required = each.value.mtls_connection_required + backup_retention_period_in_days = each.value.backup_retention_period_in_days + compute_model = each.value.compute_model + data_storage_size_in_tbs = each.value.data_storage_size_in_tbs + auto_scaling_for_storage_enabled = each.value.auto_scaling_for_storage_enabled + + admin_password = each.value.admin_password + auto_scaling_enabled = each.value.auto_scaling_enabled + character_set = each.value.character_set + compute_count = each.value.compute_count + ncharacter_set = each.value.ncharacter_set + license_model = each.value.license_model + db_version = each.value.db_version + customer_contacts = each.value.customer_contacts + tags = each.value.common_tags +} diff --git a/cd3_automation_toolkit/azurecloud/terraform/exadata-azure.tf b/cd3_automation_toolkit/azurecloud/terraform/exadata-azure.tf new file mode 100644 index 000000000..fffc6b2f1 --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/exadata-azure.tf @@ -0,0 +1,125 @@ +#data "azurerm_resource_group" "resource_group" { +# for_each = var.azurerm_oci_adb != null ? var.azurerm_oci_adb : {} +# name = each.value.resource_group_name +#} + +data "azurerm_virtual_network" "exa_vmc_virtual_networks" { + #depends_on = [module.avm_network] + for_each = var.az_oci_exa_vmclusters != null ? var.az_oci_exa_vmclusters : {} + name = each.value.virtual_network_id + resource_group_name = each.value.network_resource_group_name +} + +data "azurerm_subnet" "exa_vmc_subnets" { + #depends_on = [module.avm_network] + for_each = var.az_oci_exa_vmclusters != null ? var.az_oci_exa_vmclusters : {} + name = each.value.subnet_id + virtual_network_name = each.value.virtual_network_id + resource_group_name = each.value.network_resource_group_name +} + +data "azurerm_oracle_exadata_infrastructure" "exa_infras" { +depends_on = [ module.exa-infra-azure ] + #depends_on = [module.avm_network] + for_each = var.az_oci_exa_vmclusters != null ? var.az_oci_exa_vmclusters : {} + name = each.value.exadata_infrastructure_name + resource_group_name = each.value.resource_group_name +} + +data "azurerm_oracle_db_servers" "this" { + depends_on = [ module.exa-infra-azure] + for_each = var.az_oci_exa_vmclusters != null ? var.az_oci_exa_vmclusters : {} + resource_group_name = each.value.resource_group_name + cloud_exadata_infrastructure_name = each.value.exadata_infrastructure_name +} + +# AzureRM - Exadata Infrastructure +module "exa-infra-azure" { + for_each = var.az_oci_exa_infra != null ? var.az_oci_exa_infra : {} + source = "./modules/azurerm-oci-exa-infra" + # depends_on = [ module.azure-resource-grp ] + + # Mandatory + location = each.value.az_region + zone = each.value.az_zone + resource_group_name = each.value.resource_group_name + name = each.value.display_name + compute_count = each.value.compute_count + storage_count = each.value.storage_count + + # Optional + shape = each.value.shape + tags = each.value.common_tags + customer_contacts = each.value.customer_contacts + maintenance_window = each.value.maintenance_window + +} + + + +# Known Issue - https://docs.oracle.com/en-us/iaas/odexa/odexa-troubleshooting-and-known-issues-exadata-services.html +#resource "time_sleep" "wait_after_deletion" { +# destroy_duration = var.destroy_duration +# depends_on = [module.azurerm_exadata_infra] +#} + +# AzureRM - Exadata VM Cluster +module "exa-vmcluster-azure" { + + for_each = var.az_oci_exa_vmclusters != null ? var.az_oci_exa_vmclusters : {} + source = "./modules/azurerm-oci-exa-vmcluster" + + # VM Cluster details + resource_group_name = each.value.resource_group_name + display_name = each.value.display_name + + exadata_infrastructure_id = data.azurerm_oracle_exadata_infrastructure.exa_infras[each.key].id + exadata_infrastructure_name = each.value.exadata_infrastructure_name + db_servers = [for obj in data.azurerm_oracle_db_servers.this[each.key].db_servers : obj.ocid] + + location = each.value.az_region + cluster_name = each.value.cluster_name + hostname = each.value.hostname + time_zone = each.value.time_zone + license_model = each.value.license_model + gi_version = each.value.gi_version + system_version = each.value.system_version + ssh_public_keys = each.value.ssh_public_keys + + # Networking + vnet_id = data.azurerm_virtual_network.exa_vmc_virtual_networks[each.key].id + subnet_id = data.azurerm_subnet.exa_vmc_subnets[each.key].id + backup_subnet_cidr = each.value.backup_subnet_cidr + domain = each.value.domain + zone_id = each.value.oci_zone_id + + # VM Cluster allocation + cpu_core_count = each.value.cpu_core_count + memory_size_in_gbs = each.value.memory_size_in_gbs + dbnode_storage_size_in_gbs = each.value.db_node_storage_size_in_gbs + + # Exadata storage + data_storage_size_in_tbs = each.value.data_storage_size_in_tbs + data_storage_percentage = each.value.data_storage_percentage + is_local_backup_enabled = each.value.local_backup_enabled + is_sparse_diskgroup_enabled = each.value.sparse_diskgroup_enabled + + # Diagnostics Collection + is_diagnostic_events_enabled = each.value.diagnostics_events_enabled + is_health_monitoring_enabled = each.value.health_monitoring_enabled + is_incident_logs_enabled = each.value.incident_logs_enabled + + # Ports + scan_listener_port_tcp = each.value.scan_listener_port_tcp + scan_listener_port_tcp_ssl = each.value.scan_listener_port_tcp_ssl + + # File System Config + mount_point = each.value.mount_point + size_in_gb = each.value.size_in_gb + + + tags = each.value.common_tags + #depends_on = [time_sleep.wait_after_deletion] + # depends_on = [module.azurerm_exadata_infra] + +} diff --git a/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-adb/main.tf b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-adb/main.tf new file mode 100644 index 000000000..a2605b35c --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-adb/main.tf @@ -0,0 +1,59 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +##################################### +## Resource Block - Oracle ADB @Azure +## Create Oracle ADB @Azure +##################################### + +resource "azurerm_oracle_autonomous_database" "autonomous_database" { + name = var.name + resource_group_name = var.resource_group_name + location = var.location + subnet_id = var.subnet_id + display_name = var.display_name + db_workload = var.db_workload + mtls_connection_required = var.mtls_connection_required + backup_retention_period_in_days = var.backup_retention_period_in_days + compute_model = var.compute_model + data_storage_size_in_tbs = var.data_storage_size_in_tbs + auto_scaling_for_storage_enabled = var.auto_scaling_for_storage_enabled + virtual_network_id = var.virtual_network_id + admin_password = var.admin_password + auto_scaling_enabled = var.auto_scaling_enabled + character_set = var.character_set + compute_count = var.compute_count + national_character_set = var.ncharacter_set + license_model = var.license_model + db_version = var.db_version + customer_contacts = var.customer_contacts + tags = var.tags + lifecycle { + ignore_changes = [ + name, + display_name, + db_workload, + mtls_connection_required, + #backup_retention_period_in_days, + compute_model, + #data_storage_size_in_tbs, + #auto_scaling_for_storage_enabled, + #auto_scaling_enabled, + character_set, + admin_password, + # compute_count, + national_character_set, + license_model, + db_version, + customer_contacts + ] + } +} + +/* +data "azurerm_oracle_autonomous_database" "this" { + depends_on = [ azurerm_oracle_autonomous_database.this ] + name = azurerm_oracle_autonomous_database.this.name + resource_group_name = azurerm_oracle_autonomous_database.this.resource_group_name +} +*/ \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-adb/outputs.tf b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-adb/outputs.tf new file mode 100644 index 000000000..eaddf386e --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-adb/outputs.tf @@ -0,0 +1,37 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +##################################### +## Outputs Block - Oracle ADB @Azure +## Create Oracle ADB @Azure +##################################### + +# Azure info +output "resource_id" { + description = "Resource ID of Autonomous Database in Azure" + value = azurerm_oracle_autonomous_database.autonomous_database.id +} + +/* +output "resource" { + description = "Resource Object of Autonomous Database in Azure" + value = data.azurerm_oracle_autonomous_database.autonomous_database +} + + +# OCI info +output "oci_adbs_ocid" { + description = "OCID of Autonomous Database in OCI" + value = regex("(?:/adbs/)([^?&/]+)",data.azurerm_oracle_autonomous_database.this.oci_url)[0] +} + +output "oci_region" { + description = "Region of the Autonomous Database in OCI" + value = regex("(?:region=)([^?&/]+)",data.azurerm_oracle_autonomous_database.this.oci_url)[0] +} + +output "oci_compartment_ocid" { + description = "Compartment OCID of the Autonomous Database in OCI" + value = regex("(?:compartmentId=)([^?&/]+)",data.azurerm_oracle_autonomous_database.this.oci_url)[0] +} +*/ \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-adb/provider_req.tf b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-adb/provider_req.tf new file mode 100644 index 000000000..f51a8c14b --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-adb/provider_req.tf @@ -0,0 +1,14 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +terraform { + required_providers { + oci = { + source = "oracle/oci" + } + azurerm = { + source = "hashicorp/azurerm" + version = ">=4.9.0" + } + } +} \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-adb/variables.tf b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-adb/variables.tf new file mode 100644 index 000000000..6d2c697b3 --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-adb/variables.tf @@ -0,0 +1,126 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +##################################### +## Variables Block - Oracle ADB @Azure +## Create Oracle ADB @Azure +##################################### + +variable "name" { + description = "Azure resource name which should be used for this Autonomous Database." + type = string +} + +variable "resource_group_name" { + description = "The name of Resource Group in Azure" + type = string +} + +variable "location" { + description = "The Azure Region where the Autonomous Database should exist. Changing this forces a new Autonomous Database to be created" + type = string +} + +variable "subnet_id" { + description = "The ID of the subnet the resource is associated with." + type = string +} + +variable "display_name" { + description = "The user-friendly name for the Autonomous Database in OCI. The name does not have to be unique." + type = string +} + +variable "db_workload" { + description = "The Autonomous Database workload type. The following values are valid: OLTP, DW, AJD, APEX" + type = string + default = "OLTP" +} + +variable "mtls_connection_required" { + description = "Specifies if the Autonomous Database requires mTLS connections." + type = bool + default = false +} +variable "backup_retention_period_in_days" { + description = "Retention period, in days, for backups." + type = number + default = 60 +} + +variable "compute_model" { + description = "The compute model of the Autonomous Database. This is required if using the computeCount parameter. If using cpuCoreCount then it is an error to specify computeModel to a non-null value. ECPU compute model is the recommended model and OCPU compute model is legacy." + type = string + default = "ECPU" +} + +variable "data_storage_size_in_tbs" { + description = "The maximum storage that can be allocated for the database, in terabytes." + type = number + default = 1 +} + +variable "auto_scaling_for_storage_enabled" { + description = "Indicates if auto scaling is enabled for the Autonomous Database storage. The default value is false." + type = bool + default = false +} + +variable "virtual_network_id" { + description = "The ID of the vnet associated with the Autonomous Database." + type = string +} + +variable "admin_password" { + description = "The password must be between 12 and 30characters long, and must contain at least 1 uppercase, 1 lowercase, and 1 numeric character. It cannot contain the double quote symbol or the username 'admin', regardless of casing." + type = string + sensitive = true +} + +variable "auto_scaling_enabled" { + description = " Indicates if auto scaling is enabled for the Autonomous Database CPU core count. The default value is true." + type = bool + default = true +} + +variable "character_set" { + description = "The character set for the autonomous database. The default is AL32UTF8" + type = string + default = "AL32UTF8" +} + +variable "compute_count" { + description = "The compute amount (CPUs) available to the database. Minimum and maximum values depend on the compute model and whether the database is an Autonomous Database Serverless instance or an Autonomous Database on Dedicated Exadata Infrastructure. For an Autonomous Database Serverless instance, the ECPU compute model requires a minimum value of one, for databases in the elastic resource pool and minimum value of two, otherwise. Required when using the computeModel parameter. When using cpuCoreCount parameter, it is an error to specify computeCount to a non-null value. Providing computeModel and computeCount is the preferred method for both OCPU and ECPU." + type = number + default = 2 +} + +variable "ncharacter_set" { + description = "The national character set for the autonomous database. The default is AL16UTF16. Allowed values are: AL16UTF16 or UTF8." + type = string + default = "AL16UTF16" +} + +variable "license_model" { + description = "The Oracle license model that applies to the Oracle Autonomous Database. Bring your own license (BYOL) allows you to apply your current on-premises Oracle software licenses to equivalent, highly automated Oracle services in the cloud. License Included allows you to subscribe to new Oracle Database software licenses and the Oracle Database service. Note that when provisioning an Autonomous Database on dedicated Exadata infrastructure, this attribute must be null. It is already set at the Autonomous Exadata Infrastructure level. When provisioning an Autonomous Database Serverless database, if a value is not specified, the system defaults the value to BRING_YOUR_OWN_LICENSE. Bring your own license (BYOL) also allows you to select the DB edition using the optional parameter." + type = string + default = "LicenseIncluded" +} + +variable "db_version" { + description = "A valid Oracle Database version for Autonomous Database." + type = string + default = "19c" +} + +variable "customer_contacts" { + description = "The email address used by Oracle to send notifications regarding databases and infrastructure. Provide up to 10 unique maintenance contact email addresses." + type = list(string) + default = [] +} + +variable "tags" { + description = "Resource tags for the Cloud Exadata Infrastructure" + type = map(string) + default = null +} \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-infra/main.tf b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-infra/main.tf new file mode 100644 index 000000000..a9db112aa --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-infra/main.tf @@ -0,0 +1,56 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +##################################### +## Resource Block - Oracle ExaInfra @Azure +## Create Oracle ExaInfra @Azure +##################################### + +resource "azurerm_oracle_exadata_infrastructure" "exadata_infrastructure" { + # Required + resource_group_name = var.resource_group_name + location = var.location + zones = [var.zone] + + name = var.name + display_name = var.name + + shape = var.shape + compute_count = var.compute_count + storage_count = var.storage_count + + #database_server_type = var.database_server_type + #storage_server_type = var.storage_server_type + + # Optional + customer_contacts = var.customer_contacts + tags = var.tags + + maintenance_window { + patching_mode = var.maintenance_window.patching_mode + preference = var.maintenance_window.preference + lead_time_in_weeks = coalesce(var.maintenance_window.lead_time_in_weeks,1) + months = coalesce(var.maintenance_window.months,[]) + weeks_of_month = coalesce(var.maintenance_window.weeks_of_month,[]) + days_of_week = coalesce(var.maintenance_window.days_of_week,[]) + hours_of_day = coalesce(var.maintenance_window.hours_of_day,[]) + } + + lifecycle { + ignore_changes = [ + # Updatable from OCI + compute_count, + storage_count, + database_server_type, + storage_server_type, + customer_contacts, + maintenance_window + ] + } +} + +data "azurerm_oracle_exadata_infrastructure" "exadata_infrastructures" { + name = var.name + resource_group_name = var.resource_group_name + depends_on = [azurerm_oracle_exadata_infrastructure.exadata_infrastructure] +} \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-infra/outputs.tf b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-infra/outputs.tf new file mode 100644 index 000000000..75382e5d2 --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-infra/outputs.tf @@ -0,0 +1,28 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +##################################### +## Outputs Block - Oracle ExaInfra @Azure +## Create Oracle ExaInfra @Azure +##################################### + +output "resource_id" { + description = "Resource ID of Exadata Infrastructure in Azure" + value = azurerm_oracle_exadata_infrastructure.exadata_infrastructure.id +} + +/* +output "resource" { + description = "Resource Object of Exadata Infrastructure in Azure" + value = azurerm_oracle_exadata_infrastructure.this +} +*/ +output "oci_region" { + description = "Region of the Exadata Infrastructure in OCI" + value = regex("(?:region=)([^?&/]+)",data.azurerm_oracle_exadata_infrastructure.exadata_infrastructures.oci_url)[0] +} + +output "oci_compartment_ocid" { + description = "Compartment OCID of the Exadata Infrastructure in OCI" + value = regex("(?:compartmentId=)([^?&/]+)",data.azurerm_oracle_exadata_infrastructure.exadata_infrastructures.oci_url)[0] +} \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-infra/provider_req.tf b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-infra/provider_req.tf new file mode 100644 index 000000000..f51a8c14b --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-infra/provider_req.tf @@ -0,0 +1,14 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +terraform { + required_providers { + oci = { + source = "oracle/oci" + } + azurerm = { + source = "hashicorp/azurerm" + version = ">=4.9.0" + } + } +} \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-infra/variables.tf b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-infra/variables.tf new file mode 100644 index 000000000..dce66362a --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-infra/variables.tf @@ -0,0 +1,91 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +##################################### +## Variables Block - Oracle ExaInfra @Azure +## Create Oracle ExaInfra @Azure +##################################### + +# Mandatory +variable "location" { + description = "The name of Azure Region where the Exadata Infrastructure should be. e.g. useast" + type = string +} + +variable "name" { + description = "The name of the Exadata Infrastructure at Azure" + type = string + default = "odaaz-infra" +} + +variable "resource_group_name" { + description = "The name of Resource Group in Azure" + type = string + default = "rg-oradb" +} + +variable "zone" { + description = "The availablty zone of the Exadata Infrastructure in Azure" + type = string +} + +variable "compute_count" { + description = "The number of compute servers for the Exadata infrastructure." + type = number + default = 2 +} + +variable "storage_count" { + description = "The number of storage servers for the Exadata infrastructure." + type = number + default = 3 +} + +variable "shape" { + description = "The shape of the Exadata infrastructure resource. e.g. Exadata.X9M" + type = string + default = "Exadata.X9M" +} +/* +variable "database_server_type" { + description = "Database Server Type" + type = string + default = "X11M" +} + +variable "storage_server_type" { + description = "Storage Server Type" + type = string + default = "X11M-HC" +} +*/ + +# Optional +variable "customer_contacts" { + description = "The email address used by Oracle to send notifications regarding databases and infrastructure. Provide up to 10 unique maintenance contact email addresses." + type = list(string) + default = [] +} + +variable "maintenance_window" { + description = "maintenanceWindow properties" + type = object({ + patching_mode = string + preference = string + lead_time_in_weeks = optional(number) + months = optional(list(number)) + weeks_of_month = optional(list(number)) + days_of_week =optional(list(number)) + hours_of_day = optional(list(number)) + }) + default = { + patching_mode = "Rolling" + preference = "NoPreference" + } +} + +variable "tags" { + description = "Resource tags for the Cloud Exadata Infrastructure" + type = map(string) + default = null +} \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-vmcluster/main.tf b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-vmcluster/main.tf new file mode 100644 index 000000000..812b141e6 --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-vmcluster/main.tf @@ -0,0 +1,97 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +##################################### +## Resource Block - Oracle ExaVM Cluster @Azure +## Create Oracle ExaVM Cluster @Azure +##################################### + +resource "azurerm_oracle_cloud_vm_cluster" "vm_cluster" { + + # VM Cluster details + resource_group_name = var.resource_group_name + location = var.location + cloud_exadata_infrastructure_id = var.exadata_infrastructure_id + cluster_name = var.cluster_name + name = var.display_name + display_name = var.display_name + hostname = var.hostname + # hostname_actual = var.hostname_actual != null ? var.hostname_actual : null + time_zone = var.time_zone + license_model = var.license_model + gi_version = var.gi_version + system_version = var.system_version + + ssh_public_keys = var.ssh_public_keys + db_servers = var.db_servers + + # Networking + virtual_network_id = var.vnet_id + subnet_id = var.subnet_id + backup_subnet_cidr = var.backup_subnet_cidr + domain = var.domain != "" ? var.domain : null + zone_id = var.zone_id != "" ? var.zone_id : null + + # VM Cluster allocation + cpu_core_count = var.cpu_core_count + memory_size_in_gbs = var.memory_size_in_gbs + db_node_storage_size_in_gbs = var.dbnode_storage_size_in_gbs + + # Exadata storage + data_storage_size_in_tbs = var.data_storage_size_in_tbs + data_storage_percentage= var.data_storage_percentage + local_backup_enabled = var.is_local_backup_enabled + sparse_diskgroup_enabled = var.is_sparse_diskgroup_enabled + + # Diagnostics Collection + data_collection_options { + diagnostics_events_enabled = var.is_diagnostic_events_enabled + health_monitoring_enabled = var.is_health_monitoring_enabled + incident_logs_enabled = var.is_incident_logs_enabled + } + + # Ports + scan_listener_port_tcp = var.scan_listener_port_tcp + scan_listener_port_tcp_ssl = var.scan_listener_port_tcp_ssl + + file_system_configuration { + mount_point = var.mount_point + size_in_gb = var.size_in_gb + } + + tags = var.tags + + lifecycle { + ignore_changes = [ + # For Idempotency + id, + cluster_name, + hostname, + subnet_id, + backup_subnet_cidr, + gi_version, + system_version, + + # Updatable from OCI + license_model, + ssh_public_keys, + db_servers, + cpu_core_count, + memory_size_in_gbs, + db_node_storage_size_in_gbs, + data_storage_size_in_tbs, + ] + } +} + +# Lookup OCID of VM Cluster for output +resource "time_sleep" "wait_10s" { + create_duration = "10s" + depends_on = [azurerm_oracle_cloud_vm_cluster.vm_cluster] +} + +data "azurerm_oracle_cloud_vm_cluster" "vm_clusters" { + depends_on = [ time_sleep.wait_10s ] + name = azurerm_oracle_cloud_vm_cluster.vm_cluster.name + resource_group_name = var.resource_group_name +} \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-vmcluster/outputs.tf b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-vmcluster/outputs.tf new file mode 100644 index 000000000..eb77042cd --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-vmcluster/outputs.tf @@ -0,0 +1,47 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +##################################### +## Outputs Block - Oracle ExaVM Cluster @Azure +## Create Oracle ExaVM Cluster @Azure +##################################### + +output "resource" { + description = "Resource Object of VM Cluster in Azure" + value = data.azurerm_oracle_cloud_vm_cluster.vm_clusters +} + +output "resource_id" { + description = "Resource ID of the VM Cluster in Azure" + value = azurerm_oracle_cloud_vm_cluster.vm_cluster.id +} + +output "vm_cluster_ocid" { + description = "OCID of the VM Cluster in OCI" + value = data.azurerm_oracle_cloud_vm_cluster.vm_clusters.ocid +} + +output "vm_cluster_hostname_actual" { + description = "The actual hostname of the VM Cluster after provision" + value = data.azurerm_oracle_cloud_vm_cluster.vm_clusters.hostname_actual +} + +output "oci_region" { + description = "Region of the VM Cluster in OCI" + value = regex("(?:region=)([^?&/]+)",data.azurerm_oracle_cloud_vm_cluster.vm_clusters.oci_url)[0] +} + +output "oci_compartment_ocid" { + description = "Compartment OCID of the VM Cluster in OCI" + value = regex("(?:compartmentId=)([^?&/]+)",data.azurerm_oracle_cloud_vm_cluster.vm_clusters.oci_url)[0] +} + +output "oci_vcn_ocid" { + description = "OCID of the Virtual Cloud Network (VCN)in OCI" + value = regex("(?:networking/vcns/)([^?&/]+)",data.azurerm_oracle_cloud_vm_cluster.vm_clusters.nsg_url)[0] +} + +output "oci_nsg_ocid" { + description = "OCID of the Network Security Group (NSG) in OCI" + value = regex("(?:network-security-groups/)([^?&/]+)",data.azurerm_oracle_cloud_vm_cluster.vm_clusters.nsg_url)[0] +} \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-vmcluster/provider_req.tf b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-vmcluster/provider_req.tf new file mode 100644 index 000000000..f51a8c14b --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-vmcluster/provider_req.tf @@ -0,0 +1,14 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +terraform { + required_providers { + oci = { + source = "oracle/oci" + } + azurerm = { + source = "hashicorp/azurerm" + version = ">=4.9.0" + } + } +} \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-vmcluster/variables.tf b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-vmcluster/variables.tf new file mode 100644 index 000000000..84cef2e43 --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/modules/azurerm-oci-exa-vmcluster/variables.tf @@ -0,0 +1,178 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +##################################### +## Variables Block - Oracle ExaVM Cluster @Azure +## Create Oracle ExaVM Cluster @Azure +##################################### + +# https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/oracle_cloud_vm_cluster + +variable "resource_group_name" { + description = "The name of the Resource Group where the Cloud VM Cluster should exist" + type = string +} +variable "location" { + description = "The Azure Region where the Cloud VM Cluster should exist." + type = string +} +variable "exadata_infrastructure_id" { + description = "The OCID of the Cloud Exadata infrastructure." + type = string +} +variable "exadata_infrastructure_name" { + description = "The name of the Cloud Exadata infrastructure." + type = string +} +variable "cluster_name" { + description = "The cluster name for cloud VM cluster. The cluster name must begin with an alphabetic character, and may contain hyphens (-). Underscores (_) are not permitted. The cluster name can be no longer than 11 characters and is not case sensitive." + type = string + default = null +} +variable "display_name" { + description = "The display name for cloud VM cluster." + type = string + default = null +} +variable "hostname" { + description = "The prefix forms the first portion of the Exadata VM Cluster host name. Recommended maximum: 12 characters." + type = string + default = null +} + +variable "time_zone" { + description = "The time zone of the Cloud VM Cluster. For details, see Exadata Infrastructure Time Zones." + type = string + default = null +} +variable "license_model" { + description = "The Oracle license model that applies to the Cloud VM Cluster, either BringYourOwnLicense or LicenseIncluded." + type = string + default = "LicenseIncluded" +} +variable "gi_version" { + description = "A valid Oracle Grid Infrastructure (GI) software version." + type = string +} +variable "system_version" { + description = "Operating system version of the Exadata image." + type = string + default = null +} +variable "db_servers" { + description = "DB Serverset." + type = list(string) + default = null +} +variable "ssh_public_keys" { + description = "The public key portion of one or more key pairs used for SSH access to the Cloud VM Cluster." + type = list(string) +} +variable "tags" { + description = "A mapping of tags which should be assigned to the Cloud VM Cluster." + type = map(string) + default = null +} + +variable "vnet_id" { + description = "The ID of the Virtual Network associated with the Cloud VM Cluster." + type = string +} + +variable "subnet_id" { + description = "The ID of the subnet associated with the Cloud VM Cluster." + type = string +} + +variable "backup_subnet_cidr" { + description = "The backup subnet CIDR of the Virtual Network associated with the Cloud VM Cluster." + type = string + default = null +} + +variable "cpu_core_count" { + description = "The number of CPU cores enabled on the Cloud VM Cluster." + type = string +} + +variable "memory_size_in_gbs" { + description = "The memory to be allocated in GBs." + type = number + default = null +} + +variable "dbnode_storage_size_in_gbs" { + description = "The local node storage to be allocated in GBs." + type = number + default = null +} + +variable "data_storage_size_in_tbs" { + description = "The data disk group size to be allocated in TBs." + type = number + default = null +} +variable "data_storage_percentage" { + description = "The percentage assigned to DATA storage (user data and database files). The remaining percentage is assigned to RECO storage (database redo logs, archive logs, and recovery manager backups). Accepted values are 35, 40, 60 and 80." + type = number + default = null +} +variable "is_local_backup_enabled" { + description = "If true, database backup on local Exadata storage is configured for the Cloud VM Cluster. If false, database backup on local Exadata storage is not available in the Cloud VM Cluster." + type = bool + default = null +} +variable "is_sparse_diskgroup_enabled" { + description = "If true, the sparse disk group is configured for the Cloud VM Cluster. If false, the sparse disk group is not created." + type = bool + default = null +} +variable "is_diagnostic_events_enabled" { + description = "Indicates whether diagnostic collection is enabled for the Cloud VM Cluster. Enabling diagnostic collection allows you to receive Events service notifications for guest VM issues. Diagnostic collection also allows Oracle to provide enhanced service and proactive support for your Exadata system. You can enable diagnostic collection during VM Cluster/Cloud VM Cluster provisioning." + type = bool + default = false +} +variable "is_health_monitoring_enabled" { + description = "Indicates whether health monitoring is enabled for the Cloud VM Cluster. Enabling health monitoring allows Oracle to collect diagnostic data and share it with its operations and support personnel. You may also receive notifications for some events. Collecting health diagnostics enables Oracle to provide proactive support and enhanced service for your system. Optionally enable health monitoring while provisioning a system. " + type = bool + default = false +} +variable "is_incident_logs_enabled" { + description = "Indicates whether incident logs and trace collection are enabled for the Cloud VM Cluster. Enabling incident logs collection allows Oracle to receive Events service notifications for guest VM issues, collect incident logs and traces, and use them to diagnose issues and resolve them. Optionally enable incident logs collection while provisioning a system. " + type = bool + default = false +} +variable "domain" { + description = "The name of the existing OCI Private DNS Zone to be associated with the Cloud VM Cluster. This allow you to specify your own private domain name instead of the default OCI DNS zone (oraclevcn.com)" + type = string + default = "" +} +variable "zone_id" { + description = "The OCID of the existing OCI Private DNS Zone to be associated with the Cloud VM Cluster. This allow you to specify your own private domain name instead of the default OCI DNS zone (oraclevcn.com)" + type = string + default = "" +} + +variable "mount_point" { + description = "file mount path" + type = string + default = "" +} + +variable "size_in_gb" { + description = "file mount path size in gb" + type = string + default = "" +} + +variable "scan_listener_port_tcp" { + description = "listener port" + type = number + default = 1521 +} + +variable "scan_listener_port_tcp_ssl" { + description = "listener port ssl" + type = number + default = 2484 +} \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/terraform/provider.tf b/cd3_automation_toolkit/azurecloud/terraform/provider.tf new file mode 100644 index 000000000..70b6b0cc7 --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/provider.tf @@ -0,0 +1,9 @@ +# Azure RM Terraform Provider +provider "azurerm" { + features {} + + subscription_id = "" + tenant_id = "" + client_id = "" + client_secret = "" +} \ No newline at end of file diff --git a/cd3_automation_toolkit/azurecloud/terraform/variables.tf b/cd3_automation_toolkit/azurecloud/terraform/variables.tf new file mode 100644 index 000000000..d25fc24c6 --- /dev/null +++ b/cd3_automation_toolkit/azurecloud/terraform/variables.tf @@ -0,0 +1,118 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +############################ +# +# Variables Block +# Azure OCI +# +############################ + + +########################### +###Oracle ADB @Azure ###### +########################### + +variable "az_oci_adb" { + type = map(object({ + display_name = string + az_region = string + resource_group_name = string + network_resource_group_name = string + virtual_network_id = string + subnet_id = string + db_workload = string + mtls_connection_required = bool + backup_retention_period_in_days = number + compute_model = string + data_storage_size_in_tbs = number + auto_scaling_for_storage_enabled = bool + admin_password = string + auto_scaling_enabled = bool + character_set = string + compute_count = number + ncharacter_set = string + license_model = string + db_version = string + customer_contacts = optional(list(string)) + common_tags = optional(map(string)) + })) + default = {} +} + + +########################### +###Oracle ExaInfra @Azure ###### +########################### + +variable "az_oci_exa_infra" { + type = map(object({ + display_name = string + az_region = string + az_zone = string + resource_group_name = string + + compute_count = number + storage_count = number + shape = string + database_server_type = optional(string) + storage_server_type = optional(string) + + maintenance_window = object({ + patching_mode = string + preference = string + lead_time_in_weeks = optional(number) + months = optional(list(number)) + weeks_of_month = optional(list(number)) + days_of_week = optional(list(number)) + hours_of_day = optional(list(number)) + }) + customer_contacts = optional(list(string)) + common_tags = optional(map(string)) + + })) + default = {} +} + + +variable "az_oci_exa_vmclusters" { + type = map(object({ + display_name = string + az_region = string + resource_group_name = string + network_resource_group_name = string + virtual_network_id = string + subnet_id = string + exadata_infrastructure_name = string + hostname = string + cpu_core_count = string + gi_version = string + license_model = string + ssh_public_keys = list(string) + gi_version = string + backup_subnet_cidr = optional(string) + cluster_name = optional(string) + domain = optional(string) + oci_zone_id = optional(string) + diagnostics_events_enabled = optional(bool) + health_monitoring_enabled = optional(bool) + incident_logs_enabled = optional(bool) + data_storage_percentage = optional(number) + data_storage_size_in_tbs = optional(number) + db_node_storage_size_in_gbs = optional(number) + local_backup_enabled = optional(bool) + sparse_diskgroup_enabled = optional(bool) + memory_size_in_gbs = optional(number) + scan_listener_port_tcp = optional(number) + scan_listener_port_tcp_ssl = optional(number) + system_version = optional(string) + time_zone = optional(string) + mount_point = optional(string) + size_in_gb = optional(number) + common_tags = optional(map(string)) + + + })) + default = {} +} + diff --git a/cd3_automation_toolkit/user-scripts/OPA/Compute/oci_restrict_instance_types.rego b/cd3_automation_toolkit/common/opa/Compute/oci_restrict_instance_types.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Compute/oci_restrict_instance_types.rego rename to cd3_automation_toolkit/common/opa/Compute/oci_restrict_instance_types.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Compute/oci_secure_vm_access.rego b/cd3_automation_toolkit/common/opa/Compute/oci_secure_vm_access.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Compute/oci_secure_vm_access.rego rename to cd3_automation_toolkit/common/opa/Compute/oci_secure_vm_access.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Identity/deny_api_keys_admin_group.rego b/cd3_automation_toolkit/common/opa/Identity/deny_api_keys_admin_group.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Identity/deny_api_keys_admin_group.rego rename to cd3_automation_toolkit/common/opa/Identity/deny_api_keys_admin_group.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Identity/deny_iam_admin.rego b/cd3_automation_toolkit/common/opa/Identity/deny_iam_admin.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Identity/deny_iam_admin.rego rename to cd3_automation_toolkit/common/opa/Identity/deny_iam_admin.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Identity/deny_storage_admins_delete_resources.rego b/cd3_automation_toolkit/common/opa/Identity/deny_storage_admins_delete_resources.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Identity/deny_storage_admins_delete_resources.rego rename to cd3_automation_toolkit/common/opa/Identity/deny_storage_admins_delete_resources.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Identity/dynamic_groups_check.rego b/cd3_automation_toolkit/common/opa/Identity/dynamic_groups_check.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Identity/dynamic_groups_check.rego rename to cd3_automation_toolkit/common/opa/Identity/dynamic_groups_check.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Identity/enable_service_level_admins.rego b/cd3_automation_toolkit/common/opa/Identity/enable_service_level_admins.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Identity/enable_service_level_admins.rego rename to cd3_automation_toolkit/common/opa/Identity/enable_service_level_admins.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Identity/ensure_mfa_for_users.rego b/cd3_automation_toolkit/common/opa/Identity/ensure_mfa_for_users.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Identity/ensure_mfa_for_users.rego rename to cd3_automation_toolkit/common/opa/Identity/ensure_mfa_for_users.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Identity/ensure_tenancy_admins_access.rego b/cd3_automation_toolkit/common/opa/Identity/ensure_tenancy_admins_access.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Identity/ensure_tenancy_admins_access.rego rename to cd3_automation_toolkit/common/opa/Identity/ensure_tenancy_admins_access.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Identity/iam_email_check.rego b/cd3_automation_toolkit/common/opa/Identity/iam_email_check.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Identity/iam_email_check.rego rename to cd3_automation_toolkit/common/opa/Identity/iam_email_check.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Identity/oci_iam_policy_checks.rego b/cd3_automation_toolkit/common/opa/Identity/oci_iam_policy_checks.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Identity/oci_iam_policy_checks.rego rename to cd3_automation_toolkit/common/opa/Identity/oci_iam_policy_checks.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Logging_Monitoring/default_tags_check.rego b/cd3_automation_toolkit/common/opa/Logging_Monitoring/default_tags_check.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Logging_Monitoring/default_tags_check.rego rename to cd3_automation_toolkit/common/opa/Logging_Monitoring/default_tags_check.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Logging_Monitoring/oci_bucket_write_logging_check.rego b/cd3_automation_toolkit/common/opa/Logging_Monitoring/oci_bucket_write_logging_check.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Logging_Monitoring/oci_bucket_write_logging_check.rego rename to cd3_automation_toolkit/common/opa/Logging_Monitoring/oci_bucket_write_logging_check.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Logging_Monitoring/oci_log_groups_checks.rego b/cd3_automation_toolkit/common/opa/Logging_Monitoring/oci_log_groups_checks.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Logging_Monitoring/oci_log_groups_checks.rego rename to cd3_automation_toolkit/common/opa/Logging_Monitoring/oci_log_groups_checks.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Logging_Monitoring/oci_resource_tags.rego b/cd3_automation_toolkit/common/opa/Logging_Monitoring/oci_resource_tags.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Logging_Monitoring/oci_resource_tags.rego rename to cd3_automation_toolkit/common/opa/Logging_Monitoring/oci_resource_tags.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Logging_Monitoring/oci_vcn_flow_logs_check.rego b/cd3_automation_toolkit/common/opa/Logging_Monitoring/oci_vcn_flow_logs_check.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Logging_Monitoring/oci_vcn_flow_logs_check.rego rename to cd3_automation_toolkit/common/opa/Logging_Monitoring/oci_vcn_flow_logs_check.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Network/oci_adb_access_restrictions.rego b/cd3_automation_toolkit/common/opa/Network/oci_adb_access_restrictions.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Network/oci_adb_access_restrictions.rego rename to cd3_automation_toolkit/common/opa/Network/oci_adb_access_restrictions.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Network/oci_default_sl_checks.rego b/cd3_automation_toolkit/common/opa/Network/oci_default_sl_checks.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Network/oci_default_sl_checks.rego rename to cd3_automation_toolkit/common/opa/Network/oci_default_sl_checks.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Network/oci_deny_ingress_for_nsg.rego b/cd3_automation_toolkit/common/opa/Network/oci_deny_ingress_for_nsg.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Network/oci_deny_ingress_for_nsg.rego rename to cd3_automation_toolkit/common/opa/Network/oci_deny_ingress_for_nsg.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Network/oci_deny_ingress_for_sl.rego b/cd3_automation_toolkit/common/opa/Network/oci_deny_ingress_for_sl.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Network/oci_deny_ingress_for_sl.rego rename to cd3_automation_toolkit/common/opa/Network/oci_deny_ingress_for_sl.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Network/oci_enforce_container_config.rego b/cd3_automation_toolkit/common/opa/Network/oci_enforce_container_config.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Network/oci_enforce_container_config.rego rename to cd3_automation_toolkit/common/opa/Network/oci_enforce_container_config.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Network/oci_enforce_dns_zones.rego b/cd3_automation_toolkit/common/opa/Network/oci_enforce_dns_zones.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Network/oci_enforce_dns_zones.rego rename to cd3_automation_toolkit/common/opa/Network/oci_enforce_dns_zones.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Network/oci_enforce_lb_config.rego b/cd3_automation_toolkit/common/opa/Network/oci_enforce_lb_config.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Network/oci_enforce_lb_config.rego rename to cd3_automation_toolkit/common/opa/Network/oci_enforce_lb_config.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Network/oci_enforce_network_security.rego b/cd3_automation_toolkit/common/opa/Network/oci_enforce_network_security.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Network/oci_enforce_network_security.rego rename to cd3_automation_toolkit/common/opa/Network/oci_enforce_network_security.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Network/oci_oic_access_restrictions.rego b/cd3_automation_toolkit/common/opa/Network/oci_oic_access_restrictions.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Network/oci_oic_access_restrictions.rego rename to cd3_automation_toolkit/common/opa/Network/oci_oic_access_restrictions.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Risk_score/oci_check_score_constriant.rego b/cd3_automation_toolkit/common/opa/Risk_score/oci_check_score_constriant.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Risk_score/oci_check_score_constriant.rego rename to cd3_automation_toolkit/common/opa/Risk_score/oci_check_score_constriant.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Storage/oci_block_volumes.rego b/cd3_automation_toolkit/common/opa/Storage/oci_block_volumes.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Storage/oci_block_volumes.rego rename to cd3_automation_toolkit/common/opa/Storage/oci_block_volumes.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Storage/oci_deny_public_bucket.rego b/cd3_automation_toolkit/common/opa/Storage/oci_deny_public_bucket.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Storage/oci_deny_public_bucket.rego rename to cd3_automation_toolkit/common/opa/Storage/oci_deny_public_bucket.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Storage/oci_fss.rego b/cd3_automation_toolkit/common/opa/Storage/oci_fss.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Storage/oci_fss.rego rename to cd3_automation_toolkit/common/opa/Storage/oci_fss.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Storage/oci_secure_database.rego b/cd3_automation_toolkit/common/opa/Storage/oci_secure_database.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Storage/oci_secure_database.rego rename to cd3_automation_toolkit/common/opa/Storage/oci_secure_database.rego diff --git a/cd3_automation_toolkit/user-scripts/OPA/Storage/oci_secure_storage.rego b/cd3_automation_toolkit/common/opa/Storage/oci_secure_storage.rego similarity index 100% rename from cd3_automation_toolkit/user-scripts/OPA/Storage/oci_secure_storage.rego rename to cd3_automation_toolkit/common/opa/Storage/oci_secure_storage.rego diff --git a/cd3_automation_toolkit/common/python/__init__.py b/cd3_automation_toolkit/common/python/__init__.py new file mode 100644 index 000000000..ce4f0b656 --- /dev/null +++ b/cd3_automation_toolkit/common/python/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + + +from .commonTools import * + + + diff --git a/cd3_automation_toolkit/common/python/commonTools.py b/cd3_automation_toolkit/common/python/commonTools.py new file mode 100644 index 000000000..47d49c467 --- /dev/null +++ b/cd3_automation_toolkit/common/python/commonTools.py @@ -0,0 +1,322 @@ + +import re +import os +import shutil +import datetime +import pandas as pd +from openpyxl import load_workbook +from openpyxl.styles import PatternFill +from openpyxl.styles import Alignment +from openpyxl.styles import Border +from openpyxl.styles import Side +import collections +import warnings +warnings.simplefilter("ignore") +from contextlib import contextmanager + + +@contextmanager +def section(title='', header=False, padding=117): + separator = '-' if not header else '=' + # Not sure why 117 but thats how it was before. + print(f'{title:{separator}^{padding}}') + yield + if header: + print(separator * padding) + +def exit_menu(msg, exit_code=1): + print(msg) + exit(exit_code) + +class commonTools(): + endNames = {'', '', ''} + + def __init__(self): + pass + + # Read rows from CD3 + def data_frame(filename, sheetname): + + # Read the tab from excel, Drop null values, Reset index + df, col_headers = commonTools.read_cd3(filename, sheetname) + df = df.dropna(how='all') + df = df.reset_index(drop=True) + + return df + def read_cd3(cd3file, sheet_name): + df = {} + try: + df = pd.read_excel(cd3file, sheet_name=sheet_name, skiprows=1, dtype=object) + + except Exception as e: + if("Events" in str(e) or "Notifications" in str(e)): + print("\nTabs - \"Events\" or \"Notifications\" is missing in the CD3. Please make sure to use the correct input file for Events and Notifications in properties file...Exiting!!") + exit(1) + else: + print("Error occurred while reading the CD3 excel sheet: "+ str(e)) + exit(1) + + yield df + try: + book = load_workbook(cd3file) + sheet = book[sheet_name] + except KeyError as e: + if 'does not exist' in str(e): + print("\nTab - \""+sheet_name+"\" seems to be missing in the CD3. Please make sure to use the right CD3 in properties file.....Exiting!!") + exit(1) + except Exception as e: + print(str(e)) + print("Exiting!!") + exit(1) + + values_for_column = collections.OrderedDict() + # values_for_column={} + for j in range(0, sheet.max_column): + col_name = sheet.cell(row=2, column=j + 1).value + if (type(col_name) == str): + values_for_column[col_name] = [] + yield values_for_column + + #Write exported rows to cd3 + def write_to_cd3(values_for_column, cd3file, sheet_name,append=False): + try: + book = load_workbook(cd3file) + sheet = book[sheet_name] + + + except Exception as e: + print(str(e)) + print("Exiting!!") + exit(1) + if (sheet_name == "VCN Info"): + onprem_destinations = "" + ngw_destinations = "" + igw_destinations = "" + for destination in values_for_column["onprem_destinations"]: + onprem_destinations=destination+","+onprem_destinations + for destination in values_for_column["ngw_destinations"]: + ngw_destinations = destination + "," + ngw_destinations + for destination in values_for_column["igw_destinations"]: + igw_destinations = destination + "," + igw_destinations + + if (onprem_destinations != "" and onprem_destinations[-1] == ','): + onprem_destinations = onprem_destinations[:-1] + if (ngw_destinations != "" and ngw_destinations[-1] == ','): + ngw_destinations = ngw_destinations[:-1] + if (igw_destinations != "" and igw_destinations[-1] == ','): + igw_destinations = igw_destinations[:-1] + + sheet.cell(3,2).value = onprem_destinations + sheet.cell(4,2).value = ngw_destinations + sheet.cell(5,2).value = igw_destinations + try: + book.save(cd3file) + book.close() + except Exception as e: + print(str(e)) + print("Exiting!!") + exit(1) + return + + + #rows_len=len(rows) + rows_len = len(values_for_column["Region"]) + sheet_max_rows = sheet.max_row + #If no rows exported from OCI, remove the sample data as well + if(rows_len == 0) : + if not append: + print("0 rows exported; Nothing to write to CD3 excel; Tab "+sheet_name +" will be empty in CD3 excel!!") + for i in range(0, sheet.max_row): + for j in range(0, sheet.max_column): + sheet.cell(row=i + 3, column=j + 1).value = "" + try: + book.save(cd3file) + book.close() + except Exception as e: + print(str(e)) + print("Exiting!!") + exit(1) + return + + if append: + for x in range(1, sheet_max_rows): + if sheet['A'][x].value == None: + last_line = x + break + #rows_len +=last_line + large = rows_len + start = last_line+1 + + + else: + start = 3 + if (rows_len > sheet_max_rows): + large = rows_len + else: + large = sheet_max_rows + + df, values_for_column_sheet = commonTools.read_cd3(cd3file, sheet_name) + + #Put Data + j=0 + for i in range(0,large): + + for col_name in values_for_column.keys(): + + #Check if column name to be populated in present in the sheet. + if col_name not in values_for_column_sheet: + continue + # Data + if(i>=rows_len): + sheet.cell(row=i+start, column=j+1).value = "" + else: + sheet.cell(row=i+start, column=j+1).value = values_for_column[col_name][i] + sheet.cell(row=i+start, column=j+1).alignment = Alignment(wrap_text=True) + j=j+1 + j=0 + + + brdr = Border(left=Side(style='thin'), + right=Side(style='thin'), + top=Side(style='thin'), + bottom=Side(style='thin'), + ) + + for row in sheet.iter_rows(min_row=3): + for cell in row: + cell.border = brdr + + # Add color for exported sec rules and route rules + if (sheet_name == "RouteRulesinOCI" or sheet_name == "SecRulesinOCI" or sheet_name == "DRGRouteRulesinOCI"): + names = [] + # Add color coding to exported rules + for row in sheet.iter_rows(min_row=3): + c = 0 + region = "" + name = "" + for cell in row: + c = c + 1 + if (c == 1): + region = cell.value + continue + elif (c == 4): + name = cell.value + break + + vcn_name = region + "_" + name + if (vcn_name not in names): + names.append(vcn_name) + for cellnew in row: + if (len(names) % 2 == 0): + cellnew.fill = PatternFill(start_color="94AFAF", end_color="94AFAF", fill_type="solid") + cellnew.border = brdr + else: + cellnew.fill = PatternFill(start_color="E5DBBE", end_color="E5DBBE", fill_type="solid") + cellnew.border = brdr + else: + for cellnew in row: + if (len(names) % 2 == 0): + cellnew.fill = PatternFill(start_color="94AFAF", end_color="94AFAF", fill_type="solid") + cellnew.border = brdr + else: + cellnew.fill = PatternFill(start_color="E5DBBE", end_color="E5DBBE", fill_type="solid") + cellnew.border = brdr + try: + book.save(cd3file) + book.close() + except Exception as e: + print(str(e)) + print("Exiting!!") + exit(1) + + # Check value exported + # If None - replace with "" + # If list, convert to comma sepearted string + def check_exported_value(value): + if value == None: + value = "" + if ("list" in str(type(value))): + str1 = "" + if (value.__len__() == 0): + value = "" + for v in value: + str1 = v + "," + str1 + if (str1 != "" and str1[-1] == ','): + value = str1[:-1] + + return value + # Check TF variable Name + def check_tf_variable(var_name): + tfname = re.compile('[^a-zA-Z0-9_-]') + tfnamestart = re.compile('[A-Za-z]') + + var_name = tfname.sub("-", var_name) + x = tfnamestart.match(var_name) + # variable name doesnot start with letter; append with c + if (x == None): + var_name = "c" + var_name + return var_name + + # Process ColumnValues + def check_columnvalue(columnvalue): + + if str(columnvalue).lower() == 'true' or str(columnvalue).lower() == 'false': + columnvalue = str(columnvalue).lower() + + if (columnvalue.lower() == 'nan'): + columnvalue = "" + + # replace \ with \\ + if("\\" in columnvalue): + columnvalue = columnvalue.replace("\\", "\\\\") + + # replace " with \" + if("\"" in columnvalue): + columnvalue=columnvalue.replace("\"","\\\"") + + return columnvalue + + # Process column values with :: + def check_multivalues_columnvalue(columnvalue, columnname, tempdict): + columnvalue = str(columnvalue).strip() + columnname = commonTools.check_column_headers(columnname) + if "::" in columnvalue: + if ".Flex" in columnvalue or ".Micro" in columnvalue: + columnname = commonTools.check_column_headers(columnname) + multivalues = columnvalue.split("::") + multivalues = [str(part).strip() for part in multivalues if part] + tempdict = {columnname: multivalues} + elif columnname != 'Compartment Name' and "ipv6" not in columnname.lower(): + columnname = commonTools.check_column_headers(columnname) + multivalues = columnvalue.split("::") + multivalues = [str(part).strip() for part in multivalues ]#if part] + tempdict = {columnname: multivalues} + return tempdict + + # Check CD3 Column headers + def check_column_headers(var_name): + # replace special characters and spaces with '_' and convert to lowercase + # replaces multiple occurrence of '_' to just 1 + var_name = var_name.strip() + var_name = re.sub('[@!#$%^&*<>?/}{~: \n()|-]', '_', var_name).lower() + var_name = re.sub('_+', '_', var_name).lower() + return var_name + + def backup_file(src_dir, resource, pattern): + dest_dir = str(src_dir) + "/backup_" + resource + "/" + datetime.datetime.now().strftime("%d-%m-%H%M%S").replace('/', '-') + for f in os.listdir(str(src_dir)): + if f.endswith(pattern): + print("Backing up existing " + f + " to " + dest_dir) + if not os.path.exists(dest_dir): + # print("\nCreating backup dir " + dest_dir + "\n") + os.makedirs(dest_dir) + + src = os.path.join(str(src_dir), f) + #dest = os.path.join(dest_dir, f) + # print("backing up ....." + src +" to "+dest) + shutil.move(src, dest_dir) + """if (overwrite == 'yes'): + shutil.move(src, dest_dir) + elif (overwrite == 'no'): + shutil.copyfile(src, dest) + """ diff --git a/cd3_automation_toolkit/connectAzure.properties b/cd3_automation_toolkit/connectAzure.properties new file mode 100644 index 000000000..191f96d90 --- /dev/null +++ b/cd3_automation_toolkit/connectAzure.properties @@ -0,0 +1,23 @@ +[Default] + +################################################################################################################## + ## Required Parameters ## +################################################################################################################## + +# Friendly name for the Azure Subscription; The generated .auto.tfvars files will be prefixed with this +prefix= + +################################################################################################################## + ## Auth Details Parameters ## +# Leave below details empty if there is no need to run APIs against Azure portal +# This can be used to generate only terraform without applying it +################################################################################################################## + +subscription_id= + +tenant_id= + +client_id= + +client_secret= + diff --git a/cd3_automation_toolkit/connectCloud.py b/cd3_automation_toolkit/connectCloud.py new file mode 100644 index 000000000..e58f97c69 --- /dev/null +++ b/cd3_automation_toolkit/connectCloud.py @@ -0,0 +1,30 @@ +import sys +import subprocess + + +def main(): + if len(sys.argv) != 3: + print("Usage: python connectCloud.py ") + print("Example: python connectCloud.py oci tenancyconfig.properties") + print("Example: python connectCloud.py azure connectAzure.properties") + return + + cloud_provider = sys.argv[1].lower() + argument = sys.argv[2] + + if cloud_provider == 'oci': + script_name = 'user-scripts/createTenancyConfig.py' + elif cloud_provider == 'azure': + script_name = 'user-scripts/connectAzure.py' + else: + print("Invalid cloud provider. Use 'azure' or 'oci'.") + return + + try: + subprocess.run([sys.executable, script_name, argument], check=True) + except subprocess.CalledProcessError as e: + pass + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/cd3_automation_toolkit/connectGCP.properties b/cd3_automation_toolkit/connectGCP.properties new file mode 100644 index 000000000..ec604a3fc --- /dev/null +++ b/cd3_automation_toolkit/connectGCP.properties @@ -0,0 +1,31 @@ +[Default] + +################################################################################################################## + ## Required Parameters ## +################################################################################################################## + +# Friendly name for the Customer; The generated .auto.tfvars files will be prefixed with this +prefix=new + +organization_id=378499894770 + +# Auth Mechanism for GCP APIs - api_key +# Please make sure to add IAM policies for service account before executing connectContainerToGCP.py +auth_mechanism=api_key + +# Path to JSON file having private key information for the service account; +# Defaults to /cd3user/cloud_infra/gcp/keys/gcp_api_private.json when left empty +config_file=/Users/susingla/PyCharmProjects/orahub-develop/cd3_automation_toolkit/gcp/test-api-private.json + +# The outdir_structure_file defines the grouping of the terraform auto.tf.vars for the various generated resources. +# To have all the files generated in a single directory in the corresponding region, leave this variable blank. +# To group resources into different directories within each region - specify the absolute path to the file. +# The default file is specified below. You can make changes to the grouping in the below file to suit your deployment +#outdir_structure_file= +#or +outdir_structure_file=/Users/susingla/PyCharmProjects/orahub-develop/cd3_automation_toolkit/user-scripts/outdir_structure_file.properties + +# SSH Key for launched instances; Use '\n' as the delimiter to add multiple ssh keys. +# Example: "ssh-rsa AAXXX......yhdlo\nssh-rsa AAxxskj...edfwf" +# Optional +ssh_public_key= diff --git a/cd3_automation_toolkit/connectOCI.properties b/cd3_automation_toolkit/connectOCI.properties new file mode 100644 index 000000000..2ae1c7df9 --- /dev/null +++ b/cd3_automation_toolkit/connectOCI.properties @@ -0,0 +1,94 @@ +[Default] + +################################################################################################################## + ## Required Parameters ## +################################################################################################################## + +# Friendly name for the Customer Tenancy eg: demotenancy; The generated .auto.tfvars files will be prefixed with this. +prefix= + +tenancy_ocid= + +# Example: us-phoenix-1 +region= + +# Auth Mechanism for OCI APIs - api_key,instance_principal,session_token +# Please make sure to add IAM policies for user/instance_principal before executing createTenancyConfig.py +auth_mechanism=api_key + +################################################################################################################## + ## Auth Details Parameters ## +# Required only for ${auth_mechanism} as api_key; Leave below params empty if 'instance_principal' or 'session_token' +# is used +################################################################################################################## + +user_ocid= +#Path of API Private Key (PEM Key) File; Defaults to /cd3user/tenancies/keys/oci_api_private.pem when left empty +key_path= +fingerprint= + +################################################################################################################## + ## Deployment Parameters ## +################################################################################################################## + +# The outdir_structure_file defines the grouping of the terraform auto.tf.vars for the various generated resources. +# To have all the files generated in a single directory in the corresponding region, leave this variable blank. +# To group resources into different directories within each region - specify the absolute path to the file. +# The default file is specified below. You can make changes to the grouping in the below file to suit your deployment + +#outdir_structure_file= +#or +outdir_structure_file=/cd3user/oci_tools/cd3_automation_toolkit/user-scripts/outdir_structure_file.properties + + +# IaC Tool to be configured - Terraform or OpenTofu +# Default is terraform +# To use OpenTofu, specify tofu +tf_or_tofu=terraform + +# SSH Key for launched instances; Use '\n' as the delimiter to add multiple ssh keys. +# Example: ssh-rsa AAXXX......yhdlo\nssh-rsa AAxxskj...edfwf +# Optional +ssh_public_key= + +################################################################################################################## + ## Advanced Parameters for DevOps ## +# Below OCI Objects - Remote State Bucket Name and DevOps Project/Repo and a Notification Topic will be created/fetched +# from region specified in ${region} above. +# These values are required to be set as "yes" for Jenkins Configuration. +################################################################################################################## + + +# Compartment OCID where Bucket and DevOps Project/repo will be created; defaults to root if left empty. +compartment_ocid= + +# Remote state configuration +# Enter yes if remote state needs to be configured, else tfstate will be stored on local filesystem. +use_remote_state=no + +# Specify bucket name if you want to use existing bucket else leave empty. +# If left empty, Bucket with name ${customer_name}-automation-toolkit-bucket will be created/reused in ${region}. +remote_state_bucket_name= + +# OCI DevOps GIT configuration +# Enter yes if generated terraform_files need to be stored in OCI DevOps GIT Repo else they will be stored on local +# filesystem. Will enforce 'yes' for use_remote_state in case below is set to 'yes' +use_oci_devops_git=no + +# Specify Repo name if you want to use existing OCI Devops GIT Repository else leave empty Format: +# If left empty, DevOps items with names ${customer_name}-automation-toolkit-project/repo/topic will be created/reused +# in ${region}. +oci_devops_git_repo_name= + +# User Details to perform GIT operations in OCI Devops GIT Repo and Remote Terraform State Management; Mandatory when using $(auth_mechanism) as instance_principal +# or session_token +# Customer Secret Key will be created for this user for S3 credentials of the bucket. +# When left empty, it will be fetched from $(user_ocid) for $(auth_mechanism) as api_key. +# Format: /@ eg oracleidentitycloudservice/devopsuser@oracle.com@ocitenant +# Users in Custom Domain are not supported as of now. +oci_devops_git_user= + +# When left empty, same key file from $(key_path) used for $(auth_mechanism) as api_key will be copied to +# /cd3user/tenancies// and used for GIT Operations. +oci_devops_git_key= + diff --git a/cd3_automation_toolkit/example/CD3-Azure-template.xlsx b/cd3_automation_toolkit/example/CD3-Azure-template.xlsx new file mode 100644 index 000000000..5a1a761a5 Binary files /dev/null and b/cd3_automation_toolkit/example/CD3-Azure-template.xlsx differ diff --git a/cd3_automation_toolkit/example/CD3-Blank-template.xlsx b/cd3_automation_toolkit/example/CD3-Blank-template.xlsx index bd05b65fe..31a1405fc 100644 Binary files a/cd3_automation_toolkit/example/CD3-Blank-template.xlsx and b/cd3_automation_toolkit/example/CD3-Blank-template.xlsx differ diff --git a/cd3_automation_toolkit/example/CD3-CIS-ManagementServices-template.xlsx b/cd3_automation_toolkit/example/CD3-CIS-ManagementServices-template.xlsx index 9b19a4d69..f526717d6 100644 Binary files a/cd3_automation_toolkit/example/CD3-CIS-ManagementServices-template.xlsx and b/cd3_automation_toolkit/example/CD3-CIS-ManagementServices-template.xlsx differ diff --git a/cd3_automation_toolkit/example/CD3-CIS-template.xlsx b/cd3_automation_toolkit/example/CD3-CIS-template.xlsx index 478d13cc1..d211ab97e 100644 Binary files a/cd3_automation_toolkit/example/CD3-CIS-template.xlsx and b/cd3_automation_toolkit/example/CD3-CIS-template.xlsx differ diff --git a/cd3_automation_toolkit/example/CD3-Firewall-template.xlsx b/cd3_automation_toolkit/example/CD3-Firewall-template.xlsx index af7a8c15d..1f6c68a59 100644 Binary files a/cd3_automation_toolkit/example/CD3-Firewall-template.xlsx and b/cd3_automation_toolkit/example/CD3-Firewall-template.xlsx differ diff --git a/cd3_automation_toolkit/example/CD3-HubSpoke-template.xlsx b/cd3_automation_toolkit/example/CD3-HubSpoke-template.xlsx index 7a1471de2..92723d032 100644 Binary files a/cd3_automation_toolkit/example/CD3-HubSpoke-template.xlsx and b/cd3_automation_toolkit/example/CD3-HubSpoke-template.xlsx differ diff --git a/cd3_automation_toolkit/example/CD3-SingleVCN-template.xlsx b/cd3_automation_toolkit/example/CD3-SingleVCN-template.xlsx index bf9dcdbf2..64946d17a 100644 Binary files a/cd3_automation_toolkit/example/CD3-SingleVCN-template.xlsx and b/cd3_automation_toolkit/example/CD3-SingleVCN-template.xlsx differ diff --git a/cd3_automation_toolkit/example/OCI_Regions b/cd3_automation_toolkit/example/OCI_Regions deleted file mode 100644 index 69c334048..000000000 --- a/cd3_automation_toolkit/example/OCI_Regions +++ /dev/null @@ -1,42 +0,0 @@ -#Region:Region_Key -zurich:eu-zurich-1 -toronto:ca-toronto-1 -montreal:ca-montreal-1 -chuncheon:ap-chuncheon-1 -vinhedo:sa-vinhedo-1 -sydney:ap-sydney-1 -sanjose:us-sanjose-1 -singapore:ap-singapore-1 -santiago:sa-santiago-1 -queretaro:mx-queretaro-1 -phoenix:us-phoenix-1 -chicago:us-chicago-1 -tokyo:ap-tokyo-1 -jerusalem:il-jerusalem-1 -marseille:eu-marseille-1 -melbourne:ap-melbourne-1 -madrid:eu-madrid-1 -milan:eu-milan-1 -london:uk-london-1 -osaka:ap-osaka-1 -johannesburg:af-johannesburg-1 -jeddah:me-jeddah-1 -seoul:ap-seoul-1 -ashburn:us-ashburn-1 -hyderabad:ap-hyderabad-1 -saopaulo:sa-saopaulo-1 -frankfurt:eu-frankfurt-1 -dubai:me-dubai-1 -cardiff:uk-cardiff-1 -paris:eu-paris-1 -mumbai:ap-mumbai-1 -abudhabi:me-abudhabi-1 -stockholm:eu-stockholm-1 -amsterdam:eu-amsterdam-1 -langley:us-langley-1 -luke:us-luke-1 -gov-ashburn:us-gov-ashburn-1 -gov-chicago:us-gov-chicago-1 -gov-phoenix:us-gov-phoenix-1 -gov-london:uk-gov-london-1 -gov-cardiff:uk-gov-cardiff-1 \ No newline at end of file diff --git a/cd3_automation_toolkit/example/config b/cd3_automation_toolkit/example/config deleted file mode 100644 index 504536b7c..000000000 --- a/cd3_automation_toolkit/example/config +++ /dev/null @@ -1,7 +0,0 @@ -[DEFAULT] -tenancy = -fingerprint = -user = -key_file = -region = - diff --git a/cd3_automation_toolkit/ocicloud/__init__.py b/cd3_automation_toolkit/ocicloud/__init__.py new file mode 100644 index 000000000..066c83efc --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python3 + + diff --git a/cd3_automation_toolkit/Excel_Columns b/cd3_automation_toolkit/ocicloud/python/Excel_Columns similarity index 98% rename from cd3_automation_toolkit/Excel_Columns rename to cd3_automation_toolkit/ocicloud/python/Excel_Columns index cc0d110d8..1607075f4 100644 --- a/cd3_automation_toolkit/Excel_Columns +++ b/cd3_automation_toolkit/ocicloud/python/Excel_Columns @@ -34,7 +34,9 @@ "VCNs" : { "VCN Name" : "display_name", - "CIDR Blocks" : "cidr_blocks" + "CIDR Blocks" : "cidr_blocks", + "ULA IPv6 CIDR" : "ipv6_private_cidr_blocks" + }, "DRGs" : { @@ -53,7 +55,8 @@ { "CIDR Block" : "cidr_block", "Availability Domain(AD1|AD2|AD3|Regional)" : "availability_domain", - "Type(private|public)" : "prohibit_public_ip_on_vnic" + "Type(private|public)" : "prohibit_public_ip_on_vnic", + "IPv6CIDR Block" : "ipv6_cidr_blocks" }, "NSGs" : { diff --git a/cd3_automation_toolkit/OCI_Protocols b/cd3_automation_toolkit/ocicloud/python/OCI_Protocols similarity index 100% rename from cd3_automation_toolkit/OCI_Protocols rename to cd3_automation_toolkit/ocicloud/python/OCI_Protocols diff --git a/cd3_automation_toolkit/OCI_Regions b/cd3_automation_toolkit/ocicloud/python/OCI_Regions similarity index 80% rename from cd3_automation_toolkit/OCI_Regions rename to cd3_automation_toolkit/ocicloud/python/OCI_Regions index 824428a7a..11249805f 100644 --- a/cd3_automation_toolkit/OCI_Regions +++ b/cd3_automation_toolkit/ocicloud/python/OCI_Regions @@ -1,17 +1,12 @@ #Region:Region_Key -abilene:us-abilene-1 -saltlake:us-saltlake-2 amsterdam:eu-amsterdam-1 stockholm:eu-stockholm-1 abudhabi:me-abudhabi-1 -saltlake:us-saltlake-1 bogota:sa-bogota-1 mumbai:ap-mumbai-1 paris:eu-paris-1 cardiff:uk-cardiff-1 -dallas:us-dallas-1 dubai:me-dubai-1 -tukwila:us-tukwila-4 frankfurt:eu-frankfurt-1 saopaulo:sa-saopaulo-1 batam:ap-batam-1 @@ -21,7 +16,6 @@ seoul:ap-seoul-1 jeddah:me-jeddah-1 johannesburg:af-johannesburg-1 osaka:ap-osaka-1 -kragujevac:eu-kragujevac-1 london:uk-london-1 milan:eu-milan-1 madrid:eu-madrid-1 @@ -30,18 +24,17 @@ marseille:eu-marseille-1 monterrey:mx-monterrey-1 jerusalem:il-jerusalem-1 tokyo:ap-tokyo-1 -neom:me-neom-1 chicago:us-chicago-1 phoenix:us-phoenix-1 queretaro:mx-queretaro-1 riyadh:me-riyadh-1 santiago:sa-santiago-1 -singapore:ap-singapore-1 sanjose:us-sanjose-1 sydney:ap-sydney-1 valparaiso:sa-valparaiso-1 vinhedo:sa-vinhedo-1 -singapore:ap-singapore-2 +singapore-2:ap-singapore-2 +singapore-1:ap-singapore-1 chuncheon:ap-chuncheon-1 montreal:ca-montreal-1 toronto:ca-toronto-1 diff --git a/cd3_automation_toolkit/ocicloud/python/__init__.py b/cd3_automation_toolkit/ocicloud/python/__init__.py new file mode 100644 index 000000000..c7e18e857 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/python/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +from .cd3Validator import * +from .cd3FirewallValidator import * +from .cd3Services import * +from .ociCommonTools import * +from .compute import * +from .costmanagement import * +from .database import * +from .developerservices import * +from .governance import * +from .identity import * +from .managementservices import * +from .network import * +from .sddc import * +from .security import * +from .storage import * + + diff --git a/cd3_automation_toolkit/cd3FirewallValidator.py b/cd3_automation_toolkit/ocicloud/python/cd3FirewallValidator.py similarity index 87% rename from cd3_automation_toolkit/cd3FirewallValidator.py rename to cd3_automation_toolkit/ocicloud/python/cd3FirewallValidator.py index c98100a6c..0f6177441 100644 --- a/cd3_automation_toolkit/cd3FirewallValidator.py +++ b/cd3_automation_toolkit/ocicloud/python/cd3FirewallValidator.py @@ -15,7 +15,11 @@ from oci.core.virtual_network_client import VirtualNetworkClient from oci.vault import VaultsClient from oci.key_management import KmsVaultClient -from commonTools import * +import os +import sys +sys.path.append(os.getcwd()+"/..") +from ocicloud.python.ociCommonTools import * +#from ocicloud.python.commonTools import * import re """def get_vcn_ids(compartment_ids, config): @@ -148,6 +152,9 @@ def compare_values(list_to_check, value_to_check, index): elif index[1].lower() == 'nsg': value1 = index[3].split("_") log(f'ROW {index[0] + 3} : Invalid value for column "NSGs". NSG "{value1[1]}" for "{value1[0]}" does not exist in "{index[2]}".') + elif index[1].lower() == 'service': + value_to_check = value_to_check.split("&") + log(f'ROW {index[0] + 3} : Invalid value for column "{index[1]}". {index[3]} "{value_to_check[1]}" for policy "{value_to_check[0]}" does not exist in "{index[2]}" tab.') else: value_to_check = value_to_check.split("::") log(f'ROW {index[0] + 3} : Invalid value for column "{index[1]}". {index[3]} "{value_to_check[1]}" for policy "{value_to_check[0]}" does not exist in "{index[2]}" tab.') @@ -174,7 +181,7 @@ def validate_Firewall(filename,comp_ids,fwpolicy,config,signer, ct): fw_nsg_check = [] fw_policycheck = False - dffirewall = data_frame(filename, 'Firewall') + dffirewall = commonTools.data_frame(filename, 'Firewall') dfcolumns = dffirewall.columns.values.tolist() for i in dffirewall.index: @@ -264,7 +271,7 @@ def validate_FirewallPolicy(filename, ct): fwpolicy_comp_check = False fwpolicy_invalid_check = False - dffwpolicy = data_frame(filename, 'Firewall-Policy') + dffwpolicy = commonTools.data_frame(filename, 'Firewall-Policy') dfcolumns = dffwpolicy.columns.values.tolist() for i in dffwpolicy.index: @@ -323,7 +330,7 @@ def validate_FirewallPolicyApplist(filename, fwpolicy_list,ct): fwpolicyapp_check = [] fwpolicyapp_appg_length = False fwpolicyapp_appg_mistake = False - dffwpolicyapplist = data_frame(filename, 'Firewall-Policy-ApplicationList') + dffwpolicyapplist = commonTools.data_frame(filename, 'Firewall-Policy-ApplicationList') dfcolumns = dffwpolicyapplist.columns.values.tolist() for i in dffwpolicyapplist.index: @@ -403,7 +410,7 @@ def validate_FirewallPolicyServicelist(filename, fwpolicy_list,ct): fwpolicyservice_check = [] fwpolicyservice_serviceg_length = False fwpolicyservice_serviceg_mistake = False - dffwpolicyservicelist = data_frame(filename, 'Firewall-Policy-ServiceList') + dffwpolicyservicelist = commonTools.data_frame(filename, 'Firewall-Policy-ServiceList') dfcolumns = dffwpolicyservicelist.columns.values.tolist() for i in dffwpolicyservicelist.index: @@ -481,7 +488,7 @@ def validate_FirewallPolicyUrllist(filename, fwpolicy_list,ct): fwpolicyurl_check = [] fwpolicyurl_urlg_length = False fwpolicyurl_urlg_mistake = False - dffwpolicyUrllist = data_frame(filename, 'Firewall-Policy-UrlList') + dffwpolicyUrllist = commonTools.data_frame(filename, 'Firewall-Policy-UrlList') dfcolumns = dffwpolicyUrllist.columns.values.tolist() for i in dffwpolicyUrllist.index: @@ -540,7 +547,7 @@ def validate_FirewallPolicyAddress(filename, fwpolicy_list,ct): fwpolicyaddress_check = [] fwpolicyaddress_addressg_length = False fwpolicyaddress_addressg_mistake = False - dffwpolicyAddress = data_frame(filename, 'Firewall-Policy-AddressList') + dffwpolicyAddress = commonTools.data_frame(filename, 'Firewall-Policy-AddressList') dfcolumns = dffwpolicyAddress.columns.values.tolist() for i in dffwpolicyAddress.index: @@ -611,7 +618,7 @@ def validate_FirewallPolicySecrets(filename, fwpolicy_list, config,signer, ct): fwpolicysecret_secret_length = False fwpolicysecret_comp_check = False fwpolicysecret_vault_check = [] - dffwpolicysecret = data_frame(filename, 'Firewall-Policy-Secret') + dffwpolicysecret = commonTools.data_frame(filename, 'Firewall-Policy-Secret') dfcolumns = dffwpolicysecret.columns.values.tolist() for i in dffwpolicysecret.index: @@ -708,7 +715,7 @@ def validate_FirewallPolicyDecryption(filename, fwpolicy_list,ct): fwpolicydecrypt_invalid_check = False fwpolicydecrypt_check = [] fwpolicydecrypt_nameg_length = False - dffwpolicydecrypt = data_frame(filename, 'Firewall-Policy-DecryptProfile') + dffwpolicydecrypt = commonTools.data_frame(filename, 'Firewall-Policy-DecryptProfile') dfcolumns = dffwpolicydecrypt.columns.values.tolist() for i in dffwpolicydecrypt.index: @@ -779,10 +786,10 @@ def validate_FirewallPolicyDecryptionRule(filename, fwpolicy_list, fulladdreslis fwpolicydecryptruledp_check = [] fwpolicydecryptrulepost_check = [] - dffwpolicydecryptrule = data_frame(filename, 'Firewall-Policy-DecryptRule') + dffwpolicydecryptrule = commonTools.data_frame(filename, 'Firewall-Policy-DecryptRule') dfcolumns = dffwpolicydecryptrule.columns.values.tolist() - dffwdecryptrule = data_frame(filename, 'Firewall-Policy-DecryptRule') + dffwdecryptrule = commonTools.data_frame(filename, 'Firewall-Policy-DecryptRule') dffwdecryptrule_list = dffwdecryptrule['Rule Name'].astype(str) dffwdecryptrulepolicy_list = dffwdecryptrule['Firewall Policy'].astype(str) fulldecryptrulelist = dffwdecryptrulepolicy_list + '::' + dffwdecryptrule_list @@ -878,10 +885,10 @@ def validate_FirewallPolicyTunnelInspectRule(filename, fwpolicy_list, fulladdres fwpolicytunnelinspectruleda_check = [] fwpolicytunnelinspectrulepost_check = [] - dffwpolicytunnelinspectrule = data_frame(filename, 'Firewall-Policy-TunnelInspect') + dffwpolicytunnelinspectrule = commonTools.data_frame(filename, 'Firewall-Policy-TunnelInspect') dfcolumns = dffwpolicytunnelinspectrule.columns.values.tolist() - dffwtunnleinspectrule = data_frame(filename, 'Firewall-Policy-TunnelInspect') + dffwtunnleinspectrule = commonTools.data_frame(filename, 'Firewall-Policy-TunnelInspect') dffwtunnleinspectrule_list = dffwtunnleinspectrule['Rule Name'].astype(str) dffwtunnleinspectrulepolicy_list = dffwtunnleinspectrule['Firewall Policy'].astype(str) fulltunnelinspectrulelist = dffwtunnleinspectrulepolicy_list + '::' + dffwtunnleinspectrule_list @@ -961,6 +968,119 @@ def validate_FirewallPolicyTunnelInspectRule(filename, fwpolicy_list, fulladdres return True else: return False +def validate_FirewallPolicyNatRule(filename, fwpolicy_list, fulladdreslist, fullnatservicelist,ct): + fwpolicynatrule_empty_check = False + fwpolicynatrule_invalid_check = False + fwpolicynatrule_check = [] + fwpolicynatrule_nameg_length = False + fwpolicynatrulesa_check = [] + fwpolicynatruleda_check = [] + fwpolicynatrulepost_check = [] + + dffwpolicynatrule = commonTools.data_frame(filename, 'Firewall-Policy-NatRules') + dfcolumns = dffwpolicynatrule.columns.values.tolist() + + dffwnatrule = commonTools.data_frame(filename, 'Firewall-Policy-NatRules') + dffwnatrule_list = dffwnatrule['Rule Name'].astype(str) + dffwnatrulepolicy_list = dffwnatrule['Firewall Policy'].astype(str) + fullnatrulelist = dffwnatrulepolicy_list + '::' + dffwnatrule_list + + for i in dffwpolicynatrule.index: + region = str(dffwpolicynatrule.loc[i, 'Region']).strip().lower() + # Encountered + if (region in commonTools.endNames): + break + if region == 'nan': + log(f'ROW {i + 3} : Empty value at column "Region".') + fwpolicynatrule_empty_check = True + elif region not in ct.all_regions: + log(f'ROW {i + 3} : "Region" {region} is not subscribed for tenancy.') + fwpolicynatrule_invalid_check = True + for columnname in dfcolumns: + # Column value + columnvalue = str(dffwpolicynatrule.loc[i, columnname]).strip() + if (columnname == 'Firewall Policy'): + if columnvalue.lower() == 'nan': + log(f'ROW {i + 3} : Empty value at column Policy Name.') + fwpolicynatrule_empty_check = True + else: + # Cross check the Policy names in Firewall Policy sheet with OCI. + fwpolicynatrule_check.append( + compare_values(fwpolicy_list.tolist(), columnvalue, [i, 'Policy Name', 'Firewall-Policy'])) + if (columnname == 'Rule Name'): + if columnvalue.lower() == 'nan': + log(f'ROW {i + 3} : Empty value at column Rule Name.') + fwpolicynatrule_empty_check = True + if columnvalue.lower() != 'nan': + if (len(columnvalue) > 63) or (len(columnvalue) < 2): + log(f'ROW {i + 3} : Nat rule Name "{columnvalue}" has more alphanumeric characters than the allowed maximum limit of 63.') + fwpolicynatrule_nameg_length = True + if (validate_names(columnvalue) == True): + log(f'ROW {i + 3} : Only alphabets, digits, - and _ are allowed in the Tunnel inspection Rule Name') + fwpolicynatrule_invalid_check == True + if (columnname == 'Source Address'): + if columnvalue.lower() != 'nan': + sa_list = columnvalue.split(",") + for eachsa in sa_list: + fwpolicyname = str(dffwpolicynatrule.loc[i, 'Firewall Policy']).strip() + finalsalist = fwpolicyname + '::' + eachsa + fwpolicynatrulesa_check.append(compare_values(fulladdreslist.tolist(), finalsalist,[i, 'Source Address','Firewall-Policy-Address','Address list'])) + if (columnname == 'Destination Address'): + if columnvalue.lower() != 'nan': + da_list = columnvalue.split(",") + for eachda in da_list: + fwpolicyname = str(dffwpolicynatrule.loc[i, 'Firewall Policy']).strip() + finaldalist = fwpolicyname + '::' + eachda + fwpolicynatruleda_check.append(compare_values(fulladdreslist.tolist(), finaldalist,[i, 'Destination Address','Firewall-Policy-Address','Address list'])) + if (columnname == 'Service'): + if columnvalue.lower() != 'nan': + finalservice = [] + fwpolicyname = str(dffwpolicynatrule.loc[i, 'Firewall Policy']).strip() + finalservicelist = fwpolicyname + '&' + columnvalue + natserv = fullnatservicelist.tolist() + for service1 in natserv: + spolicy,snst = service1.split("&") + if snst is not "nan" and "\n" in snst: + tempservices = snst.split("\n") + for intservice in tempservices: + finalservice_value = spolicy + "&" + intservice.split("::")[0] + finalservice.append(finalservice_value) + else: + finalservice_value = spolicy + "&" + snst.split("::")[0] + finalservice.append(finalservice_value) + fwpolicynatruleda_check.append(compare_values(finalservice, finalservicelist, [i, 'Service', 'Firewall-Policy-ServiceList', 'Services'])) + if (columnname == 'Action'): + if (columnvalue not in ['DIPP_SRC_NAT', 'Dipp_Src_Nat', 'Dipp_src_nat', 'dipp_src_nat']): + log(f'ROW {i + 3} : Action "{columnvalue}" is not a valid option, it should be either DIPP_SRC_NAT/Dipp_Src_Nat/Dipp_src_nat/dipp_src_nat.') + fwpolicynatrule_invalid_check = True + if (columnname == 'Type'): + if (columnvalue not in ['NATV4', 'Natv4', 'natv4', 'NatV4']): + log(f'ROW {i + 3} : Action "{columnvalue}" is not a valid option, it should be either NATV4/Natv4/NatV4/natv4.') + fwpolicynatrule_invalid_check = True + + if (columnname == 'Position'): + if columnvalue.lower() != 'nan': + post = columnvalue.split('::') + if len(post) != 2: + log(f'ROW {i + 3} : Position value in "{post}" does not have all/correct required details') + fwpolicynatrule_invalid_check = True + else: + if (post[0] not in ['before_rule', 'after_rule']): + log(f'ROW {i + 3} : Position condition in "{post[0]}" is not a valid option, it should be either before_rule/after_rule') + if post[1].lower() != 'nan': + fwpolicyname = str(dffwpolicynatrule.loc[i, 'Firewall Policy']).strip() + finalrulepost = fwpolicyname + '::' + post[1] + fwpolicynatrulepost_check.append( + compare_values(fullnatrulelist.tolist(), finalrulepost,[i, 'Position', 'Firewall-Policy-TunnelInspect', 'Rule name'])) + + if any([fwpolicynatrule_empty_check, fwpolicynatrule_invalid_check, + fwpolicynatrule_nameg_length]) or any(fwpolicynatrule_check) or any( + fwpolicynatrulesa_check) or any(fwpolicynatruleda_check) or any( + fwpolicynatrulepost_check): + print("Null or Wrong value Check failed!!") + return True + else: + return False def validate_FirewallPolicySecurityRule(filename, fwpolicy_list, fulladdreslist, fullservicelist, fullappslist, fullurlslist,ct): fwpolicysecurityrule_empty_check = False @@ -974,10 +1094,10 @@ def validate_FirewallPolicySecurityRule(filename, fwpolicy_list, fulladdreslist, fwpolicysecurityruleul_check = [] fwpolicysecurityrule_action_mistake = False fwpolicysecurityrulepost_check = [] - dffwpolicysecurityrule = data_frame(filename, 'Firewall-Policy-SecRule') + dffwpolicysecurityrule = commonTools.data_frame(filename, 'Firewall-Policy-SecRule') dfcolumns = dffwpolicysecurityrule.columns.values.tolist() - dffwsecurityrule = data_frame(filename, 'Firewall-Policy-SecRule') + dffwsecurityrule = commonTools.data_frame(filename, 'Firewall-Policy-SecRule') dffwsecurityrule_list = dffwsecurityrule['Rule Name'].astype(str) dffwsecurityrulepolicy_list = dffwsecurityrule['Firewall Policy'].astype(str) fullsecurityrulelist = dffwsecurityrulepolicy_list + '::' + dffwsecurityrule_list @@ -1101,7 +1221,7 @@ def validate_compartments(filename,ct): comp_invalid_check = False parent_comp_check= False # Read the Compartments tab from excel - dfcomp = data_frame(filename ,'Compartments') + dfcomp = commonTools.data_frame(filename ,'Compartments') for i in dfcomp.index: region = str(dfcomp.loc[i, 'Region']).strip().lower() @@ -1170,35 +1290,38 @@ def validate_firewall_cd3(filename, var_file, prefix, outdir, config,signer,ct): print("Getting Compartments OCIDs...") ct.get_compartment_map(var_file,'Validator') - dffwpolicy = data_frame(filename, 'Firewall-Policy') + dffwpolicy = commonTools.data_frame(filename, 'Firewall-Policy') fwpolicy_list = dffwpolicy['Policy Name'].astype(str) - dffwaddress = data_frame(filename, 'Firewall-Policy-AddressList') + dffwaddress = commonTools.data_frame(filename, 'Firewall-Policy-AddressList') dffwaddress_list = dffwaddress['List Name'].astype(str) dffwaddresspolicy_list = dffwaddress['Firewall Policy'].astype(str) fulladdreslist = dffwaddresspolicy_list + '::' + dffwaddress_list - dffwservice = data_frame(filename, 'Firewall-Policy-ServiceList') + dffwservice = commonTools.data_frame(filename, 'Firewall-Policy-ServiceList') dffwservice_list = dffwservice['Service List'].astype(str) + dffwnatservice_list = dffwservice['Services'].astype(str) + #spdffwnatservice_list = dffwnatservice_list.str.split(pat="::",n=1).astype(str) dffwservicepolicy_list = dffwservice['Firewall Policy'].astype(str) fullservicelist = dffwservicepolicy_list + '::' + dffwservice_list + fullnatservicelist = dffwservicepolicy_list + '&' + dffwnatservice_list - dffwapps = data_frame(filename, 'Firewall-Policy-ApplicationList') + dffwapps = commonTools.data_frame(filename, 'Firewall-Policy-ApplicationList') dffwapps_list = dffwapps['Application List'].astype(str) dffwappspolicy_list = dffwapps['Firewall Policy'].astype(str) fullappslist = dffwappspolicy_list + '::' + dffwapps_list - dffwurls = data_frame(filename, 'Firewall-Policy-UrlList') + dffwurls = commonTools.data_frame(filename, 'Firewall-Policy-UrlList') dffwurls_list = dffwurls['List Name'].astype(str) dffwurlspolicy_list = dffwurls['Firewall Policy'].astype(str) fullurlslist = dffwurlspolicy_list + '::' + dffwurls_list - dffwsecrets = data_frame(filename, 'Firewall-Policy-Secret') + dffwsecrets = commonTools.data_frame(filename, 'Firewall-Policy-Secret') dffwsecrets_list = dffwsecrets['Secret Name'].astype(str) dffwsecrestspolicy_list = dffwsecrets['Firewall Policy'].astype(str) fullsecretslist = dffwsecrestspolicy_list + '::' + dffwsecrets_list - dffwdecrypt = data_frame(filename, 'Firewall-Policy-DecryptProfile') + dffwdecrypt = commonTools.data_frame(filename, 'Firewall-Policy-DecryptProfile') dffwdecrypt_list = dffwdecrypt['Decryption Profile Name'].astype(str) dffwdecryptpolicy_list = dffwdecrypt['Firewall Policy'].astype(str) fulldecryptlist = dffwdecryptpolicy_list + '::' + dffwdecrypt_list @@ -1233,10 +1356,14 @@ def validate_firewall_cd3(filename, var_file, prefix, outdir, config,signer,ct): log("\n============================= Verifying Firewall-Policy-SecRule Tab ==========================================\n") print("\nProcessing Firewall-Policy-Secrules Tab..") fw_policysecurityrule_check = validate_FirewallPolicySecurityRule(filename, fwpolicy_list, fulladdreslist, fullservicelist, fullappslist, fullurlslist,ct) + log("\n============================= Verifying Firewall-Policy-TunnelInspect Tab ==========================================\n") print("\nProcessing Firewall-Policy-TunnelInspect Tab..") fw_policytunnelinspect_check = validate_FirewallPolicyTunnelInspectRule(filename, fwpolicy_list, fulladdreslist, ct) + log("\n============================= Verifying Firewall-Policy-NatRules Tab ==========================================\n") + print("\nProcessing Firewall-Policy-NatRules Tab..") + fw_policynatrules_check = validate_FirewallPolicyNatRule(filename, fwpolicy_list, fulladdreslist,fullnatservicelist, ct) # Prints the final result; once the validation is complete - if any([Firewall_check, fw_policy_check, fw_policyapp_check, fw_policyurl_check, fw_policyservice_check, fw_policyaddress_check, fw_policysecrets_check, fw_policydecryption_check, fw_policydecryptionrule_check, fw_policysecurityrule_check, fw_policytunnelinspect_check]): + if any([Firewall_check, fw_policy_check, fw_policyapp_check, fw_policyurl_check, fw_policyservice_check, fw_policyaddress_check, fw_policysecrets_check, fw_policydecryption_check, fw_policydecryptionrule_check, fw_policysecurityrule_check, fw_policytunnelinspect_check, fw_policynatrules_check]): log("=======") log("Summary:") log("=======") diff --git a/cd3_automation_toolkit/ocicloud/python/cd3Services.py b/cd3_automation_toolkit/ocicloud/python/cd3Services.py new file mode 100644 index 000000000..a88dcf373 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/python/cd3Services.py @@ -0,0 +1,138 @@ +from oci.identity import IdentityClient +import oci +import os +import xml.etree.ElementTree as ET +import datetime +import ssl +import pathlib +import urllib +import shutil +import sys +class cd3Services(): + + #Get OCI Cloud Regions + regions_list = "" + def fetch_regions(self,config,signer): + #config = oci.config.from_file(file_location=configFileName) + idc = IdentityClient(config=config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY,signer=signer) + try: + regions_list = idc.list_regions().data + except Exception as e: + print(e) + if ('NotAuthenticated' in str(e)): + print("\nInvalid Credetials - check your keypair/fingerprint/region...Exiting!!!") + exit(1) + + if ("OCSWorkVM" in os.getcwd() or 'user-scripts' in os.getcwd()): + os.chdir("../") + + tempStr = '#Region:Region_Key\n' + reg_dict = {} + + for reg in regions_list: + cd3key = str(reg.name.split('-',1)[1]).lower() + + if 'dcc' in cd3key: + cd3key = str(cd3key.split('-',1)[1]).lower() + + name = str(reg.name).lower() + reg_dict[cd3key] = name + + keys = [] + new_reg_dict={} + for key,val in reg_dict.items(): + keyy = key.split("-")[0] + if keyy not in keys: + keys.append(keyy) + new_reg_dict[keyy]=val + else: + new_reg_dict[key] = val + + #replace prev + old_val = new_reg_dict[keyy] + old_val_key = str(old_val.split('-', 1)[1]).lower() + if 'dcc' in old_val_key: + old_val_key = str(old_val_key.split('-', 1)[1]).lower() + + new_reg_dict[old_val_key] = old_val + new_reg_dict.pop(keyy) + + for cd3key,name in new_reg_dict.items(): + line = cd3key + ":" + name + tempStr = tempStr + line + '\n' + + with open('OCI_Regions', 'w+') as f: + f.write(tempStr) + f.close() + print("Updated OCI_Regions file !!!\n") + + # Parse XML - Used by OCI Protocols + def parse_xml(source: str) -> ET.Element: + it = ET.iterparse(open(source)) + # strip namespaces + for _, el in it: + if "}" in el.tag: + el.tag = el.tag.split("}", 1)[1] + root = it.root # mypy: ignore + return root + + # Parse Date - Used by OCI Protocols + def parse_date(root_xml: ET.Element) -> datetime: + updated = root_xml.find("updated") + assert updated is not None and isinstance(updated.text, str) + return datetime.datetime.strptime(updated.text, "%Y-%m-%d") + + # write_protocols_file - Used for OCI Protocols + def write_protocols_file(source: str, destination: str) -> datetime: + root = cd3Services.parse_xml(source) + updated = cd3Services.parse_date(root) + destination = str(pathlib.Path.cwd())+"/"+destination + with open(destination,"w+") as dst: + dst.write("#protocol number:protocol name\n") + for r in root.iter("record"): + desc_ = r.find("description") + if desc_ is None or desc_.text is None: + desc = "" + else: + desc = desc_.text + name_ = r.find("name") + value_ = r.find("value") + if (value_ is None + or value_.text is None): + continue + if (name_ is None + or name_.text is None): + name = desc + dst.write(str(value_.text) + ":" + name + "\n") + continue + alias = name_.text.split()[0] + value = int(value_.text) + dst.write(str(value)+":"+alias+"\n") + return updated + + def download(url: str, path: str) -> None: + with open(path, "wb") as dst, urllib.request.urlopen(url) as src: + shutil.copyfileobj(src, dst) + + #Get OCI Protocols + def fetch_protocols(self) -> None: + + ssl._create_default_https_context = ssl._create_unverified_context + PROTOCOLS_URL = "https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xml" + PROTOCOLS_XML = "protocol-numbers.xml" + PROTOCOLS_FILE = "OCI_Protocols" + + ##### main code for oci protocols #### + protocols_xml = str(pathlib.Path.cwd()) + "/" + PROTOCOLS_XML + try: + cd3Services.download(PROTOCOLS_URL, protocols_xml) + except OSError as e: + print("Could not download iana service names and port numbers: {}".format(e), + file=sys.stderr, + ) + sys.exit(1) + cd3Services.write_protocols_file(protocols_xml, PROTOCOLS_FILE) + rem_file = pathlib.Path(protocols_xml) + rem_file.unlink() + + print("Updated OCI_Protocols file !!!\n") \ No newline at end of file diff --git a/cd3_automation_toolkit/cd3Validator.py b/cd3_automation_toolkit/ocicloud/python/cd3Validator.py similarity index 98% rename from cd3_automation_toolkit/cd3Validator.py rename to cd3_automation_toolkit/ocicloud/python/cd3Validator.py index 7045be5d7..c08f842c7 100644 --- a/cd3_automation_toolkit/cd3Validator.py +++ b/cd3_automation_toolkit/ocicloud/python/cd3Validator.py @@ -14,8 +14,12 @@ import os from functools import partial import inspect +import sys from oci.core.virtual_network_client import VirtualNetworkClient -from commonTools import * +sys.path.append(os.getcwd()+"/..") +from common.python.commonTools import * +from ocicloud.python.ociCommonTools import * +#import ocicloud.python.ociCommonTools as ociCommonTools ''' def get_vcn_ids(compartment_ids, config): @@ -163,7 +167,7 @@ def validate_nsgs_column(i,region,columnvalue,subnet_name,subnetobj,vcn_nsg_list def fetch_vcn_cidrs(filename): vcn_cidrs = {} # List of the column headers - dfv = data_frame(filename, 'VCNs') + dfv = commonTools.data_frame(filename, 'VCNs') dfcolumns = dfv.columns.values.tolist() # Loop through each row @@ -204,8 +208,8 @@ def validate_subnets(filename, comp_ids, vcnobj): log("Start Null or Wrong value check in each row-----------------") - dfsub = data_frame(filename, 'SubnetsVLANs') - dfdhcp = data_frame(filename, 'DHCP') + dfsub = commonTools.data_frame(filename, 'SubnetsVLANs') + dfdhcp = commonTools.data_frame(filename, 'DHCP') # List of the column headers dfcolumns = dfsub.columns.values.tolist() @@ -316,7 +320,7 @@ def validate_subnets(filename, comp_ids, vcnobj): for j in dfsub.keys(): if (str(dfsub[j][i]).strip() == "NaN" or str(dfsub[j][i]).strip() == "nan" or str(dfsub[j][i]).strip() == ""): # only dhcp_option_name, route table name, seclist_names and dns_label columns can be empty - if j in labels or commonTools.check_column_headers(j) in commonTools.tagColumns or "ipv6" in j.lower(): + if j in labels or commonTools.check_column_headers(j) in ociCommonTools.tagColumns or "ipv6" in j.lower(): pass else: if j == "Type(private|public)" and subnet_or_vlan.lower() == "vlan": @@ -377,7 +381,7 @@ def validate_subnets(filename, comp_ids, vcnobj): def validate_vcns(filename, comp_ids, vcnobj):# config): # ,vcn_cidrs,vcn_compartment_ids): #vcn_ids = get_vcn_ids(comp_ids, config) - dfv = data_frame(filename, 'VCNs') + dfv = commonTools.data_frame(filename, 'VCNs') # Counter to fetch the row number count = 0 @@ -460,7 +464,7 @@ def validate_vcns(filename, comp_ids, vcnobj):# config): # ,vcn_cidrs,vcn_compa # Check for null values and display appropriate message for j in dfv.keys(): if (str(dfv[j][i]).strip() == "NaN" or str(dfv[j][i]).strip() == "nan" or str(dfv[j][i]).strip() == ""): - if j == 'DNS Label' or commonTools.check_column_headers(j) in commonTools.tagColumns or "ipv6" in j.lower() or "Hub/Spoke" in j: + if j == 'DNS Label' or commonTools.check_column_headers(j) in ociCommonTools.tagColumns or "ipv6" in j.lower() or "Is Oracle GUA" in j or "Hub/Spoke" in j: continue else: log(f'ROW {count+2} : Empty value at column "{j}".') @@ -540,7 +544,7 @@ def validate_vcns(filename, comp_ids, vcnobj):# config): # ,vcn_cidrs,vcn_compa # Checks if the fields in DHCP tab are compliant def validate_dhcp(filename, comp_ids, vcnobj): - dfdhcp = data_frame(filename, 'DHCP') + dfdhcp = commonTools.data_frame(filename, 'DHCP') empty = ['', 'Nan', 'NaN', 'nan'] dhcp_empty_check = False dhcp_wrong_check = False @@ -598,7 +602,7 @@ def validate_dhcp(filename, comp_ids, vcnobj): else: # Check if there are any field that is empty; display appropriate message if str(dfdhcp[j][i]).strip() in empty and j != 'Search Domain' and commonTools.check_column_headers( - j) not in commonTools.tagColumns: + j) not in ociCommonTools.tagColumns: log(f'ROW {count+2} : Empty value at column {j}.') dhcp_empty_check = True @@ -613,7 +617,7 @@ def validate_dhcp(filename, comp_ids, vcnobj): # Checks if the fields in DRGv2 tab are compliant def validate_drgv2(filename, comp_ids, vcnobj): - dfdrgv2 = data_frame(filename, 'DRGs') + dfdrgv2 = commonTools.data_frame(filename, 'DRGs') drgv2_empty_check = False drgv2_invalid_check = False drgv2_comp_check = False @@ -713,9 +717,9 @@ def validate_dns(filename,comp_ids): subnet_check = False nsg_check = False endpoint_type_check = False - dfdns = data_frame(filename, 'DNS-Views-Zones-Records') + dfdns = commonTools.data_frame(filename, 'DNS-Views-Zones-Records') dfdnscolumns = dfdns.columns.values.tolist() - dfres = data_frame(filename, 'DNS-Resolvers') + dfres = commonTools.data_frame(filename, 'DNS-Resolvers') dfrescolumns = dfres.columns.values.tolist() log(f'Checking for DNS-Views-Zones-Records') for i in dfdns.index: @@ -841,7 +845,7 @@ def validate_instances(filename,comp_ids,subnetobj,vcn_subnet_list,vcn_nsg_list) vcn_subnet_check = False vcn_nsg_check= False - dfinst = data_frame(filename, 'Instances') + dfinst = commonTools.data_frame(filename, 'Instances') dfcolumns = dfinst.columns.values.tolist() for i in dfinst.index: @@ -959,8 +963,8 @@ def validate_blockvols(filename,comp_ids): instance_name_check = False bv_ad_check = False ADS = ["AD1","AD2","AD3"] - dfvol = data_frame(filename, 'BlockVolumes') - dfinst = data_frame(filename, 'Instances') + dfvol = commonTools.data_frame(filename, 'BlockVolumes') + dfinst = commonTools.data_frame(filename, 'Instances') values_list = dfinst['Display Name'].tolist() inst_ad_list = dfinst['Display Name']+'_'+dfinst['Availability Domain(AD1|AD2|AD3)'] dfcolumns = dfvol.columns.values.tolist() @@ -1112,7 +1116,7 @@ def validate_fss(filename,comp_ids,subnetobj,vcn_subnet_list,vcn_nsg_list): vcn_subnet_check = False vcn_nsg_check= False - df_fss = data_frame(filename, 'FSS') + df_fss = commonTools.data_frame(filename, 'FSS') dfcolumns = df_fss.columns.values.tolist() for i in df_fss.index: @@ -1180,7 +1184,7 @@ def validate_compartments(filename): comp_invalid_check = False parent_comp_check= False # Read the Compartments tab from excel - dfcomp = data_frame(filename,'Compartments') + dfcomp = commonTools.data_frame(filename,'Compartments') for i in dfcomp.index: region = str(dfcomp.loc[i, 'Region']).strip().lower() @@ -1213,7 +1217,7 @@ def validate_groups(filename): groups_empty_check = False groups_invalid_check = False # Read the Groups tab from excel - dfg = data_frame(filename, 'Groups') + dfg = commonTools.data_frame(filename, 'Groups') for i in dfg.index: region = str(dfg.loc[i, 'Region']).strip().lower() @@ -1241,7 +1245,7 @@ def validate_policies(filename,comp_ids): policies_invalid_check = False # Read the Policies tab from excel - dfp = data_frame(filename,'Policies') + dfp = commonTools.data_frame(filename,'Policies') for i in dfp.index: region = str(dfp.loc[i, 'Region']).strip().lower() @@ -1333,7 +1337,7 @@ def validate_tags(filename,comp_ids): tag_comp_check = False # Read the Compartments tab from excel - dftag = data_frame(filename,'Tags') + dftag = commonTools.data_frame(filename,'Tags') for i in dftag.index: region = str(dftag.loc[i, 'Region']).strip().lower() @@ -1415,7 +1419,7 @@ def validate_budgets(filename,comp_ids): # Read the Compartments tab from excel - dfbudget = data_frame(filename, 'Budgets') + dfbudget = commonTools.data_frame(filename, 'Budgets') for i in dfbudget.index: region = str(dfbudget.loc[i, 'Region']).strip().lower() @@ -1460,9 +1464,9 @@ def validate_budgets(filename,comp_ids): budget_check_result.append(False) if budget_check_result and False in budget_check_result: - return False - else: return True + else: + return False def validate_buckets(filename, comp_ids): @@ -1474,7 +1478,7 @@ def validate_buckets(filename, comp_ids): bucket_name_check = False # Read the Compartments tab from excel - dfbuckets = data_frame(filename, 'Buckets') + dfbuckets = commonTools.data_frame(filename, 'Buckets') for i in dfbuckets.index: region = str(dfbuckets.loc[i, 'Region']).strip().lower() @@ -1734,7 +1738,7 @@ def validate_buckets(filename, comp_ids): #validate_kms def validate_kms(filename,comp_ids): - dfkms = data_frame(filename, 'KMS') + dfkms = commonTools.data_frame(filename, 'KMS') kms_invalid_check = False prev_vault_type = "" @@ -1951,12 +1955,11 @@ def validate_cd3(choices, filename, var_file, prefix, outdir, ct1): #config1, si vcnobj = parseVCNs(filename) subnetobj = parseSubnets(filename) - dfsub = data_frame(filename, 'SubnetsVLANs') + dfsub = commonTools.data_frame(filename, 'SubnetsVLANs') vcn_subnet_list = dfsub['VCN Name'].astype(str)+'_'+dfsub['Display Name'] - dfnsg = data_frame(filename, 'NSGs') + dfnsg = commonTools.data_frame(filename, 'NSGs') vcn_nsg_list = dfnsg['Region'].astype(str).str.lower() + '_' + dfnsg['VCN Name'].astype(str) + '_' + dfnsg['NSG Name'] - val_net=False for options in choices: if ('Validate Compartments' in options[0]): @@ -1985,7 +1988,7 @@ def validate_cd3(choices, filename, var_file, prefix, outdir, ct1): #config1, si print("\nValidating Budgets Tab..") budgets_check = validate_budgets(filename,all_comp_ocids) errors = budgets_check - final_check.append(budgets_check) + #final_check.append(budgets_check) if ('Validate KMS' in options[0]): log("\n============================= Verifying KMS Tab ==========================================\n") @@ -2050,7 +2053,9 @@ def validate_cd3(choices, filename, var_file, prefix, outdir, ct1): #config1, si errors = buckets_check # Prints the final result; once the validation is complete - if any([comp_check, groups_check, policies_check, tags_check, instances_check, dns_check, bvs_check,fss_check, vcn_check, vcn_cidr_check, vcn_peer_check, subnet_check, subnet_cidr_check, dhcp_check, drgv2_check,buckets_check, kms_check]) or False in final_check: + final_errors = False + if any([comp_check, groups_check, policies_check, tags_check, instances_check, dns_check, bvs_check,fss_check, vcn_check, vcn_cidr_check, vcn_peer_check, subnet_check, subnet_cidr_check, dhcp_check, drgv2_check,buckets_check, kms_check, budgets_check]):# or True in final_check: + final_errors = True log("=======") log("Summary:") log("=======") @@ -2080,7 +2085,8 @@ def validate_cd3(choices, filename, var_file, prefix, outdir, ct1): #config1, si print("Invalid Choice....Exiting!!") exit(1) - if inspect.stack()[1].function == 'validate_cd3' or errors: + #if inspect.stack()[1].function == 'validate_cd3' or final_errors: + if final_errors: print("Please check the log file at " + customer_tenancy_dir + "/" + file + "\n") del(log) diff --git a/cd3_automation_toolkit/Compute/__init__.py b/cd3_automation_toolkit/ocicloud/python/compute/__init__.py similarity index 100% rename from cd3_automation_toolkit/Compute/__init__.py rename to cd3_automation_toolkit/ocicloud/python/compute/__init__.py diff --git a/cd3_automation_toolkit/Compute/create_terraform_dedicatedhosts.py b/cd3_automation_toolkit/ocicloud/python/compute/create_terraform_dedicatedhosts.py similarity index 94% rename from cd3_automation_toolkit/Compute/create_terraform_dedicatedhosts.py rename to cd3_automation_toolkit/ocicloud/python/compute/create_terraform_dedicatedhosts.py index 95b04ba64..13c534637 100644 --- a/cd3_automation_toolkit/Compute/create_terraform_dedicatedhosts.py +++ b/cd3_automation_toolkit/ocicloud/python/compute/create_terraform_dedicatedhosts.py @@ -14,8 +14,9 @@ from oci.config import DEFAULT_LOCATION from pathlib import Path -sys.path.append(os.getcwd() + "/../..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from jinja2 import Environment, FileSystemLoader @@ -92,8 +93,8 @@ def create_terraform_dedicatedhosts(inputfile, outdir, service_dir,prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'Display Name': columnvalue = columnvalue.strip() diff --git a/cd3_automation_toolkit/Compute/create_terraform_instances.py b/cd3_automation_toolkit/ocicloud/python/compute/create_terraform_instances.py similarity index 97% rename from cd3_automation_toolkit/Compute/create_terraform_instances.py rename to cd3_automation_toolkit/ocicloud/python/compute/create_terraform_instances.py index e94057b2b..0503cc7fc 100755 --- a/cd3_automation_toolkit/Compute/create_terraform_instances.py +++ b/cd3_automation_toolkit/ocicloud/python/compute/create_terraform_instances.py @@ -14,9 +14,9 @@ import fnmatch from oci.config import DEFAULT_LOCATION from pathlib import Path - -sys.path.append(os.getcwd() + "/../..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from jinja2 import Environment, FileSystemLoader @@ -115,8 +115,8 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname in plugin_column: columnvalue = columnvalue.strip() diff --git a/cd3_automation_toolkit/Compute/export_dedicatedvmhosts_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/compute/export_dedicatedvmhosts_nonGreenField.py similarity index 91% rename from cd3_automation_toolkit/Compute/export_dedicatedvmhosts_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/compute/export_dedicatedvmhosts_nonGreenField.py index b21412fb5..5cb2488c2 100644 --- a/cd3_automation_toolkit/Compute/export_dedicatedvmhosts_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/compute/export_dedicatedvmhosts_nonGreenField.py @@ -9,17 +9,18 @@ # import oci -import os +import os, sys import subprocess as sp from oci.config import DEFAULT_LOCATION -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * importCommands = {} oci_obj_names = {} -def print_dedicatedvmhosts(region, dedicatedvmhost, values_for_column, ntk_compartment_name,export_tags, state): +def print_dedicatedvmhosts(region, dedicatedvmhost, values_for_column, ntk_compartment_name,export_compartments, export_regions, export_tags, state): # Tags filter defined_tags = dedicatedvmhost.defined_tags @@ -58,11 +59,11 @@ def print_dedicatedvmhosts(region, dedicatedvmhost, values_for_column, ntk_compa elif ("AD-3" in value or "ad-3" in value): ad = "AD3" values_for_column[col_header].append(ad) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(dedicatedvmhost, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(dedicatedvmhost, col_header, values_for_column) else: oci_objs = [dedicatedvmhost] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) # Execution of the code begins here def export_dedicatedvmhosts(inputfile, outdir, service_dir, config, signer, ct, export_compartments=[], export_regions=[],export_tags=[]): diff --git a/cd3_automation_toolkit/Compute/export_instances_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/compute/export_instances_nonGreenField.py similarity index 97% rename from cd3_automation_toolkit/Compute/export_instances_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/compute/export_instances_nonGreenField.py index f04f23403..74502df66 100644 --- a/cd3_automation_toolkit/Compute/export_instances_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/compute/export_instances_nonGreenField.py @@ -8,9 +8,9 @@ import oci import os import subprocess as sp -sys.path.append(os.getcwd() + "/..") -from commonTools import * - +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools def adding_columns_values(region, ad, fd, vs, publicip, privateip, os_dname, shape, key_name, c_name, bkp_policy_name, nsgs, d_host, instance_data, values_for_column_instances, bdet, @@ -68,12 +68,12 @@ def adding_columns_values(region, ad, fd, vs, publicip, privateip, os_dname, sha values_for_column_instances[col_header].append(d_host.data.display_name) elif (col_header == "Custom Policy Compartment Name"): values_for_column_instances[col_header].append(cpcn) - elif str(col_header).lower() in commonTools.tagColumns: - values_for_column_instances = commonTools.export_tags(instance_data, col_header, + elif str(col_header).lower() in ociCommonTools.tagColumns: + values_for_column_instances = ociCommonTools.export_tags(instance_data, col_header, values_for_column_instances) else: oci_objs = [instance_data, bdet, shape_config, vnic_info, d_host, launch_options, avail_config, ins_options, platform_config] - values_for_column_instances = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_instances, + values_for_column_instances = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_instances, values_for_column_instances) diff --git a/cd3_automation_toolkit/Compute/templates/dedicatedvmhosts-template b/cd3_automation_toolkit/ocicloud/python/compute/templates/dedicatedvmhosts-template similarity index 100% rename from cd3_automation_toolkit/Compute/templates/dedicatedvmhosts-template rename to cd3_automation_toolkit/ocicloud/python/compute/templates/dedicatedvmhosts-template diff --git a/cd3_automation_toolkit/Compute/templates/instances-template b/cd3_automation_toolkit/ocicloud/python/compute/templates/instances-template similarity index 100% rename from cd3_automation_toolkit/Compute/templates/instances-template rename to cd3_automation_toolkit/ocicloud/python/compute/templates/instances-template diff --git a/cd3_automation_toolkit/ocicloud/python/costmanagement/__init__.py b/cd3_automation_toolkit/ocicloud/python/costmanagement/__init__.py new file mode 100644 index 000000000..4c3a5c86d --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/python/costmanagement/__init__.py @@ -0,0 +1 @@ +from .budget import * \ No newline at end of file diff --git a/cd3_automation_toolkit/CostManagement/Budget/__init__.py b/cd3_automation_toolkit/ocicloud/python/costmanagement/budget/__init__.py similarity index 100% rename from cd3_automation_toolkit/CostManagement/Budget/__init__.py rename to cd3_automation_toolkit/ocicloud/python/costmanagement/budget/__init__.py diff --git a/cd3_automation_toolkit/CostManagement/Budget/create_terraform_budget.py b/cd3_automation_toolkit/ocicloud/python/costmanagement/budget/create_terraform_budget.py similarity index 95% rename from cd3_automation_toolkit/CostManagement/Budget/create_terraform_budget.py rename to cd3_automation_toolkit/ocicloud/python/costmanagement/budget/create_terraform_budget.py index 4170c90fb..bc4a249c7 100644 --- a/cd3_automation_toolkit/CostManagement/Budget/create_terraform_budget.py +++ b/cd3_automation_toolkit/ocicloud/python/costmanagement/budget/create_terraform_budget.py @@ -12,8 +12,10 @@ from oci.config import DEFAULT_LOCATION from pathlib import Path from jinja2 import Environment, FileSystemLoader -sys.path.append(os.getcwd() + "/../..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools + ###### # Required Inputs-CD3 excel file, Config file, prefix AND outdir @@ -37,7 +39,7 @@ def create_terraform_budgets(inputfile, outdir, service_dir, prefix,ct): budget_alert_template = env.get_template('budget-alert-rule-template') # Read CD3 - df = data_frame(filename, sheetName) + df = commonTools.data_frame(filename, sheetName) regions = df['Region'] regions.dropna() @@ -119,8 +121,8 @@ def create_terraform_budgets(inputfile, outdir, service_dir, prefix,ct): tempdict = {'description': columnvalue} # Process Freeform Tags and Defined Tags - elif columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + elif columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) elif columnname == "Scope": tempdict = {'target_type': columnvalue.upper().strip()} @@ -149,7 +151,7 @@ def create_terraform_budgets(inputfile, outdir, service_dir, prefix,ct): tempdict = {'budget_end_date': columnvalue.strip()} elif columnname == "Alert Recipients" and columnvalue != 'nan': - tempdict = {'alert_recipients': columnvalue.strip()} + tempdict = {'alert_recipients': columnvalue.strip().replace("\n", "\\n")} elif columnname == "Alert Message" and columnvalue != 'nan': tempdict = {'alert_message': columnvalue} diff --git a/cd3_automation_toolkit/CostManagement/Budget/export_budgets_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/costmanagement/budget/export_budgets_nonGreenField.py similarity index 94% rename from cd3_automation_toolkit/CostManagement/Budget/export_budgets_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/costmanagement/budget/export_budgets_nonGreenField.py index 44c0faac3..5e7a57b73 100644 --- a/cd3_automation_toolkit/CostManagement/Budget/export_budgets_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/costmanagement/budget/export_budgets_nonGreenField.py @@ -11,7 +11,9 @@ import os import subprocess as sp -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools sys.path.append(os.getcwd()+"/..") @@ -32,7 +34,7 @@ def print_budgets(values_for_columns, region, budget,budget_name,budget_alert_ru value = budget_name if budget else "" values_for_columns[col_header].append(value) - if (col_header == "Description"): + elif (col_header == "Description"): value = budget.description if budget else "" values_for_columns[col_header].append(value) elif (col_header == "Scope"): @@ -76,14 +78,14 @@ def print_budgets(values_for_columns, region, budget,budget_name,budget_alert_ru values_for_columns[col_header].append(budget_end_date) - elif col_header.lower() in commonTools.tagColumns: + elif col_header.lower() in ociCommonTools.tagColumns: if budget: - values_for_columns = commonTools.export_tags(budget, col_header, values_for_columns) + values_for_columns = ociCommonTools.export_tags(budget, col_header, values_for_columns) else: values_for_columns[col_header].append("") - if (col_header == "Alert Rules"): + elif (col_header == "Alert Rules"): alert_rule = "" if budget_alert_rule: alert_rule = str(budget_alert_rule.type)+"::"+str(budget_alert_rule.threshold) @@ -103,6 +105,7 @@ def print_budgets(values_for_columns, region, budget,budget_name,budget_alert_ru + # Execution of the code begins here def export_budgets_nongreenfield(inputfile, outdir, service_dir, config, signer, ct,export_regions=[],export_tags=[]): global importCommands @@ -184,6 +187,8 @@ def export_budgets_nongreenfield(inputfile, outdir, service_dir, config, signer, alert_id = "budgets/"+budget_id+"/alertRules/"+str(budget_alert_rule.id) if budget_tf_name in budget_done : budget = [] + # if budget_done and budget_tf_name == budget_done[-1]: + # continue print_budgets(values_for_column_budgets, region, budget,budget_name,budget_alert_rule,ct) budget_done.append(budget_tf_name) tf_resource = f'module.budget-alert-rules[\\"{alert_tf_name}\\"].oci_budget_alert_rule.alert_rule' diff --git a/cd3_automation_toolkit/CostManagement/Budget/templates/budget-alert-rule-template b/cd3_automation_toolkit/ocicloud/python/costmanagement/budget/templates/budget-alert-rule-template similarity index 100% rename from cd3_automation_toolkit/CostManagement/Budget/templates/budget-alert-rule-template rename to cd3_automation_toolkit/ocicloud/python/costmanagement/budget/templates/budget-alert-rule-template diff --git a/cd3_automation_toolkit/CostManagement/Budget/templates/budget-template b/cd3_automation_toolkit/ocicloud/python/costmanagement/budget/templates/budget-template similarity index 100% rename from cd3_automation_toolkit/CostManagement/Budget/templates/budget-template rename to cd3_automation_toolkit/ocicloud/python/costmanagement/budget/templates/budget-template diff --git a/cd3_automation_toolkit/Database/__init__.py b/cd3_automation_toolkit/ocicloud/python/database/__init__.py similarity index 100% rename from cd3_automation_toolkit/Database/__init__.py rename to cd3_automation_toolkit/ocicloud/python/database/__init__.py diff --git a/cd3_automation_toolkit/Database/create_terraform_adb.py b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_adb.py similarity index 74% rename from cd3_automation_toolkit/Database/create_terraform_adb.py rename to cd3_automation_toolkit/ocicloud/python/database/create_terraform_adb.py index 0ea9ac644..d2d067d06 100644 --- a/cd3_automation_toolkit/Database/create_terraform_adb.py +++ b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_adb.py @@ -8,11 +8,13 @@ # Oracle Consulting # Modified (TF Upgrade): Divya Das # -import os +import os, sys from jinja2 import Environment, FileSystemLoader from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### @@ -72,9 +74,9 @@ def create_terraform_adb(inputfile, outdir, service_dir, prefix, ct): # Check if values are entered for mandatory fields if str(df.loc[i, 'Region']).lower() == 'nan' or \ str(df.loc[i, 'Compartment Name']).lower() == 'nan' or \ - str(df.loc[i, 'CPU Core Count']).lower() == 'nan' or \ + str(df.loc[i, 'CPU Detail']).lower() == 'nan' or \ str(df.loc[i, 'DB Name']).lower() == 'nan': - print("\nRegion, Compartment Name, CPU Core Count and DB Name fields are mandatory. Please enter a value and try again !!") + print("\nRegion, Compartment Name, CPU Detail and DB Name fields are mandatory. Please enter a value and try again !!") print("\n** Exiting **") exit(1) @@ -96,14 +98,44 @@ def create_terraform_adb(inputfile, outdir, service_dir, prefix, ct): tempdict = {columnname: columnvalue} # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "ADB Display Name": display_tf_name = columnvalue.strip() display_tf_name = commonTools.check_tf_variable(display_tf_name) tempdict = {'display_tf_name': display_tf_name} + if columnname == 'CPU Detail': + columnvalue = columnvalue.strip() + cpu_detail = columnvalue.split("::") + tempdict = {'compute_model': cpu_detail[0],'compute_count':cpu_detail[1].split(".")[0]} + + if columnname == 'Is Auto Scaling Enabled': + columnvalue = columnvalue.strip() + tempdict = {'is_auto_scaling_enabled': columnvalue} + + if columnname == 'Data Storage Size' and columnvalue != '': + columnvalue = columnvalue.strip().lower() + data_storage = columnvalue.split("::") + if len(data_storage) > 1: + if data_storage[0] == "gb": + tempdict = {'data_storage_size_in_gb' : data_storage[1]} + elif data_storage[0] == "tb": + tempdict = {'data_storage_size_in_tbs': data_storage[1]} + elif len(data_storage) == 1: + tempdict = {'data_storage_size_in_tbs': columnvalue} + + if columnname.lower() == 'source' and columnvalue != '': + columnvalue = columnvalue.strip() + tempdict = {'adb_source': columnvalue} + + + if columnname == 'Database Workload': + columnvalue = columnvalue.strip() + autonomous_value = commonTools.check_tf_variable(columnvalue).lower() + tempdict = {'autonomous_value': autonomous_value} + if columnname == 'Database Workload': columnvalue = columnvalue.strip() autonomous_value = commonTools.check_tf_variable(columnvalue).lower() @@ -132,16 +164,27 @@ def create_terraform_adb(inputfile, outdir, service_dir, prefix, ct): subnet_id = '' network_compartment_id = '' + subnet_compartment_id = '' vcn_name = '' if columnname == "Network Details": columnvalue = columnvalue.strip() if ("ocid1.subnet.oc" in columnvalue): network_compartment_id = "root" vcn_name = "" + subnet_compartment_id = "root" subnet_id = columnvalue elif columnvalue.lower() != 'nan' and columnvalue.lower() != '': - if len(columnvalue.split("@")) == 2: + if len(columnvalue.split("@")) == 3: + network_compartment_id, part2 = columnvalue.split("@", 1) + network_compartment_id = commonTools.check_tf_variable(network_compartment_id) + vcn_name, part2 = part2.split("::", 1) + subnet_compartment_id, subnet_name = part2.rsplit("@", 1) + subnet_compartment_id = commonTools.check_tf_variable(subnet_compartment_id) + vcn_subnet_name = f'{vcn_name}::{subnet_name}' + + elif len(columnvalue.split("@")) == 2: network_compartment_id = commonTools.check_tf_variable(columnvalue.split("@")[0].strip()) + subnet_compartment_id = network_compartment_id vcn_subnet_name = columnvalue.split("@")[1].strip() else: network_compartment_id = commonTools.check_tf_variable( @@ -156,6 +199,7 @@ def create_terraform_adb(inputfile, outdir, service_dir, prefix, ct): tempdict = {'network_compartment_id': network_compartment_id, 'vcn_name': vcn_name, + 'subnet_compartment_id': subnet_compartment_id, 'subnet_id': subnet_id} if columnname == "License Model" and columnvalue.strip() == "LICENSE_INCLUDED": diff --git a/cd3_automation_toolkit/Database/create_terraform_dbsystems_vm_bm.py b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_dbsystems_vm_bm.py similarity index 97% rename from cd3_automation_toolkit/Database/create_terraform_dbsystems_vm_bm.py rename to cd3_automation_toolkit/ocicloud/python/database/create_terraform_dbsystems_vm_bm.py index 2d8595924..58a98caa3 100644 --- a/cd3_automation_toolkit/Database/create_terraform_dbsystems_vm_bm.py +++ b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_dbsystems_vm_bm.py @@ -8,12 +8,13 @@ # Oracle Consulting # -import sys -import os +import os, sys from jinja2 import Environment, FileSystemLoader from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### @@ -135,8 +136,8 @@ def create_terraform_dbsystems_vm_bm(inputfile, outdir, service_dir, prefix, ct) tempdict = {columnname: columnvalue} # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "DB System Display Name": display_tf_name = columnvalue.strip() diff --git a/cd3_automation_toolkit/Database/create_terraform_exa_infra.py b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_exa_infra.py similarity index 94% rename from cd3_automation_toolkit/Database/create_terraform_exa_infra.py rename to cd3_automation_toolkit/ocicloud/python/database/create_terraform_exa_infra.py index a2deb9963..ba99ecfab 100644 --- a/cd3_automation_toolkit/Database/create_terraform_exa_infra.py +++ b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_exa_infra.py @@ -8,11 +8,13 @@ # Oracle Consulting # -import os +import os, sys from jinja2 import Environment, FileSystemLoader from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### @@ -104,8 +106,8 @@ def create_terraform_exa_infra(inputfile, outdir, service_dir, prefix, ct): tempdict = {columnname: columnvalue} # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Exadata Infra Display Name": display_tf_name = columnvalue.strip() diff --git a/cd3_automation_toolkit/Database/create_terraform_exa_vmclusters.py b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_exa_vmclusters.py similarity index 97% rename from cd3_automation_toolkit/Database/create_terraform_exa_vmclusters.py rename to cd3_automation_toolkit/ocicloud/python/database/create_terraform_exa_vmclusters.py index 737261a6f..7b6f00216 100644 --- a/cd3_automation_toolkit/Database/create_terraform_exa_vmclusters.py +++ b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_exa_vmclusters.py @@ -8,11 +8,13 @@ # Oracle Consulting # Modified (TF Upgrade): Kartikey Rajput # -import os +import os, sys from jinja2 import Environment, FileSystemLoader from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### @@ -114,8 +116,8 @@ def create_terraform_exa_vmclusters(inputfile, outdir, service_dir, prefix, ct): tempdict = {columnname: columnvalue} # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "VM Cluster Display Name": display_tf_name = columnvalue.strip() diff --git a/cd3_automation_toolkit/Database/create_terraform_mysql_configuration.py b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_mysql_configuration.py similarity index 97% rename from cd3_automation_toolkit/Database/create_terraform_mysql_configuration.py rename to cd3_automation_toolkit/ocicloud/python/database/create_terraform_mysql_configuration.py index 3ca3504bb..ad9a79f94 100644 --- a/cd3_automation_toolkit/Database/create_terraform_mysql_configuration.py +++ b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_mysql_configuration.py @@ -8,12 +8,14 @@ # Oracle Consulting # -import os +import os, sys import re from jinja2 import Environment, FileSystemLoader from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools def create_terraform_mysql_configuration(inputfile, outdir, service_dir, prefix, ct): filename = inputfile diff --git a/cd3_automation_toolkit/Database/create_terraform_mysql_db.py b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_mysql_db.py similarity index 99% rename from cd3_automation_toolkit/Database/create_terraform_mysql_db.py rename to cd3_automation_toolkit/ocicloud/python/database/create_terraform_mysql_db.py index 35cd3fced..a5e73efae 100644 --- a/cd3_automation_toolkit/Database/create_terraform_mysql_db.py +++ b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_mysql_db.py @@ -13,7 +13,9 @@ from jinja2 import Environment, FileSystemLoader from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools def create_terraform_mysql_db(inputfile, outdir, service_dir, prefix, ct): ADS = ["AD1", "AD2", "AD3"] diff --git a/cd3_automation_toolkit/Database/export_adb_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/database/export_adb_nonGreenField.py similarity index 83% rename from cd3_automation_toolkit/Database/export_adb_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/database/export_adb_nonGreenField.py index 5691602fa..409846c8f 100644 --- a/cd3_automation_toolkit/Database/export_adb_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/database/export_adb_nonGreenField.py @@ -7,9 +7,11 @@ # Oracle Consulting # import oci -import os +import os, sys import subprocess as sp -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from oci.config import DEFAULT_LOCATION importCommands = {} @@ -23,20 +25,28 @@ def print_adbs(region, vnc_client, adb, values_for_column, ntk_compartment_name, for item in adb.customer_contacts: customer_emails += ","+item.email adb_subnet_id = adb.subnet_id - + cpu_detail = f'{adb.compute_model}::{str(adb.compute_count).split(".")[0]}' + if hasattr(adb, 'data_storage_size_in_gbs') : + data_storage = f'GB::{str(adb.data_storage_size_in_gbs)}' + # data_storage_size_in_tbs will be used over data_storage_size_in_gbs if available + if hasattr(adb, 'data_storage_size_in_tbs') and str(adb.data_storage_size_in_tbs) != "None": + data_storage = f'TB::{str(adb.data_storage_size_in_tbs)}' if (adb_subnet_id is not None): adb_subnet_info = vnc_client.get_subnet(adb_subnet_id) adb_subnet_name = adb_subnet_info.data.display_name # Subnet-Name adb_vcn_name = vnc_client.get_vcn(adb_subnet_info.data.vcn_id).data.display_name - + snet_comp_id = adb_subnet_info.data.compartment_id ntk_compartment_id = vnc_client.get_vcn(adb_subnet_info.data.vcn_id).data.compartment_id # compartment-id network_compartment_name = ntk_compartment_name for comp_name, comp_id in ct.ntk_compartment_ids.items(): if comp_id == ntk_compartment_id: network_compartment_name = comp_name - - vs = network_compartment_name + "@" + adb_vcn_name + "::" + adb_subnet_name - + if comp_id == snet_comp_id: + ntk_comp_name = comp_name + if ntk_compartment_id == snet_comp_id: + vs = network_compartment_name + "@" + adb_vcn_name + "::" + adb_subnet_name + else: + vs = network_compartment_name + "@" + adb_vcn_name + "::"+ ntk_comp_name + "@" + adb_subnet_name # Fetch NSGs NSGs = adb.nsg_ids @@ -79,10 +89,13 @@ def print_adbs(region, vnc_client, adb, values_for_column, ntk_compartment_name, values_for_column[col_header].append(adb.database_edition) else: values_for_column[col_header].append("") - elif col_header == 'CPU Core Count': - values_for_column[col_header].append(adb.cpu_core_count) - elif col_header == 'Data Storage Size in TB': - values_for_column[col_header].append(adb.data_storage_size_in_tbs) + elif col_header == 'CPU Detail': + values_for_column[col_header].append(cpu_detail) + elif col_header == 'Auto Scaling CPU': + values_for_column[col_header].append(str(adb.is_auto_scaling_enabled)) + elif col_header == 'Data Storage Size': + values_for_column[col_header].append(data_storage) + elif col_header == 'Database Workload': val= adb.db_workload if adb.db_workload == "DW": @@ -110,11 +123,11 @@ def print_adbs(region, vnc_client, adb, values_for_column, ntk_compartment_name, values_for_column[col_header].append(customer_emails.lstrip(',')) elif col_header == "NSGs": values_for_column[col_header].append(nsg_names) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(adb, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(adb, col_header, values_for_column) else: oci_objs = [adb] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) # Execution of the code begins here def export_adbs(inputfile, outdir, service_dir, config, signer, ct, export_compartments=[],export_regions=[],export_tags=[]): diff --git a/cd3_automation_toolkit/Database/export_dbsystems_vm_bm_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/database/export_dbsystems_vm_bm_nonGreenField.py similarity index 93% rename from cd3_automation_toolkit/Database/export_dbsystems_vm_bm_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/database/export_dbsystems_vm_bm_nonGreenField.py index d29b33caa..3058f68e6 100644 --- a/cd3_automation_toolkit/Database/export_dbsystems_vm_bm_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/database/export_dbsystems_vm_bm_nonGreenField.py @@ -8,9 +8,11 @@ # Oracle Consulting # import oci -import os +import os, sys from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from jinja2 import Environment, FileSystemLoader import json import re @@ -66,11 +68,11 @@ def print_dbsystem_vm_bm(region, db_system_vm_bm, count,db_home, database ,vnc_c elif col_header == 'DB Admin Password': values_for_column[col_header].append('nullval') - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(db_system_vm_bm, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(db_system_vm_bm, col_header, values_for_column) else: oci_objs = [db_home,database,db_backup_config,connection_strings,database_management_config] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) #count = 1 else: @@ -97,11 +99,11 @@ def print_dbsystem_vm_bm(region, db_system_vm_bm, count,db_home, database ,vnc_c values_for_column[col_header].append(ad) elif (col_header == "NSGs"): values_for_column[col_header].append(nsg_names) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(db_system_vm_bm, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(db_system_vm_bm, col_header, values_for_column) else: oci_objs = [db_system_vm_bm,db_system_options,maintenance_window,db_home,database,db_backup_config,connection_strings,database_management_config] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) # Execution of the code begins here diff --git a/cd3_automation_toolkit/Database/export_exa_infra_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_nonGreenField.py similarity index 92% rename from cd3_automation_toolkit/Database/export_exa_infra_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_nonGreenField.py index 20656d825..61032bdec 100644 --- a/cd3_automation_toolkit/Database/export_exa_infra_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_nonGreenField.py @@ -9,9 +9,11 @@ # import oci -import os +import os, sys import subprocess as sp -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools importCommands = {} oci_obj_names = {} @@ -39,11 +41,11 @@ def print_exa_infra(region, exa_infra, values_for_column, ntk_compartment_name,s elif ("AD-3" in value or "ad-3" in value): ad = "AD3" values_for_column[col_header].append(ad) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(exa_infra, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(exa_infra, col_header, values_for_column) else: oci_objs = [exa_infra,maintenance_window] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) # Execution of the code begins here diff --git a/cd3_automation_toolkit/Database/export_exa_vmclusters_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/database/export_exa_vmclusters_nonGreenField.py similarity index 96% rename from cd3_automation_toolkit/Database/export_exa_vmclusters_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/database/export_exa_vmclusters_nonGreenField.py index ca260a2fd..0d0e6567f 100644 --- a/cd3_automation_toolkit/Database/export_exa_vmclusters_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/database/export_exa_vmclusters_nonGreenField.py @@ -9,12 +9,14 @@ # import oci -import os +import os, sys import json import re import subprocess as sp from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from jinja2 import Environment, FileSystemLoader importCommands = {} @@ -89,11 +91,11 @@ def print_exa_vmcluster(region, vnc_client,exa_infra, exa_vmcluster, key_name,va values_for_column[col_header].append(backup_nsg_names) elif (col_header == "DB Servers"): values_for_column[col_header].append(db_servers) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(exa_vmcluster, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(exa_vmcluster, col_header, values_for_column) else: oci_objs = [exa_vmcluster,exa_infra] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) # Execution of the code begins here diff --git a/cd3_automation_toolkit/Database/export_mysql_configuration_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/database/export_mysql_configuration_nonGreenField.py similarity index 99% rename from cd3_automation_toolkit/Database/export_mysql_configuration_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/database/export_mysql_configuration_nonGreenField.py index f2de93436..a48308e78 100644 --- a/cd3_automation_toolkit/Database/export_mysql_configuration_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/database/export_mysql_configuration_nonGreenField.py @@ -8,8 +8,10 @@ # import oci -import os -from commonTools import * +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from oci.exceptions import ServiceError def export_mysql_configurations(inputfile, outdir, service_dir, config, signer, ct, export_regions=[], export_compartments=[]): diff --git a/cd3_automation_toolkit/Database/export_mysql_db_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/database/export_mysql_db_nonGreenField.py similarity index 96% rename from cd3_automation_toolkit/Database/export_mysql_db_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/database/export_mysql_db_nonGreenField.py index 3abec6b91..4a2c5a74f 100644 --- a/cd3_automation_toolkit/Database/export_mysql_db_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/database/export_mysql_db_nonGreenField.py @@ -7,9 +7,11 @@ # Oracle Consulting # import oci -import os +import os, sys import subprocess as sp -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from oci.config import DEFAULT_LOCATION importCommands = {} @@ -158,11 +160,11 @@ def print_mysql(region, vnc_client, mysql_db, values_for_column, ntk_compartment values_for_column[col_header].append(mysql_db.maintenance.window_start_time if mysql_db.maintenance else "") elif col_header == 'Database Management is Enabled': values_for_column[col_header].append(mysql_db.database_management if mysql_db.database_management else "DISABLED") - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(mysql_db, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(mysql_db, col_header, values_for_column) else: oci_objs = [mysql_db] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) def export_mysql_db(inputfile, outdir, service_dir, config, signer, ct, export_compartments=[], export_regions=[],export_tags=[]): global tf_import_cmd diff --git a/cd3_automation_toolkit/ocicloud/python/database/templates/adb-template b/cd3_automation_toolkit/ocicloud/python/database/templates/adb-template new file mode 100644 index 000000000..fdfc67c47 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/python/database/templates/adb-template @@ -0,0 +1,260 @@ +{% if count == 0 %} +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +############################ +# ADB +# ADB - tfvars +# Allowed Values: +# compartment_id and network_compartment_id can be the ocid or the name of the compartment hierarchy delimited by double hiphens "--" +# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Database--Prod" where "Database" is the parent of "Prod" compartment +############################ + +adb = { + ##Add New ADB for {{ region|lower }} here## +} +{% else %} + + {{ display_tf_name }} = { + + compartment_id = "{{ compartment_name }}" + db_name = "{{ db_name }}" + display_name = "{{ adb_display_name }}" + + admin_password = "{{ admin_password }}" + + {% if are_primary_whitelisted_ips_used and are_primary_whitelisted_ips_used.strip().lower() not in ["", "nan"] %} + are_primary_whitelisted_ips_used = {{ are_primary_whitelisted_ips_used | lower }} + {% endif %} + + {% if auto_refresh_frequency_in_seconds and auto_refresh_frequency_in_seconds.strip().lower() not in ["", "nan"] %} + auto_refresh_frequency_in_seconds = {{ auto_refresh_frequency_in_seconds }} + {% endif %} + + + {% if auto_refresh_point_lag_in_seconds and auto_refresh_point_lag_in_seconds.strip().lower() not in ["", "nan"] %} + auto_refresh_point_lag_in_seconds = {{ auto_refresh_point_lag_in_seconds }} + {% endif %} + + + {% if adb_source and adb_source.strip().lower() not in ["", "nan"] %} + adb_source = {{ adb_source }} + {% endif %} + + {% if source_id and source_id.strip().lower() not in ["", "nan"] %} + source_id = "{{ source_id }}" + {% endif %} + + {% if autonomous_database_source_backup_id and autonomous_database_source_backup_id.strip().lower() not in ["", "nan"] %} + autonomous_database_source_backup_id = "{{ autonomous_database_source_backup_id.strip() }}" + {% endif %} + + {% if autonomous_database_id and autonomous_database_id.strip().lower() not in ["", "nan"] %} + autonomous_database_id = "{{ autonomous_database_id.strip() }}" + {% endif %} + + {% if is_auto_scaling_for_storage_enabled and is_auto_scaling_for_storage_enabled.strip().lower() not in ["", "nan"] %} + is_auto_scaling_for_storage_enabled = {{ is_auto_scaling_for_storage_enabled | lower }} + {% endif %} + + {% if data_storage_size_in_gb and data_storage_size_in_gb.strip().lower() not in ["", "nan"] %} + data_storage_size_in_gb = {{ data_storage_size_in_gb }} + {% endif %} + + {% if data_storage_size_in_tbs and data_storage_size_in_tbs.strip().lower() not in ["", "nan"] %} + data_storage_size_in_tbs = {{ data_storage_size_in_tbs }} + {% endif %} + + {% if autonomous_maintenance_schedule_type and autonomous_maintenance_schedule_type.strip().lower() not in ["", "nan"] %} + autonomous_maintenance_schedule_type = "{{ autonomous_maintenance_schedule_type.strip() | upper }}" + {% endif %} + + {% if character_set != "" and character_set != "nan" %} + character_set = "{{ character_set }}" + {% endif %} + + compute_count = {{ compute_count }} + compute_model = "{{ compute_model }}" + + {% if customer_contacts and customer_contacts.strip().lower() not in ["", "nan"] %} + customer_contacts = {{ customer_contacts | replace("\'","\"") }} + {% endif %} + + {% if data_safe_status and data_safe_status.strip().lower() not in ["", "nan"] %} + data_safe_status = "{{ data_safe_status.strip() }}" + {% endif %} + + {% if database_edition == "" %} + database_edition = null + {% else %} + database_edition = "{{ database_edition }}" #Only for BYOL license model + {% endif %} + + {% if db_version and db_version.strip().lower() not in ["", "nan"] %} + db_version = "{{ db_version.strip() }}" + {% endif %} + + {% if autonomous_value == 'adw' %} + db_workload = "DW" + {% elif autonomous_value == 'atp' %} + db_workload = "OLTP" + {% elif autonomous_value == 'json' %} + db_workload = "AJD" + {% elif autonomous_value == 'apex' %} + db_workload = "APEX" + {% else %} + db_workload = "{{ autonomous_value }}" + {% endif %} + + {% if is_auto_scaling_enabled and is_auto_scaling_enabled.strip().lower() not in ["", "nan"] %} + is_auto_scaling_enabled = {{ is_auto_scaling_enabled | lower }} + {% endif %} + + + {% if is_dedicated and is_dedicated.strip().lower() not in ["", "nan"] %} + is_dedicated = {{ is_dedicated | lower }} + {% endif %} + + {% if autonomous_container_database_id and autonomous_container_database_id.strip().lower() not in ["", "nan"] %} + autonomous_container_database_id = "{{ autonomous_container_database_id.strip() }}" + {% endif %} + + {% if kms_key_id and kms_key_id.strip().lower() not in ["", "nan"] %} + kms_key_id = "{{ kms_key_id.strip() }}" + {% endif %} + + {% if vault_id and vault_id.strip().lower() not in ["", "nan"] %} + vault_id = "{{ vault_id.strip() }}" + {% endif %} + + + {% if in_memory_percentage and in_memory_percentage.strip().lower() not in ["", "nan"] %} + in_memory_percentage = {{ in_memory_percentage }} + {% endif %} + + + {% if is_local_data_guard_enabled and is_local_data_guard_enabled.strip().lower() not in ["", "nan"] %} + is_local_data_guard_enabled = {{ is_local_data_guard_enabled | lower }} + {% endif %} + + + {% if is_mtls_connection_required and is_mtls_connection_required.strip().lower() not in ["", "nan"] %} + is_mtls_connection_required = {{ is_mtls_connection_required | lower }} + {% endif %} + + {% if tde_kms_key_id and tde_kms_key_id.strip().lower() not in ["", "nan"] %} + tde_kms_key_id = "{{ tde_kms_key_id.strip() }}" + {% endif %} + + license_model = "{{ license_model }}" + + {% if ncharacter_set != "" and ncharacter_set != "nan" %} + ncharacter_set = "{{ ncharacter_set }}" + {% endif %} + + {% if ocpu_count and ocpu_count.strip().lower() not in ["", "nan"] %} + ocpu_count = {{ ocpu_count }} + {% endif %} + + {% if private_endpoint_ip and private_endpoint_ip.strip().lower() not in ["", "nan"] %} + private_endpoint_ip = "{{ private_endpoint_ip.strip() }}" + {% endif %} + + {% if private_endpoint_label and private_endpoint_label.strip().lower() not in ["", "nan"] %} + private_endpoint_label = "{{ private_endpoint_label.strip() }}" + {% endif %} + + {% if refreshable_mode and refreshable_mode.strip().lower() not in ["", "nan"] %} + refreshable_mode = "{{ refreshable_mode.strip() }}" + {% endif %} + + {% if time_of_auto_refresh_start and time_of_auto_refresh_start.strip().lower() not in ["", "nan"] %} + time_of_auto_refresh_start = "{{ time_of_auto_refresh_start.strip() }}" + {% endif %} + + {% if network_compartment_id == "" %} + network_compartment_id = null + {% else %} + network_compartment_id = "{{ network_compartment_id }}" + {% endif %} + {% if subnet_compartment_id and subnet_compartment_id.strip().lower() not in ["", "nan"] %} + subnet_compartment_id = "{{subnet_compartment_id}}" + {% else %} + subnet_compartment_id = null + {% endif %} + {% if subnet_id == "" %} + subnet_id = null + {% else %} + subnet_id = "{{ subnet_id }}" + {% endif %} + {% if vcn_name == "" %} + vcn_name = null + {% else %} + vcn_name = "{{ vcn_name }}" + {% endif %} + + nsg_ids = [{{ nsg_ids }}] + + {% if backup_retention_period_in_days and backup_retention_period_in_days.strip().lower() not in ["", "nan"] %} + backup_retention_period_in_days = {{ backup_retention_period_in_days }} + {% endif %} + + + {% if is_backup_retention_locked and is_backup_retention_locked.strip().lower() not in ["", "nan"] %} + is_backup_retention_locked = {{ is_backup_retention_locked | lower }} + {% endif %} + + + {% if is_replicate_automatic_backups and is_replicate_automatic_backups.strip().lower() not in ["", "nan"] %} + is_replicate_automatic_backups = {{ is_replicate_automatic_backups | lower }} + {% endif %} + + {% if remote_disaster_recovery_type and remote_disaster_recovery_type.strip().lower() not in ["", "nan"] %} + remote_disaster_recovery_type = "{{ remote_disaster_recovery_type.strip() }}" + {% endif %} + + {% if autonomous_database_timestamp and autonomous_database_timestamp.strip().lower() not in ["", "nan"] %} + timestamp = "{{ autonomous_database_timestamp.strip() }}" + {% endif %} + + + {% if use_latest_available_backup_time_stamp and use_latest_available_backup_time_stamp.strip().lower() not in ["", "nan"] %} + use_latest_available_backup_time_stamp = {{ use_latest_available_backup_time_stamp | lower }} + {% endif %} + + whitelisted_ips = [{{ whitelisted_ips }}] + + {# ##Do not modify below this line## #} + {# #} + {# ###Section for adding Defined and Freeform Tags### #} + {% if defined_tags and defined_tags != 'nan' and defined_tags != '' and defined_tags != [['nan']] %} + {% if defined_tags[0] %} + defined_tags = { + {% for tags in defined_tags %} + {% if not loop.last %} + "{{ tags[0] }}"= "{{ tags[1] }}" , + {% else %} + "{{ tags[0] }}"= "{{ tags[1] }}" + {% endif %} + {% endfor %} + } + {% endif %} + {% endif %} + {% if freeform_tags and freeform_tags != 'nan' and freeform_tags != '' and freeform_tags != [['nan']] %} + {% if freeform_tags[0] %} + freeform_tags = { + {% for tags in freeform_tags %} + {% if not loop.last %} + "{{ tags[0] }}"="{{ tags[1] }}", + {% else %} + "{{ tags[0] }}"="{{ tags[1] }}" + {% endif %} + {% endfor %} + } + {% endif %} + {% endif %} + {# ###Section for adding Defined and Freeform Tags ends here### #} + }, + +{% endif %} + diff --git a/cd3_automation_toolkit/Database/templates/dbsystems-vm-bm-template b/cd3_automation_toolkit/ocicloud/python/database/templates/dbsystems-vm-bm-template similarity index 100% rename from cd3_automation_toolkit/Database/templates/dbsystems-vm-bm-template rename to cd3_automation_toolkit/ocicloud/python/database/templates/dbsystems-vm-bm-template diff --git a/cd3_automation_toolkit/Database/templates/exa-infra-template b/cd3_automation_toolkit/ocicloud/python/database/templates/exa-infra-template similarity index 100% rename from cd3_automation_toolkit/Database/templates/exa-infra-template rename to cd3_automation_toolkit/ocicloud/python/database/templates/exa-infra-template diff --git a/cd3_automation_toolkit/Database/templates/exa-vmclusters-template b/cd3_automation_toolkit/ocicloud/python/database/templates/exa-vmclusters-template similarity index 100% rename from cd3_automation_toolkit/Database/templates/exa-vmclusters-template rename to cd3_automation_toolkit/ocicloud/python/database/templates/exa-vmclusters-template diff --git a/cd3_automation_toolkit/Database/templates/mysql-configuration-template b/cd3_automation_toolkit/ocicloud/python/database/templates/mysql-configuration-template similarity index 100% rename from cd3_automation_toolkit/Database/templates/mysql-configuration-template rename to cd3_automation_toolkit/ocicloud/python/database/templates/mysql-configuration-template diff --git a/cd3_automation_toolkit/Database/templates/mysql-template b/cd3_automation_toolkit/ocicloud/python/database/templates/mysql-template similarity index 100% rename from cd3_automation_toolkit/Database/templates/mysql-template rename to cd3_automation_toolkit/ocicloud/python/database/templates/mysql-template diff --git a/cd3_automation_toolkit/ocicloud/python/developerservices/__init__.py b/cd3_automation_toolkit/ocicloud/python/developerservices/__init__.py new file mode 100644 index 000000000..7fddee9e4 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/python/developerservices/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 + +from .resourcemanager import * +from .oke import * + diff --git a/cd3_automation_toolkit/DeveloperServices/OKE/__init__.py b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/__init__.py similarity index 100% rename from cd3_automation_toolkit/DeveloperServices/OKE/__init__.py rename to cd3_automation_toolkit/ocicloud/python/developerservices/oke/__init__.py diff --git a/cd3_automation_toolkit/DeveloperServices/OKE/create_terraform_oke.py b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/create_terraform_oke.py similarity index 98% rename from cd3_automation_toolkit/DeveloperServices/OKE/create_terraform_oke.py rename to cd3_automation_toolkit/ocicloud/python/developerservices/oke/create_terraform_oke.py index 30e95fc11..405ea1dc3 100644 --- a/cd3_automation_toolkit/DeveloperServices/OKE/create_terraform_oke.py +++ b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/create_terraform_oke.py @@ -7,13 +7,14 @@ # Author: Divya Das # Oracle Consulting # -import os +import os, sys import json import pandas as pd from oci.config import DEFAULT_LOCATION from pathlib import Path -import commonTools -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from jinja2 import Environment, FileSystemLoader @@ -175,8 +176,8 @@ def create_terraform_oke(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'Availability Domain(AD1|AD2|AD3)': columnname = 'availability_domain' diff --git a/cd3_automation_toolkit/DeveloperServices/OKE/export_oke_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/export_oke_nonGreenField.py similarity index 93% rename from cd3_automation_toolkit/DeveloperServices/OKE/export_oke_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/developerservices/oke/export_oke_nonGreenField.py index 2988e240b..d6878cdb8 100644 --- a/cd3_automation_toolkit/DeveloperServices/OKE/export_oke_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/export_oke_nonGreenField.py @@ -15,7 +15,9 @@ from oci.core.virtual_network_client import VirtualNetworkClient from oci.container_engine import ContainerEngineClient from oci.config import DEFAULT_LOCATION -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools sys.path.append(os.getcwd() + "/..") @@ -354,29 +356,29 @@ def print_oke(values_for_column_oke, reg, compartment_name, compartment_name_nod values_for_column_oke[col_header].append(val) # Process the Node Label Columns - elif 'oke labels' in col_header.lower() and col_header.lower() in commonTools.tagColumns: + elif 'oke labels' in col_header.lower() and col_header.lower() in ociCommonTools.tagColumns: if (nodepool_info != None): - values_for_column_oke = commonTools.export_tags(nodepool_info, col_header,values_for_column_oke) + values_for_column_oke = ociCommonTools.export_tags(nodepool_info, col_header,values_for_column_oke) else: values_for_column_oke[col_header].append(None) - elif 'nodepool defined tags' in col_header.lower() and col_header.lower() in commonTools.tagColumns: + elif 'nodepool defined tags' in col_header.lower() and col_header.lower() in ociCommonTools.tagColumns: if (nodepool_info != None): - values_for_column_oke = commonTools.export_tags(nodepool_info, col_header,values_for_column_oke) + values_for_column_oke = ociCommonTools.export_tags(nodepool_info, col_header,values_for_column_oke) else: values_for_column_oke[col_header].append(None) - elif 'nodepool freeform tags' in col_header.lower() and col_header.lower() in commonTools.tagColumns: + elif 'nodepool freeform tags' in col_header.lower() and col_header.lower() in ociCommonTools.tagColumns: if (nodepool_info != None): - values_for_column_oke = commonTools.export_tags(nodepool_info, col_header,values_for_column_oke) + values_for_column_oke = ociCommonTools.export_tags(nodepool_info, col_header,values_for_column_oke) else: values_for_column_oke[col_header].append(None) - elif 'node defined tags' in col_header.lower() and col_header.lower() in commonTools.tagColumns: + elif 'node defined tags' in col_header.lower() and col_header.lower() in ociCommonTools.tagColumns: if (nodepool_info != None): if nodepool_type=='managed': - values_for_column_oke = commonTools.export_tags(nodepool_info.node_config_details, col_header,values_for_column_oke) + values_for_column_oke = ociCommonTools.export_tags(nodepool_info.node_config_details, col_header,values_for_column_oke) elif (nodepool_type=='virtual' and nodepool_info.virtual_node_tags!=None): - values_for_column_oke = commonTools.export_tags(nodepool_info.virtual_node_tags , col_header, + values_for_column_oke = ociCommonTools.export_tags(nodepool_info.virtual_node_tags , col_header, values_for_column_oke) else: values_for_column_oke[col_header].append(None) @@ -384,12 +386,12 @@ def print_oke(values_for_column_oke, reg, compartment_name, compartment_name_nod values_for_column_oke[col_header].append(None) - elif 'node freeform tags' in col_header.lower() and col_header.lower() in commonTools.tagColumns: + elif 'node freeform tags' in col_header.lower() and col_header.lower() in ociCommonTools.tagColumns: if (nodepool_info != None): if nodepool_type == 'managed': - values_for_column_oke = commonTools.export_tags(nodepool_info.node_config_details, col_header,values_for_column_oke) + values_for_column_oke = ociCommonTools.export_tags(nodepool_info.node_config_details, col_header,values_for_column_oke) elif (nodepool_type == 'virtual' and nodepool_info.virtual_node_tags!=None): - values_for_column_oke = commonTools.export_tags(nodepool_info.virtual_node_tags, col_header, + values_for_column_oke = ociCommonTools.export_tags(nodepool_info.virtual_node_tags, col_header, values_for_column_oke) else: values_for_column_oke[col_header].append(None) @@ -397,41 +399,41 @@ def print_oke(values_for_column_oke, reg, compartment_name, compartment_name_nod else: values_for_column_oke[col_header].append(None) - elif 'lb defined tags' in col_header.lower() and col_header.lower() in commonTools.tagColumns: + elif 'lb defined tags' in col_header.lower() and col_header.lower() in ociCommonTools.tagColumns: if (nodepool_count <= 1): - values_for_column_oke = commonTools.export_tags(cluster_info.options.service_lb_config, col_header, + values_for_column_oke = ociCommonTools.export_tags(cluster_info.options.service_lb_config, col_header, values_for_column_oke) else: values_for_column_oke[col_header].append(None) - elif 'lb freeform tags' in col_header.lower() and col_header.lower() in commonTools.tagColumns: + elif 'lb freeform tags' in col_header.lower() and col_header.lower() in ociCommonTools.tagColumns: if (nodepool_count <= 1): - values_for_column_oke = commonTools.export_tags(cluster_info.options.service_lb_config, col_header, + values_for_column_oke = ociCommonTools.export_tags(cluster_info.options.service_lb_config, col_header, values_for_column_oke) else: values_for_column_oke[col_header].append(None) - elif 'volume defined tags' in col_header.lower() and col_header.lower() in commonTools.tagColumns: + elif 'volume defined tags' in col_header.lower() and col_header.lower() in ociCommonTools.tagColumns: if (nodepool_count <= 1): - values_for_column_oke = commonTools.export_tags(cluster_info.options.persistent_volume_config, + values_for_column_oke = ociCommonTools.export_tags(cluster_info.options.persistent_volume_config, col_header, values_for_column_oke) else: values_for_column_oke[col_header].append(None) - elif 'volume freeform tags' in col_header.lower() and col_header.lower() in commonTools.tagColumns: + elif 'volume freeform tags' in col_header.lower() and col_header.lower() in ociCommonTools.tagColumns: if (nodepool_count <= 1): - values_for_column_oke = commonTools.export_tags(cluster_info.options.persistent_volume_config, + values_for_column_oke = ociCommonTools.export_tags(cluster_info.options.persistent_volume_config, col_header, values_for_column_oke) else: values_for_column_oke[col_header].append(None) # Process tag columns - elif col_header.lower() in commonTools.tagColumns: - values_for_column_oke = commonTools.export_tags(cluster_info, col_header, values_for_column_oke) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_oke = ociCommonTools.export_tags(cluster_info, col_header, values_for_column_oke) else: oci_objs = [cluster_info,image_policy_config, nodepool_info] - values_for_column_oke = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_oke,values_for_column_oke) + values_for_column_oke = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_oke,values_for_column_oke) # Execution of the code begins here def export_oke(inputfile, outdir,service_dir, config, signer, ct, export_compartments=[], export_regions=[],export_tags=[]): @@ -541,6 +543,7 @@ def export_oke(inputfile, outdir,service_dir, config, signer, ct, export_compart for nodepool_info in nodepoolList: if nodepool_info.lifecycle_state!="ACTIVE": + nodepool_info = None continue # Tags filter diff --git a/cd3_automation_toolkit/DeveloperServices/OKE/templates/cluster-template b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/cluster-template similarity index 100% rename from cd3_automation_toolkit/DeveloperServices/OKE/templates/cluster-template rename to cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/cluster-template diff --git a/cd3_automation_toolkit/DeveloperServices/OKE/templates/nodepool-template b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/nodepool-template similarity index 100% rename from cd3_automation_toolkit/DeveloperServices/OKE/templates/nodepool-template rename to cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/nodepool-template diff --git a/cd3_automation_toolkit/DeveloperServices/OKE/templates/virtual-nodepool-template b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/virtual-nodepool-template similarity index 100% rename from cd3_automation_toolkit/DeveloperServices/OKE/templates/virtual-nodepool-template rename to cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/virtual-nodepool-template diff --git a/cd3_automation_toolkit/DeveloperServices/ResourceManager/__init__.py b/cd3_automation_toolkit/ocicloud/python/developerservices/resourcemanager/__init__.py similarity index 100% rename from cd3_automation_toolkit/DeveloperServices/ResourceManager/__init__.py rename to cd3_automation_toolkit/ocicloud/python/developerservices/resourcemanager/__init__.py diff --git a/cd3_automation_toolkit/DeveloperServices/ResourceManager/create_resource_manager_stack.py b/cd3_automation_toolkit/ocicloud/python/developerservices/resourcemanager/create_resource_manager_stack.py similarity index 99% rename from cd3_automation_toolkit/DeveloperServices/ResourceManager/create_resource_manager_stack.py rename to cd3_automation_toolkit/ocicloud/python/developerservices/resourcemanager/create_resource_manager_stack.py index ee4397138..4e6b28f23 100644 --- a/cd3_automation_toolkit/DeveloperServices/ResourceManager/create_resource_manager_stack.py +++ b/cd3_automation_toolkit/ocicloud/python/developerservices/resourcemanager/create_resource_manager_stack.py @@ -7,14 +7,15 @@ # Author: Shruthi Subramanian # -import argparse -import os +import os, sys from zipfile import ZipFile import shutil import time import csv import base64 -from commonTools import * +import oci +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * from oci.config import DEFAULT_LOCATION from oci.resource_manager.models import CreateStackDetails from oci.resource_manager.models import UpdateStackDetails diff --git a/cd3_automation_toolkit/ocicloud/python/governance/__init__.py b/cd3_automation_toolkit/ocicloud/python/governance/__init__.py new file mode 100644 index 000000000..3204e8908 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/python/governance/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + + +from .tagging import * +from .quota import * + diff --git a/cd3_automation_toolkit/Governance/Quota/__init__.py b/cd3_automation_toolkit/ocicloud/python/governance/quota/__init__.py similarity index 100% rename from cd3_automation_toolkit/Governance/Quota/__init__.py rename to cd3_automation_toolkit/ocicloud/python/governance/quota/__init__.py diff --git a/cd3_automation_toolkit/Governance/Quota/create_terraform_quotas.py b/cd3_automation_toolkit/ocicloud/python/governance/quota/create_terraform_quotas.py similarity index 94% rename from cd3_automation_toolkit/Governance/Quota/create_terraform_quotas.py rename to cd3_automation_toolkit/ocicloud/python/governance/quota/create_terraform_quotas.py index bbe55bae7..8d1a9fcb2 100644 --- a/cd3_automation_toolkit/Governance/Quota/create_terraform_quotas.py +++ b/cd3_automation_toolkit/ocicloud/python/governance/quota/create_terraform_quotas.py @@ -6,11 +6,13 @@ # Author: Bhanu P. Lohumi # Oracle Consulting # -import os +import os, sys from jinja2 import Environment, FileSystemLoader from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### @@ -90,8 +92,8 @@ def create_terraform_quotas(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'Name': columnvalue = columnvalue.strip() diff --git a/cd3_automation_toolkit/Governance/Quota/export_quotas_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/governance/quota/export_quotas_nonGreenField.py similarity index 89% rename from cd3_automation_toolkit/Governance/Quota/export_quotas_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/governance/quota/export_quotas_nonGreenField.py index 3c8b14ab1..f80c98483 100644 --- a/cd3_automation_toolkit/Governance/Quota/export_quotas_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/governance/quota/export_quotas_nonGreenField.py @@ -10,9 +10,9 @@ import oci import os import subprocess as sp -from commonTools import * - -sys.path.append(os.getcwd()+"/..") +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools compartment_ids={} importCommands={} @@ -28,8 +28,13 @@ def print_quotas(values_for_columns,region, quota,quota_policy): values_for_columns[col_header].append(quota.description) elif (col_header == "Quota Policy"): values_for_columns[col_header].append(quota_policy) - elif col_header.lower() in commonTools.tagColumns: - values_for_columns = commonTools.export_tags(quota, col_header, values_for_columns) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_columns = ociCommonTools.export_tags(quota, col_header, values_for_columns) + else: + oci_objs = [quota] + values_for_columns = ociCommonTools.export_extra_columns(oci_objs, col_header, + sheet_dict_quotas,values_for_columns) + # Execution of the code begins here def export_quotas_nongreenfield(inputfile, outdir, service_dir, config, signer, ct,export_tags): diff --git a/cd3_automation_toolkit/Governance/Quota/templates/quota-template b/cd3_automation_toolkit/ocicloud/python/governance/quota/templates/quota-template similarity index 100% rename from cd3_automation_toolkit/Governance/Quota/templates/quota-template rename to cd3_automation_toolkit/ocicloud/python/governance/quota/templates/quota-template diff --git a/cd3_automation_toolkit/Governance/Tagging/__init__.py b/cd3_automation_toolkit/ocicloud/python/governance/tagging/__init__.py similarity index 100% rename from cd3_automation_toolkit/Governance/Tagging/__init__.py rename to cd3_automation_toolkit/ocicloud/python/governance/tagging/__init__.py diff --git a/cd3_automation_toolkit/Governance/Tagging/create_terraform_tags.py b/cd3_automation_toolkit/ocicloud/python/governance/tagging/create_terraform_tags.py similarity index 97% rename from cd3_automation_toolkit/Governance/Tagging/create_terraform_tags.py rename to cd3_automation_toolkit/ocicloud/python/governance/tagging/create_terraform_tags.py index f2a97d348..3365a4399 100644 --- a/cd3_automation_toolkit/Governance/Tagging/create_terraform_tags.py +++ b/cd3_automation_toolkit/ocicloud/python/governance/tagging/create_terraform_tags.py @@ -13,8 +13,9 @@ import pandas as pd import os from pathlib import Path -sys.path.append(os.getcwd()+"/../..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from jinja2 import Environment, FileSystemLoader ###### @@ -133,8 +134,8 @@ def create_terraform_tags(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Compartment Name": columnvalue = str(columnvalue).strip() diff --git a/cd3_automation_toolkit/Governance/Tagging/export_tags_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/governance/tagging/export_tags_nonGreenField.py similarity index 96% rename from cd3_automation_toolkit/Governance/Tagging/export_tags_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/governance/tagging/export_tags_nonGreenField.py index 003e27372..126fdaa2a 100644 --- a/cd3_automation_toolkit/Governance/Tagging/export_tags_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/governance/tagging/export_tags_nonGreenField.py @@ -12,9 +12,9 @@ from oci.identity import IdentityClient import os import subprocess as sp -from commonTools import * - -sys.path.append(os.getcwd()+"/..") +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools compartment_ids={} tf_name_namespace_list = [] @@ -76,11 +76,11 @@ def print_tags(values_for_column_tags,region, ntk_compartment_name, tag, tag_ke else: values_for_column_tags[col_header].append('') - elif col_header.lower() in commonTools.tagColumns: - values_for_column_tags = commonTools.export_tags(tag, col_header, values_for_column_tags) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_tags = ociCommonTools.export_tags(tag, col_header, values_for_column_tags) else: oci_objs = [tag,tag_key]#,tag_default] - values_for_column_tags = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_tags,values_for_column_tags) + values_for_column_tags = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_tags,values_for_column_tags) tf_name_namespace = commonTools.check_tf_variable(tagname) tf_name_key = commonTools.check_tf_variable(tag_key_name) diff --git a/cd3_automation_toolkit/Governance/Tagging/templates/tags-defaults-template b/cd3_automation_toolkit/ocicloud/python/governance/tagging/templates/tags-defaults-template similarity index 100% rename from cd3_automation_toolkit/Governance/Tagging/templates/tags-defaults-template rename to cd3_automation_toolkit/ocicloud/python/governance/tagging/templates/tags-defaults-template diff --git a/cd3_automation_toolkit/Governance/Tagging/templates/tags-keys-template b/cd3_automation_toolkit/ocicloud/python/governance/tagging/templates/tags-keys-template similarity index 100% rename from cd3_automation_toolkit/Governance/Tagging/templates/tags-keys-template rename to cd3_automation_toolkit/ocicloud/python/governance/tagging/templates/tags-keys-template diff --git a/cd3_automation_toolkit/Governance/Tagging/templates/tags-namespaces-template b/cd3_automation_toolkit/ocicloud/python/governance/tagging/templates/tags-namespaces-template similarity index 100% rename from cd3_automation_toolkit/Governance/Tagging/templates/tags-namespaces-template rename to cd3_automation_toolkit/ocicloud/python/governance/tagging/templates/tags-namespaces-template diff --git a/cd3_automation_toolkit/ocicloud/python/identity/__init__.py b/cd3_automation_toolkit/ocicloud/python/identity/__init__.py new file mode 100644 index 000000000..ff94f3eb2 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/python/identity/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 + +from .compartments import * +from .users import * +from .groups import * +from .policies import * +from .export_identity_nonGreenField import * +from .networksources import * diff --git a/cd3_automation_toolkit/Identity/Compartments/__init__.py b/cd3_automation_toolkit/ocicloud/python/identity/compartments/__init__.py similarity index 100% rename from cd3_automation_toolkit/Identity/Compartments/__init__.py rename to cd3_automation_toolkit/ocicloud/python/identity/compartments/__init__.py diff --git a/cd3_automation_toolkit/Identity/Compartments/create_terraform_compartments.py b/cd3_automation_toolkit/ocicloud/python/identity/compartments/create_terraform_compartments.py similarity index 97% rename from cd3_automation_toolkit/Identity/Compartments/create_terraform_compartments.py rename to cd3_automation_toolkit/ocicloud/python/identity/compartments/create_terraform_compartments.py index 1a89d0cab..c65465a62 100644 --- a/cd3_automation_toolkit/Identity/Compartments/create_terraform_compartments.py +++ b/cd3_automation_toolkit/ocicloud/python/identity/compartments/create_terraform_compartments.py @@ -9,11 +9,13 @@ # Modified (TF if (columnvalue not in ckeys):Upgrade): Shruthi Subramanian # -import os +import os, sys from oci.config import DEFAULT_LOCATION from jinja2 import Environment, FileSystemLoader from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools import oci ###### @@ -127,8 +129,8 @@ def travel(parent, keys, values, c): tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Parent Compartment": columnname = commonTools.check_column_headers(columnname) diff --git a/cd3_automation_toolkit/Identity/Compartments/templates/compartments-template b/cd3_automation_toolkit/ocicloud/python/identity/compartments/templates/compartments-template similarity index 100% rename from cd3_automation_toolkit/Identity/Compartments/templates/compartments-template rename to cd3_automation_toolkit/ocicloud/python/identity/compartments/templates/compartments-template diff --git a/cd3_automation_toolkit/Identity/export_identity_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/identity/export_identity_nonGreenField.py similarity index 94% rename from cd3_automation_toolkit/Identity/export_identity_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/identity/export_identity_nonGreenField.py index be2266c59..ea6c512ab 100644 --- a/cd3_automation_toolkit/Identity/export_identity_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/identity/export_identity_nonGreenField.py @@ -13,9 +13,11 @@ import oci from oci.identity import IdentityClient import os +import time import subprocess as sp -sys.path.append(os.getcwd()+"/..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools # Execution of the code begins here def export_identity(inputfile, outdir, service_dir,resource, config, signer, ct, export_compartments=[],export_domains={}): @@ -132,13 +134,13 @@ def export_identity(inputfile, outdir, service_dir,resource, config, signer, ct, values_for_column_comps[col_header].append(comp_display_name) elif (col_header == "Parent Compartment"): values_for_column_comps[col_header].append(comp_parent_name) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_comps = commonTools.export_tags(c_details, col_header, + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_comps = ociCommonTools.export_tags(c_details, col_header, values_for_column_comps ) else: oci_objs = [c_details] - values_for_column_comps = commonTools.export_extra_columns(oci_objs, col_header, + values_for_column_comps = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_comps, values_for_column_comps ) @@ -233,13 +235,13 @@ def export_identity(inputfile, outdir, service_dir,resource, config, signer, ct, values_for_column_policies[col_header].append("") elif (col_header == "Policy Statements"): values_for_column_policies[col_header].append(stmt) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_policies = commonTools.export_tags(policy, col_header, + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_policies = ociCommonTools.export_tags(policy, col_header, values_for_column_policies ) else: oci_objs = [policy] - values_for_column_policies = commonTools.export_extra_columns(oci_objs, col_header, + values_for_column_policies = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_policies, values_for_column_policies ) @@ -347,10 +349,10 @@ def process_group(grp_info, members_list,membership_id_list, domain_name, is_dyn grp_defined_tags = ";".join(grp_defined_tags) values_for_column_groups[col_header].append(grp_defined_tags if grp_defined_tags else "") elif col_header == "Defined Tags" and non_domain: - values_for_column_groups = commonTools.export_tags(grp_info, col_header, values_for_column_groups) + values_for_column_groups = ociCommonTools.export_tags(grp_info, col_header, values_for_column_groups) else: oci_objs = [grp_info] - values_for_column_groups = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_groups, values_for_column_groups) + values_for_column_groups = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_groups, values_for_column_groups) return importCommands, values_for_column_groups @@ -365,8 +367,12 @@ def process_group(grp_info, members_list,membership_id_list, domain_name, is_dyn response = domain_client.list_groups( attributes=['members'], attribute_sets=['all'], - page=next_page + page=next_page, + sort_by="displayName", + sort_order="ASCENDING" ) + # added sleep time and sorting to handle inconsistency in export data + time.sleep(5) groups.extend(response.data.resources) if not response.next_page or len(groups) == response.data.total_results: break @@ -385,8 +391,12 @@ def process_group(grp_info, members_list,membership_id_list, domain_name, is_dyn response = domain_client.list_dynamic_resource_groups( attributes=['matching_rule'], attribute_sets=['all'], - page=next_page + page=next_page, + sort_by="displayName", + sort_order="ASCENDING" ) + # added sleep time and sorting to handle inconsistency in export data + time.sleep(5) dyngroups.extend(response.data.resources) if not response.next_page or len(dyngroups) == response.data.total_results: break diff --git a/cd3_automation_toolkit/Identity/Groups/__init__.py b/cd3_automation_toolkit/ocicloud/python/identity/groups/__init__.py similarity index 100% rename from cd3_automation_toolkit/Identity/Groups/__init__.py rename to cd3_automation_toolkit/ocicloud/python/identity/groups/__init__.py diff --git a/cd3_automation_toolkit/Identity/Groups/create_terraform_groups.py b/cd3_automation_toolkit/ocicloud/python/identity/groups/create_terraform_groups.py similarity index 92% rename from cd3_automation_toolkit/Identity/Groups/create_terraform_groups.py rename to cd3_automation_toolkit/ocicloud/python/identity/groups/create_terraform_groups.py index cc8e0f3a0..a225507b0 100644 --- a/cd3_automation_toolkit/Identity/Groups/create_terraform_groups.py +++ b/cd3_automation_toolkit/ocicloud/python/identity/groups/create_terraform_groups.py @@ -8,11 +8,13 @@ # Oracle Consulting # Modified by: Ranjini Rajendran # -import os +import os, sys from pathlib import Path from oci.config import DEFAULT_LOCATION from jinja2 import Environment, FileSystemLoader -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs- CD3 excel file, Config file, prefix AND outdir @@ -110,7 +112,8 @@ def create_terraform_groups(inputfile, outdir, service_dir, prefix, ct): # Column value if 'description' in columnname.lower(): columnvalue = str(df[columnname][i]) - tempdict = {'description': columnvalue} + if columnvalue.lower() != 'nan' and columnvalue != '': + tempdict = {'description': columnvalue} else: columnvalue = str(df[columnname][i]).strip() @@ -118,8 +121,8 @@ def create_terraform_groups(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'Name': group_tf_name = f"{domain}_{commonTools.check_tf_variable(columnvalue)}" @@ -135,11 +138,11 @@ def create_terraform_groups(inputfile, outdir, service_dir, prefix, ct): tempdict = {'members': members} # Process Defined and Freeform Tags based on columnname and 'Domain Name' - if columnname.lower() in commonTools.tagColumns: + if columnname.lower() in ociCommonTools.tagColumns: # for IDCS tenancies if not ct.identity_domain_enabled: # Process tags using the existing code - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) else: if columnname == 'Defined Tags': defined_tags = columnvalue.strip() diff --git a/cd3_automation_toolkit/Identity/Groups/templates/groups-template b/cd3_automation_toolkit/ocicloud/python/identity/groups/templates/groups-template similarity index 100% rename from cd3_automation_toolkit/Identity/Groups/templates/groups-template rename to cd3_automation_toolkit/ocicloud/python/identity/groups/templates/groups-template diff --git a/cd3_automation_toolkit/Identity/Groups/templates/identity-domain-groups-template b/cd3_automation_toolkit/ocicloud/python/identity/groups/templates/identity-domain-groups-template similarity index 100% rename from cd3_automation_toolkit/Identity/Groups/templates/identity-domain-groups-template rename to cd3_automation_toolkit/ocicloud/python/identity/groups/templates/identity-domain-groups-template diff --git a/cd3_automation_toolkit/Identity/NetworkSources/__init__.py b/cd3_automation_toolkit/ocicloud/python/identity/networksources/__init__.py similarity index 100% rename from cd3_automation_toolkit/Identity/NetworkSources/__init__.py rename to cd3_automation_toolkit/ocicloud/python/identity/networksources/__init__.py diff --git a/cd3_automation_toolkit/Identity/NetworkSources/create_terraform_networkSources.py b/cd3_automation_toolkit/ocicloud/python/identity/networksources/create_terraform_networkSources.py similarity index 95% rename from cd3_automation_toolkit/Identity/NetworkSources/create_terraform_networkSources.py rename to cd3_automation_toolkit/ocicloud/python/identity/networksources/create_terraform_networkSources.py index cc6113a60..bab23048c 100644 --- a/cd3_automation_toolkit/Identity/NetworkSources/create_terraform_networkSources.py +++ b/cd3_automation_toolkit/ocicloud/python/identity/networksources/create_terraform_networkSources.py @@ -7,11 +7,13 @@ # Author: Gaurav Goyal # Oracle Consulting # -import os +import os, sys from pathlib import Path from oci.config import DEFAULT_LOCATION from jinja2 import Environment, FileSystemLoader -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs- CD3 excel file, Config file, prefix AND outdir @@ -134,8 +136,8 @@ def create_terraform_networkSources(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) # Check for boolean/null in column values diff --git a/cd3_automation_toolkit/Identity/NetworkSources/export_networkSources_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/identity/networksources/export_networkSources_nonGreenField.py similarity index 94% rename from cd3_automation_toolkit/Identity/NetworkSources/export_networkSources_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/identity/networksources/export_networkSources_nonGreenField.py index 352be8894..744e37cab 100644 --- a/cd3_automation_toolkit/Identity/NetworkSources/export_networkSources_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/identity/networksources/export_networkSources_nonGreenField.py @@ -14,7 +14,9 @@ import os import subprocess as sp sys.path.append(os.getcwd()+"/..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools # Execution of the code begins here def export_networkSources(inputfile, outdir, service_dir, config, signer, ct): @@ -110,7 +112,8 @@ def export_networkSources(inputfile, outdir, service_dir, config, signer, ct): values_for_column_networkSources[col_header].append(strIPRange) elif (col_header == "Defined Tags"): - values_for_column_networkSources = commonTools.export_tags(network_source_info, col_header, values_for_column_networkSources) + values_for_column_networkSources = ociCommonTools.export_tags(network_source_info, col_header, values_for_column_networkSources) + commonTools.write_to_cd3(values_for_column_networkSources, cd3file, sheetName) print("{0} Network Sources exported into CD3.\n".format(total_resource)) diff --git a/cd3_automation_toolkit/Identity/NetworkSources/templates/network-sources-template b/cd3_automation_toolkit/ocicloud/python/identity/networksources/templates/network-sources-template similarity index 100% rename from cd3_automation_toolkit/Identity/NetworkSources/templates/network-sources-template rename to cd3_automation_toolkit/ocicloud/python/identity/networksources/templates/network-sources-template diff --git a/cd3_automation_toolkit/Identity/Policies/__init__.py b/cd3_automation_toolkit/ocicloud/python/identity/policies/__init__.py similarity index 100% rename from cd3_automation_toolkit/Identity/Policies/__init__.py rename to cd3_automation_toolkit/ocicloud/python/identity/policies/__init__.py diff --git a/cd3_automation_toolkit/Identity/Policies/create_terraform_policies.py b/cd3_automation_toolkit/ocicloud/python/identity/policies/create_terraform_policies.py similarity index 96% rename from cd3_automation_toolkit/Identity/Policies/create_terraform_policies.py rename to cd3_automation_toolkit/ocicloud/python/identity/policies/create_terraform_policies.py index c3326539b..3c95fdce3 100644 --- a/cd3_automation_toolkit/Identity/Policies/create_terraform_policies.py +++ b/cd3_automation_toolkit/ocicloud/python/identity/policies/create_terraform_policies.py @@ -9,11 +9,13 @@ # Modified (TF Upgrade): Shruthi Subramanian # -import os +import os, sys from pathlib import Path from oci.config import DEFAULT_LOCATION from jinja2 import Environment, FileSystemLoader -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### @@ -38,7 +40,7 @@ def create_terraform_policies(inputfile, outdir, service_dir, prefix, ct): policies_template = env.get_template('policies-template') # Read CD3 - df = data_frame(filename, sheetName) + df = commonTools.data_frame(filename, sheetName) regions = df['Region'] regions.dropna() @@ -85,8 +87,8 @@ def create_terraform_policies(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Compartment Name": columnname = commonTools.check_column_headers(columnname) diff --git a/cd3_automation_toolkit/Identity/Policies/templates/policies-template b/cd3_automation_toolkit/ocicloud/python/identity/policies/templates/policies-template similarity index 100% rename from cd3_automation_toolkit/Identity/Policies/templates/policies-template rename to cd3_automation_toolkit/ocicloud/python/identity/policies/templates/policies-template diff --git a/cd3_automation_toolkit/Identity/Users/__init__.py b/cd3_automation_toolkit/ocicloud/python/identity/users/__init__.py similarity index 100% rename from cd3_automation_toolkit/Identity/Users/__init__.py rename to cd3_automation_toolkit/ocicloud/python/identity/users/__init__.py diff --git a/cd3_automation_toolkit/Identity/Users/create_terraform_users.py b/cd3_automation_toolkit/ocicloud/python/identity/users/create_terraform_users.py similarity index 94% rename from cd3_automation_toolkit/Identity/Users/create_terraform_users.py rename to cd3_automation_toolkit/ocicloud/python/identity/users/create_terraform_users.py index 5ae39550f..0af79096d 100644 --- a/cd3_automation_toolkit/Identity/Users/create_terraform_users.py +++ b/cd3_automation_toolkit/ocicloud/python/identity/users/create_terraform_users.py @@ -8,11 +8,13 @@ # Oracle Consulting # Modified by: Ranjini Rajendran -import os +import os, sys from pathlib import Path from oci.config import DEFAULT_LOCATION from jinja2 import Environment, FileSystemLoader -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs- CD3 excel file, Config file, prefix AND outdir @@ -117,8 +119,8 @@ def create_terraform_users(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'User Name': columnvalue = columnvalue.strip() @@ -140,10 +142,10 @@ def create_terraform_users(inputfile, outdir, service_dir, prefix, ct): tempStr.update(tempdict) # Process Defined and Freeform Tags based on columnname and 'Domain Name' - if columnname.lower() in commonTools.tagColumns: + if columnname.lower() in ociCommonTools.tagColumns: if not ct.identity_domain_enabled: # Process tags using the existing code - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) else: # When 'Domain Name' is not 'nan', process 'Defined Tags' differently if columnname == 'Defined Tags': diff --git a/cd3_automation_toolkit/Identity/Users/export_users_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/identity/users/export_users_nonGreenField.py similarity index 90% rename from cd3_automation_toolkit/Identity/Users/export_users_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/identity/users/export_users_nonGreenField.py index df9319079..e07a9cfdc 100644 --- a/cd3_automation_toolkit/Identity/Users/export_users_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/identity/users/export_users_nonGreenField.py @@ -12,9 +12,11 @@ import oci from oci.identity import IdentityClient import os +import time import subprocess as sp -sys.path.append(os.getcwd() + "/..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools def append_user_info(values_for_column_users,sheet_dict_users, ct, user_info, username, family_name, given_name, recovery_email, display_name, description, email, domain_key, user_defined_tags): def add_capabilities(user_info): capabilities = [] @@ -63,7 +65,7 @@ def map_columns(col_header): values_for_column_users[col_header].append(value) else: oci_objs = [user_info] - values_for_column_users = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_users, values_for_column_users) + values_for_column_users = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_users, values_for_column_users) # Execution start here def export_users(inputfile, outdir, service_dir, config, signer, ct,export_domains={}): global sheet_dict_users @@ -115,7 +117,9 @@ def export_users(inputfile, outdir, service_dir, config, signer, ct,export_domai users = [] next_page = None while True: - response = domain_client.list_users(page=next_page) + response = domain_client.list_users(page=next_page,sort_by="displayName",sort_order="ASCENDING") + # added sleep time and sorting to handle inconsistency in export data + time.sleep(5) users.extend(response.data.resources) if not response.next_page or len(users) == response.data.total_results: break @@ -138,7 +142,8 @@ def export_users(inputfile, outdir, service_dir, config, signer, ct,export_domai user_defined_tags = ";".join(user_defined_tags) if user_defined_tags else "" user_info = user - if user_info.urn_ietf_params_scim_schemas_oracle_idcs_extension_user_user.is_federated_user != "True" and user_info.active !="False": + #if user_info.urn_ietf_params_scim_schemas_oracle_idcs_extension_user_user.is_federated_user != "True" and user_info.active !="False": + if (user_info.active != "False" and (user_info.urn_ietf_params_scim_schemas_oracle_idcs_extension_user_user is None or user_info.urn_ietf_params_scim_schemas_oracle_idcs_extension_user_user.is_federated_user != "True")): username = user_info.user_name family_name = user_info.name.family_name given_name = user_info.name.given_name @@ -194,8 +199,8 @@ def export_users(inputfile, outdir, service_dir, config, signer, ct,export_domai importCommands += f'\n{tf_or_tofu} import "{tf_resource}" capabilities/{user_info.id}' for col_header in values_for_column_users.keys(): - if col_header.lower() in commonTools.tagColumns: - values_for_column_users = commonTools.export_tags(user, col_header, values_for_column_users) + if col_header.lower() in ociCommonTools.tagColumns: + values_for_column_users = ociCommonTools.export_tags(user, col_header, values_for_column_users) max_list_length = max(len(lst) for lst in values_for_column_users.values()) for col_name in values_for_column_users: diff --git a/cd3_automation_toolkit/Identity/Users/templates/identity-domain-users-template b/cd3_automation_toolkit/ocicloud/python/identity/users/templates/identity-domain-users-template similarity index 100% rename from cd3_automation_toolkit/Identity/Users/templates/identity-domain-users-template rename to cd3_automation_toolkit/ocicloud/python/identity/users/templates/identity-domain-users-template diff --git a/cd3_automation_toolkit/Identity/Users/templates/users-template b/cd3_automation_toolkit/ocicloud/python/identity/users/templates/users-template similarity index 100% rename from cd3_automation_toolkit/Identity/Users/templates/users-template rename to cd3_automation_toolkit/ocicloud/python/identity/users/templates/users-template diff --git a/cd3_automation_toolkit/ocicloud/python/managementservices/__init__.py b/cd3_automation_toolkit/ocicloud/python/managementservices/__init__.py new file mode 100755 index 000000000..25e351456 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/python/managementservices/__init__.py @@ -0,0 +1,4 @@ +from .eventsnotifications import * +from .monitoring import * +from .logging import * +from .serviceconnectorhub import * \ No newline at end of file diff --git a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/__init__.py b/cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/__init__.py similarity index 100% rename from cd3_automation_toolkit/ManagementServices/EventsAndNotifications/__init__.py rename to cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/__init__.py diff --git a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/create_terraform_events.py b/cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/create_terraform_events.py similarity index 97% rename from cd3_automation_toolkit/ManagementServices/EventsAndNotifications/create_terraform_events.py rename to cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/create_terraform_events.py index b790488c7..e41b25662 100644 --- a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/create_terraform_events.py +++ b/cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/create_terraform_events.py @@ -9,11 +9,14 @@ # Modified (TF Upgrade): Shravanthi Lingam # +import os, sys import json -from commonTools import * from oci.config import DEFAULT_LOCATION from pathlib import Path from jinja2 import Environment, FileSystemLoader +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools #Method to extend conditions with resources def extend_event(service_name, resources, listeventid): @@ -109,8 +112,8 @@ def create_terraform_events(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Compartment Name": compartmentVarName = columnvalue.strip() diff --git a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/create_terraform_notifications.py b/cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/create_terraform_notifications.py similarity index 70% rename from cd3_automation_toolkit/ManagementServices/EventsAndNotifications/create_terraform_notifications.py rename to cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/create_terraform_notifications.py index a9ca8c708..463b65eb0 100644 --- a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/create_terraform_notifications.py +++ b/cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/create_terraform_notifications.py @@ -9,11 +9,15 @@ #shravanthi.lingam@oracle.com # Modified (TF Upgrade): Shravanthi Lingam # +import os, sys import datetime from jinja2 import Environment, FileSystemLoader from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * +from copy import deepcopy +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs- CD3 excel file, Config file, prefix AND outdir @@ -25,23 +29,21 @@ def create_terraform_notifications(inputfile, outdir, service_dir, prefix, ct): outdir = outdir sheetName="Notifications" topics_auto_tfvars_filename = '_' + sheetName.lower() + '-topics.auto.tfvars' - subs_auto_tfvars_filename = '_' + sheetName.lower() + '-subscriptions.auto.tfvars' x = datetime.datetime.now() date = x.strftime("%f").strip() tempStr={} tfStr={} + topic_dict = {} tfStr1 = {} outfile={} oname={} Notifications_names={} - Subscriptions_names={} # Load the template file file_loader = FileSystemLoader(f'{Path(__file__).parent}/templates') env = Environment(loader=file_loader, keep_trailing_newline=True, trim_blocks=True, lstrip_blocks=True) - notifications_template = env.get_template('notifications-topics-template') - subscriptions_template = env.get_template('notifications-subscriptions-template') + notifications_template = env.get_template('notifications-template') # Read cd3 using pandas dataframe df, col_headers = commonTools.read_cd3(filename, sheetName) @@ -58,13 +60,11 @@ def create_terraform_notifications(inputfile, outdir, service_dir, prefix, ct): tfStr[eachregion] = '' tfStr1[eachregion] = '' Notifications_names[eachregion]=[] - Subscriptions_names[eachregion]=[] # Take backup of files resource = sheetName.lower() srcdir = outdir + "/" + eachregion + "/" + service_dir + "/" commonTools.backup_file(srcdir, resource, topics_auto_tfvars_filename) - commonTools.backup_file(srcdir, resource, subs_auto_tfvars_filename) # Iterate over rows count = 1 @@ -103,8 +103,8 @@ def create_terraform_notifications(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Compartment Name": compartmentVarName = commonTools.check_tf_variable(columnvalue) @@ -134,30 +134,30 @@ def create_terraform_notifications(inputfile, outdir, service_dir, prefix, ct): tempStr[columnname] = str(columnvalue).strip() tempStr.update(tempdict) - if(topic in Notifications_names[region]): - count = count + 1 - if(topic not in Notifications_names[region]): - count = 1 - Notifications_names[region].append(topic) - # Write all info to TF string - tfStr[region]=tfStr[region][:-1] + notifications_template.render(tempStr) + subscription = [] + if (str(df.loc[i, 'Protocol']).lower() != 'nan' and str(df.loc[i, 'Endpoint']).lower() != 'nan'): + subscription = [{ + 'protocol': protocol, + 'endpoint': endpoint.split("::")[-1] + }] - #Empty Topic - if(str(df.loc[i, 'Protocol']).lower() == 'nan' and str(df.loc[i, 'Endpoint']).lower() == 'nan'): - continue - - subscription = tf_name_topic + "_sub" + str(count) - tempdict = {'subscription_tf_name': subscription} + tempdict = {'subscriptions': subscription} tempStr.update(tempdict) - if(subscription.strip() not in Subscriptions_names[region]): - Subscriptions_names[region].append(subscription.strip()) - if ( protocol in "ORACLE_FUNCTIONS" ): - endpoint = endpoint.split("::") - endpoint = endpoint[1] - tempdict = {'endpoint': endpoint} - tempStr.update(tempdict) - tfStr1[region]=tfStr1[region][:-1] + subscriptions_template.render(tempStr) + topic_key = f'{region}_{topic}' + + if topic_key in topic_dict: + topic_dict[topic_key]['subscriptions'].extend(subscription) + else: + topic_dict[topic_key] = deepcopy(tempStr) + + for key in topic_dict: + reg_key = key.split("_")[0] + + # Write all info to TF string + tfStr[reg_key]=tfStr[reg_key][:-1] + notifications_template.render(topic_dict[key]) + + #Empty Topic # Write to output for reg in ct.all_regions: reg_out_dir = outdir + "/" + reg + "/" + service_dir @@ -171,13 +171,3 @@ def create_terraform_notifications(inputfile, outdir, service_dir, prefix, ct): oname[reg].close() print(outfile[reg] + " for Notifications_Topics has been created for region " + reg) - if (tfStr1[reg] != ''): - outfile[reg] = reg_out_dir + "/" + prefix + subs_auto_tfvars_filename - srcStr = "##Add New Subscriptions for "+str(reg).lower()+" here##" - tfStr1[reg] = subscriptions_template.render(skeleton=True, region=reg).replace(srcStr,tfStr1[reg] + "\n" + srcStr) - tfStr1[reg] = "".join([s for s in tfStr1[reg].strip().splitlines(True) if s.strip("\r\n").strip()]) - oname[reg] = open(outfile[reg], 'w+') - oname[reg].write(tfStr1[reg]) - oname[reg].close() - print(outfile[reg] + " for Notifications_Subscriptions has been created for region " + reg) - diff --git a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/export_events_notifications_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/export_events_notifications_nonGreenField.py similarity index 93% rename from cd3_automation_toolkit/ManagementServices/EventsAndNotifications/export_events_notifications_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/export_events_notifications_nonGreenField.py index 34131f2f3..029252813 100644 --- a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/export_events_notifications_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/export_events_notifications_nonGreenField.py @@ -17,8 +17,9 @@ from oci.ons import NotificationDataPlaneClient from oci.functions import FunctionsManagementClient from oci.config import DEFAULT_LOCATION -from commonTools import * -sys.path.append(os.getcwd() + "/..") +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools compartment_ids={} importCommands={} @@ -56,19 +57,19 @@ def print_notifications(values_for_column_notifications,region, ntk_compartment if (sbpn == None): values_for_column_notifications[col_header].append("") else: - values_for_column_notifications = commonTools.export_tags(sbpn, col_header, values_for_column_notifications) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_notifications = commonTools.export_tags(nftn_info, col_header, values_for_column_notifications) + values_for_column_notifications = ociCommonTools.export_tags(sbpn, col_header, values_for_column_notifications) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_notifications = ociCommonTools.export_tags(nftn_info, col_header, values_for_column_notifications) else: oci_objs = [nftn_info,sbpn] - values_for_column_notifications = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_notifications,values_for_column_notifications) - tf_resource = f'module.notifications-topics[\\"{tf_name_nftn}\\"].oci_ons_notification_topic.topic' + values_for_column_notifications = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_notifications,values_for_column_notifications) + tf_resource = f'module.notifications[\\"{tf_name_nftn}\\"].oci_ons_notification_topic.topic' if (i ==0 or i == 1) and tf_resource not in state["resources"]: importCommands[region.lower()] += f'\n{tf_or_tofu} import "{tf_resource}" {str(nftn_info.topic_id)}' - tf_resource = f'module.notifications-subscriptions[\\"{tf_name_sbpn}\\"].oci_ons_subscription.subscription' + tf_resource = f'module.notifications[\\"{tf_name_nftn}\\"].oci_ons_subscription.subscription' if(i!=0) and tf_resource not in state["resources"]: - importCommands[region.lower()] += f'\n{tf_or_tofu} import "{tf_resource}" {str(sbpn.id)}' + importCommands[region.lower()] += f'\n{tf_or_tofu} import "{tf_resource}[{i-1}]" {str(sbpn.id)}' def print_events(values_for_column_events, region, ntk_compartment_name, event, event_info, ncpc, fun,state): @@ -92,9 +93,11 @@ def print_events(values_for_column_events, region, ntk_compartment_name, event, action_type = action.action_type action_is_enabled = str(action.is_enabled) if (action_type == "OSS"): - print("Ignoring Event "+event_name +" because action is OSS") - action_name = "" - continue + action_info = {} + action_id = action.stream_id + #print("Ignoring Event "+event_name +" because action is OSS") + action_name = action_id + action_comp = "" if ( action_type == "FAAS" ): action_id = action.function_id try: @@ -162,11 +165,11 @@ def events_rows(values_for_column_events, region, ntk_compartment_name, event_na values_for_column_events[col_header].append(event_is_enabled) elif (col_header == "Topic"): values_for_column_events[col_header].append(action_name) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_events = commonTools.export_tags(event, col_header, values_for_column_events) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_events = ociCommonTools.export_tags(event, col_header, values_for_column_events) else: oci_objs = [event,event_info] - values_for_column_events = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_events,values_for_column_events) + values_for_column_events = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_events,values_for_column_events) # Execution for Events export starts here def export_events(inputfile, outdir, service_dir, config, signer, ct,export_compartments=[], export_regions=[],export_tags=[]): diff --git a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/templates/actions-template b/cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/templates/actions-template similarity index 100% rename from cd3_automation_toolkit/ManagementServices/EventsAndNotifications/templates/actions-template rename to cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/templates/actions-template diff --git a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/templates/events-template b/cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/templates/events-template similarity index 100% rename from cd3_automation_toolkit/ManagementServices/EventsAndNotifications/templates/events-template rename to cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/templates/events-template diff --git a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/templates/notifications-topics-template b/cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/templates/notifications-template similarity index 82% rename from cd3_automation_toolkit/ManagementServices/EventsAndNotifications/templates/notifications-topics-template rename to cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/templates/notifications-template index a813356fd..7d242495b 100644 --- a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/templates/notifications-topics-template +++ b/cd3_automation_toolkit/ocicloud/python/managementservices/eventsnotifications/templates/notifications-template @@ -4,13 +4,13 @@ # ############################ # ManagementServices -# Notifications_Topics - tfvars +# Notifications - tfvars # Allowed Values: # compartment_id can be the ocid or the name of the compartment hierarchy delimited by double hiphens "--" # Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Security--Prod" where "Security" is the parent of "Prod" compartment ############################ -notifications_topics = { +notifications = { ##Add New Topics for {{ region|lower }} here## } {% else %} @@ -18,7 +18,18 @@ notifications_topics = { compartment_id = "{{ compartment_tf_name }}" description = "{{ description }}" topic_name = "{{ topic }}" - + subscriptions = [ + {% for sub in subscriptions %} + { + {% if '::' in sub.endpoint %} + endpoint = "{{ sub.endpoint.split('::')[1] }}" + {% else %} + endpoint = "{{ sub.endpoint }}" + {% endif %} + protocol = "{{ sub.protocol }}" + }{% if not loop.last %},{% endif %} + {% endfor %} + ] {# ##Do not modify below this line## #} {# #} {# ###Section for adding Defined and Freeform Tags### #} diff --git a/cd3_automation_toolkit/ManagementServices/Logging/__init__.py b/cd3_automation_toolkit/ocicloud/python/managementservices/logging/__init__.py similarity index 100% rename from cd3_automation_toolkit/ManagementServices/Logging/__init__.py rename to cd3_automation_toolkit/ocicloud/python/managementservices/logging/__init__.py diff --git a/cd3_automation_toolkit/ManagementServices/Logging/enable_terraform_logging.py b/cd3_automation_toolkit/ocicloud/python/managementservices/logging/enable_terraform_logging.py similarity index 95% rename from cd3_automation_toolkit/ManagementServices/Logging/enable_terraform_logging.py rename to cd3_automation_toolkit/ocicloud/python/managementservices/logging/enable_terraform_logging.py index 9cbc1a528..63934d7fd 100644 --- a/cd3_automation_toolkit/ManagementServices/Logging/enable_terraform_logging.py +++ b/cd3_automation_toolkit/ocicloud/python/managementservices/logging/enable_terraform_logging.py @@ -9,11 +9,13 @@ # Modified (TF Upgrade): Shruthi Subramanian # -import os +import os, sys from jinja2 import Environment, FileSystemLoader from pathlib import Path from oci.config import DEFAULT_LOCATION -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs- Config file, prefix AND outdir @@ -135,10 +137,10 @@ def enable_service_logging(filename, outdir, prefix, ct, service_dir, option='') loggroup_tf_name = commonTools.check_tf_variable(obj_name) + "_" + "flow-log-group" for columnname in dfcolumns: - if columnname.lower() in commonTools.tagColumns: + if columnname.lower() in ociCommonTools.tagColumns: columnvalue = str(df[columnname][i]).strip() if columnvalue != 'nan' and columnvalue != '': - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) tempStr.update(tempdict) tempStr['compartment_tf_name'] = commonTools.check_tf_variable(compartment_name) diff --git a/cd3_automation_toolkit/ManagementServices/Logging/templates/logging-template b/cd3_automation_toolkit/ocicloud/python/managementservices/logging/templates/logging-template similarity index 100% rename from cd3_automation_toolkit/ManagementServices/Logging/templates/logging-template rename to cd3_automation_toolkit/ocicloud/python/managementservices/logging/templates/logging-template diff --git a/cd3_automation_toolkit/ManagementServices/Monitoring/__init__.py b/cd3_automation_toolkit/ocicloud/python/managementservices/monitoring/__init__.py similarity index 100% rename from cd3_automation_toolkit/ManagementServices/Monitoring/__init__.py rename to cd3_automation_toolkit/ocicloud/python/managementservices/monitoring/__init__.py diff --git a/cd3_automation_toolkit/ManagementServices/Monitoring/create_terraform_alarms.py b/cd3_automation_toolkit/ocicloud/python/managementservices/monitoring/create_terraform_alarms.py similarity index 95% rename from cd3_automation_toolkit/ManagementServices/Monitoring/create_terraform_alarms.py rename to cd3_automation_toolkit/ocicloud/python/managementservices/monitoring/create_terraform_alarms.py index aa1ce0cbf..9c0a4332a 100644 --- a/cd3_automation_toolkit/ManagementServices/Monitoring/create_terraform_alarms.py +++ b/cd3_automation_toolkit/ocicloud/python/managementservices/monitoring/create_terraform_alarms.py @@ -9,10 +9,13 @@ # import datetime -from commonTools import * +import os, sys from oci.config import DEFAULT_LOCATION from pathlib import Path from jinja2 import Environment, FileSystemLoader +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools # Execution of the code begins here @@ -106,8 +109,8 @@ def create_terraform_alarms(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Compartment Name": compartmentVarName = commonTools.check_tf_variable(columnvalue) diff --git a/cd3_automation_toolkit/ManagementServices/Monitoring/export_alarms_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/managementservices/monitoring/export_alarms_nonGreenField.py similarity index 93% rename from cd3_automation_toolkit/ManagementServices/Monitoring/export_alarms_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/managementservices/monitoring/export_alarms_nonGreenField.py index d470d0033..7fa69492a 100644 --- a/cd3_automation_toolkit/ManagementServices/Monitoring/export_alarms_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/managementservices/monitoring/export_alarms_nonGreenField.py @@ -9,9 +9,11 @@ # import oci -import os +import os, sys import subprocess as sp -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from oci.config import DEFAULT_LOCATION @@ -60,11 +62,11 @@ def print_alarms(region, alarm, ncpclient,values_for_column, ntk_compartment_nam if (topic_name == "Not_Found"): skip_row=1 values_for_column[col_header].append(topic_name) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(alarm, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(alarm, col_header, values_for_column) else: oci_objs = [alarm,suppression] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) tf_resource = f'module.alarms[\\"{comp_tf_name}_{alarm_tf_name}\\"].oci_monitoring_alarm.alarm' if skip_row == 0 and tf_resource not in state["resources"]: diff --git a/cd3_automation_toolkit/ManagementServices/Monitoring/templates/alarms-template b/cd3_automation_toolkit/ocicloud/python/managementservices/monitoring/templates/alarms-template similarity index 100% rename from cd3_automation_toolkit/ManagementServices/Monitoring/templates/alarms-template rename to cd3_automation_toolkit/ocicloud/python/managementservices/monitoring/templates/alarms-template diff --git a/cd3_automation_toolkit/ManagementServices/ServiceConnectorHub/__init__.py b/cd3_automation_toolkit/ocicloud/python/managementservices/serviceconnectorhub/__init__.py similarity index 100% rename from cd3_automation_toolkit/ManagementServices/ServiceConnectorHub/__init__.py rename to cd3_automation_toolkit/ocicloud/python/managementservices/serviceconnectorhub/__init__.py diff --git a/cd3_automation_toolkit/ManagementServices/ServiceConnectorHub/create_terraform_service_connectors.py b/cd3_automation_toolkit/ocicloud/python/managementservices/serviceconnectorhub/create_terraform_service_connectors.py similarity index 97% rename from cd3_automation_toolkit/ManagementServices/ServiceConnectorHub/create_terraform_service_connectors.py rename to cd3_automation_toolkit/ocicloud/python/managementservices/serviceconnectorhub/create_terraform_service_connectors.py index af2741d6f..cb93f1d3e 100644 --- a/cd3_automation_toolkit/ManagementServices/ServiceConnectorHub/create_terraform_service_connectors.py +++ b/cd3_automation_toolkit/ocicloud/python/managementservices/serviceconnectorhub/create_terraform_service_connectors.py @@ -10,9 +10,9 @@ import sys import os from pathlib import Path - -sys.path.append(os.getcwd() + "/../..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from jinja2 import Environment, FileSystemLoader # Execution of the code begins here @@ -95,8 +95,8 @@ def create_service_connectors(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'Display Name': # columnvalue = columnvalue.strip() diff --git a/cd3_automation_toolkit/ManagementServices/ServiceConnectorHub/export_sch_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/managementservices/serviceconnectorhub/export_sch_nonGreenField.py similarity index 97% rename from cd3_automation_toolkit/ManagementServices/ServiceConnectorHub/export_sch_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/managementservices/serviceconnectorhub/export_sch_nonGreenField.py index e93e3b67f..66c8239d9 100755 --- a/cd3_automation_toolkit/ManagementServices/ServiceConnectorHub/export_sch_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/managementservices/serviceconnectorhub/export_sch_nonGreenField.py @@ -9,9 +9,11 @@ # import oci -import os +import os, sys import subprocess as sp -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools importCommands = {} oci_obj_names = {} @@ -278,7 +280,7 @@ def get_comp_details(comp_data): elif col_header == 'Source Kind': values_for_column[col_header].append(source_kind) elif col_header == "Source Log Group Names" and source_kind == "logging": - log_source_list = [*set(log_source_list)] + #log_source_list = [*set(log_source_list)] values_for_column[col_header].append(",".join(log_source_list)) elif col_header == 'Target Kind': values_for_column[col_header].append(target_kind) @@ -306,11 +308,11 @@ def get_comp_details(comp_data): # elif col_header == 'Stream Partitions' and target_kind == "streaming": # stream_partitions = getattr(target_data, 'stream_partitions') # values_for_column[col_header].append(stream_partitions) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(schs, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(schs, col_header, values_for_column) else: oci_objs = [schs] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) diff --git a/cd3_automation_toolkit/ManagementServices/ServiceConnectorHub/templates/service-connectors-template b/cd3_automation_toolkit/ocicloud/python/managementservices/serviceconnectorhub/templates/service-connectors-template similarity index 100% rename from cd3_automation_toolkit/ManagementServices/ServiceConnectorHub/templates/service-connectors-template rename to cd3_automation_toolkit/ocicloud/python/managementservices/serviceconnectorhub/templates/service-connectors-template diff --git a/cd3_automation_toolkit/Network/Global/__init__.py b/cd3_automation_toolkit/ocicloud/python/network/Global/__init__.py similarity index 100% rename from cd3_automation_toolkit/Network/Global/__init__.py rename to cd3_automation_toolkit/ocicloud/python/network/Global/__init__.py diff --git a/cd3_automation_toolkit/Network/Global/create_rpc_resources.py b/cd3_automation_toolkit/ocicloud/python/network/Global/create_rpc_resources.py similarity index 95% rename from cd3_automation_toolkit/Network/Global/create_rpc_resources.py rename to cd3_automation_toolkit/ocicloud/python/network/Global/create_rpc_resources.py index 23f1ed6d7..622e21361 100755 --- a/cd3_automation_toolkit/Network/Global/create_rpc_resources.py +++ b/cd3_automation_toolkit/ocicloud/python/network/Global/create_rpc_resources.py @@ -13,9 +13,9 @@ import sys import os from pathlib import Path - -sys.path.append(os.getcwd() + "/../..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from jinja2 import Environment, FileSystemLoader # Setting current working dir. @@ -61,9 +61,14 @@ def find_subscribed_regions(inputfile, outdir, service_dir, prefix, config, sign os.makedirs(Path(f"{srcdir}/modules/rpc")) # to save the results + if "user-scripts" in os.getcwd(): + path=f"{owd}/../ocicloud/python/network/Global/templates/rpc-module/rpc-variables-terraform-template" + else: + path=f"{owd}/ocicloud/python/network/Global/templates/rpc-module/rpc-variables-terraform-template" shutil.copyfile( - Path(f"{owd}/Network/Global/templates/rpc-module/rpc-variables-terraform-template"), + Path(path), Path(f"{srcdir}/modules/rpc/variables.tf")) + os.chdir(Path(rf"{srcdir}/rpc/")) with open("rpc.tf", "w") as fh: fh.write(output) @@ -213,8 +218,8 @@ def create_rpc_resource(inputfile, outdir, service_dir, prefix, auth_mechanism, tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'RPC Display Name': columnvalue = columnvalue.strip() diff --git a/cd3_automation_toolkit/Network/Global/templates/rpc-module-template b/cd3_automation_toolkit/ocicloud/python/network/Global/templates/rpc-module-template similarity index 100% rename from cd3_automation_toolkit/Network/Global/templates/rpc-module-template rename to cd3_automation_toolkit/ocicloud/python/network/Global/templates/rpc-module-template diff --git a/cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-provider-terraform-template b/cd3_automation_toolkit/ocicloud/python/network/Global/templates/rpc-module/rpc-provider-terraform-template similarity index 100% rename from cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-provider-terraform-template rename to cd3_automation_toolkit/ocicloud/python/network/Global/templates/rpc-module/rpc-provider-terraform-template diff --git a/cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-source-destination-terraform-template b/cd3_automation_toolkit/ocicloud/python/network/Global/templates/rpc-module/rpc-source-destination-terraform-template similarity index 100% rename from cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-source-destination-terraform-template rename to cd3_automation_toolkit/ocicloud/python/network/Global/templates/rpc-module/rpc-source-destination-terraform-template diff --git a/cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-variables-terraform-template b/cd3_automation_toolkit/ocicloud/python/network/Global/templates/rpc-module/rpc-variables-terraform-template similarity index 100% rename from cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-variables-terraform-template rename to cd3_automation_toolkit/ocicloud/python/network/Global/templates/rpc-module/rpc-variables-terraform-template diff --git a/cd3_automation_toolkit/Network/Global/templates/rpc-root-terraform-template b/cd3_automation_toolkit/ocicloud/python/network/Global/templates/rpc-root-terraform-template similarity index 100% rename from cd3_automation_toolkit/Network/Global/templates/rpc-root-terraform-template rename to cd3_automation_toolkit/ocicloud/python/network/Global/templates/rpc-root-terraform-template diff --git a/cd3_automation_toolkit/ocicloud/python/network/__init__.py b/cd3_automation_toolkit/ocicloud/python/network/__init__.py new file mode 100644 index 000000000..4fb6c107a --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/python/network/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from .basenetwork import * +from .loadbalancers import * +from .dns import * +from .Global import * +#from .IPManagement import * \ No newline at end of file diff --git a/cd3_automation_toolkit/Network/BaseNetwork/__init__.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/__init__.py similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/__init__.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/__init__.py diff --git a/cd3_automation_toolkit/Network/BaseNetwork/cd3parser.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/cd3parser.py similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/cd3parser.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/cd3parser.py diff --git a/cd3_automation_toolkit/Network/BaseNetwork/create_all_tf_objects.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_all_tf_objects.py similarity index 94% rename from cd3_automation_toolkit/Network/BaseNetwork/create_all_tf_objects.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_all_tf_objects.py index 4cde85d72..471986fc3 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/create_all_tf_objects.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_all_tf_objects.py @@ -9,8 +9,10 @@ # Modified (TF Upgrade): Shruthi Subramanian # Modified Rework: Stefen Ramirez (stefen.ramirez@oracle.com) -import os -from commonTools import section +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from .create_major_objects import create_major_objects from .create_terraform_defaults import create_terraform_defaults from .create_terraform_dhcp_options import create_terraform_dhcp_options @@ -19,6 +21,8 @@ from .create_terraform_seclist import create_terraform_seclist from .create_terraform_subnet_vlan import create_terraform_subnet_vlan + + # Execution starts here def create_all_tf_objects(inputfile, outdir, service_dir,prefix, ct, non_gf_tenancy, modify_network=False,network_vlan_in_setupoci="network",network_connectivity_in_setupoci='network'): if not os.path.exists(outdir): diff --git a/cd3_automation_toolkit/Network/BaseNetwork/create_major_objects.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_major_objects.py similarity index 93% rename from cd3_automation_toolkit/Network/BaseNetwork/create_major_objects.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_major_objects.py index 4b1e9840e..de7adab82 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/create_major_objects.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_major_objects.py @@ -9,12 +9,14 @@ # Modified (TF Upgrade): Shruthi Subramanian # import re -import os +import os, sys import json from pathlib import Path from jinja2 import Environment, FileSystemLoader -#sys.path.append(os.getcwd() + "/../../..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +from ocicloud.python.ociCommonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Files @@ -135,7 +137,7 @@ def create_drg_and_attachments(inputfile, outdir): #prevline = line drg_versions[region,drg] = drg_version if (drg_version == "DRGv2"): - for drg_auto_rt_name in commonTools.drg_auto_RTs: + for drg_auto_rt_name in ociCommonTools.drg_auto_RTs: temp = {} drg_auto_rt_tf_name = commonTools.check_tf_variable(drg + "_" + drg_auto_rt_name) temp['drg_auto_rt_tf_name'] = drg_auto_rt_tf_name @@ -143,7 +145,7 @@ def create_drg_and_attachments(inputfile, outdir): temp['drg_tf_name'] = commonTools.check_tf_variable(drg) drg_rt_data[region] = drg_rt_data[region] + drg_datasource_template.render(temp) - for drg_auto_rd_name in commonTools.drg_auto_RDs: + for drg_auto_rd_name in ociCommonTools.drg_auto_RDs: temp = {} drg_auto_rd_tf_name = commonTools.check_tf_variable(drg + "_" + drg_auto_rd_name) temp['drg_auto_rd_tf_name'] = drg_auto_rd_tf_name @@ -233,8 +235,8 @@ def create_drg_and_attachments(inputfile, outdir): tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined Tags and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'Compartment Name': compartment_var_name = columnvalue.strip() @@ -328,7 +330,7 @@ def create_drg_and_attachments(inputfile, outdir): if columnname == 'DRG RT Name': #if it is Auto Generated RT(during export) dont attach any RT to DRG attachment - if(columnvalue in commonTools.drg_auto_RTs): + if(columnvalue in ociCommonTools.drg_auto_RTs): drg_rt_tf_name = '' elif("ocid1.drgroutetable.oc" in columnvalue): drg_rt_tf_name = columnvalue @@ -620,11 +622,12 @@ def processVCN(tempStr): columnvalue = commonTools.check_columnvalue(columnvalue) # Check for multivalued columns - tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) + if columnname not in ["ULA IPv6 CIDR","BYOIP IPv6 Details"]: + tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined Tags and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'Compartment Name': compartment_var_name = columnvalue.strip() @@ -639,6 +642,27 @@ def processVCN(tempStr): cidr_blocks = json.dumps(cidr_blocks) tempdict = {'cidr_blocks': cidr_blocks} + if columnname == 'Oracle GUA IPv6 Enabled' and columnvalue != '': + is_oracle_gua_allocation_enabled = columnvalue.strip() + is_oracle_gua_allocation_enabled = commonTools.check_tf_variable(is_oracle_gua_allocation_enabled) + tempdict = {'is_oracle_gua_allocation_enabled': is_oracle_gua_allocation_enabled.lower()} + + if columnname == "ULA IPv6 CIDR" and columnvalue != '': + ipv6private_cidr_blocks = [x.strip() for x in columnvalue.split(',')] + # reverses the order while exporting into excel so use reverse to avoid terraform change + if (non_gf_tenancy): + ipv6private_cidr_blocks.reverse() + ipv6private_cidr_blocks = json.dumps(ipv6private_cidr_blocks) + tempdict = {'ipv6private_cidr_blocks': ipv6private_cidr_blocks} + + if columnname == "BYOIP IPv6 Details" and columnvalue != '': + byoipv6cidr_details = [x.strip() for x in columnvalue.split(',')] + # reverses the order while exporting into excel so use reverse to avoid terraform change + if (non_gf_tenancy): + byoipv6cidr_details.reverse() + byoipv6cidr_details = json.dumps(byoipv6cidr_details) + tempdict = {'byoipv6cidr_details': byoipv6cidr_details} + if columnname == "DNS Label": # check if vcn_dns_label is not given by user in input use vcn name if str(columnvalue).lower() == 'nan' or str(columnvalue).lower() == '': diff --git a/cd3_automation_toolkit/Network/BaseNetwork/create_terraform_defaults.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_defaults.py similarity index 98% rename from cd3_automation_toolkit/Network/BaseNetwork/create_terraform_defaults.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_defaults.py index 49674e763..34d8f06e1 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/create_terraform_defaults.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_defaults.py @@ -10,7 +10,10 @@ import json from pathlib import Path from jinja2 import Environment, FileSystemLoader -from commonTools import * +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Files @@ -100,8 +103,8 @@ def generate_route_table_string(region_rt_name, region, routetableStr, tempStr, tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) tempStr.update(tempdict) if columnname == 'Compartment Name': diff --git a/cd3_automation_toolkit/Network/BaseNetwork/create_terraform_dhcp_options.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_dhcp_options.py similarity index 97% rename from cd3_automation_toolkit/Network/BaseNetwork/create_terraform_dhcp_options.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_dhcp_options.py index 1c3df843d..2daf9f35f 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/create_terraform_dhcp_options.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_dhcp_options.py @@ -13,9 +13,11 @@ import datetime import os from pathlib import Path -sys.path.append(os.getcwd() + "/../../..") from jinja2 import Environment, FileSystemLoader -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +from ocicloud.python.ociCommonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Files @@ -128,8 +130,8 @@ def processDHCP(tempStr, template, defaultdhcp): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Compartment Name": compartment_var_name = columnvalue.strip() diff --git a/cd3_automation_toolkit/Network/BaseNetwork/create_terraform_nsg.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_nsg.py similarity index 96% rename from cd3_automation_toolkit/Network/BaseNetwork/create_terraform_nsg.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_nsg.py index bd31aff7c..cbb957951 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/create_terraform_nsg.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_nsg.py @@ -11,7 +11,10 @@ from pathlib import Path from oci.config import DEFAULT_LOCATION from jinja2 import Environment, FileSystemLoader -from commonTools import * +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs- CD3 excel file, Config file, prefix AND outdir @@ -67,7 +70,7 @@ def getProtocolNumber(protocol): if protocol.lower() == 'all': return "all" else: - protocol_dict = commonTools().protocol_dict + protocol_dict = ociCommonTools().protocol_dict for k, v in protocol_dict.items(): if (protocol).lower() == v.lower(): return k @@ -204,8 +207,8 @@ def create_terraform_nsg(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'NSG Name': columnvalue = columnvalue.strip() diff --git a/cd3_automation_toolkit/Network/BaseNetwork/create_terraform_route.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_route.py similarity index 98% rename from cd3_automation_toolkit/Network/BaseNetwork/create_terraform_route.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_route.py index 0083caedc..d47925b7b 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/create_terraform_route.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_route.py @@ -13,10 +13,11 @@ import re import os from pathlib import Path -sys.path.append(os.getcwd() + "/../../..") -from commonTools import * from jinja2 import Environment, FileSystemLoader - +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +from ocicloud.python.ociCommonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file, Modify Network AND outdir @@ -216,7 +217,7 @@ def create_terraform_drg_route(inputfile, outdir, service_dir, prefix, ct, non_g continue # Dont create any route table or route distribution name if using Auto Generated ones - if (DRG_RT in commonTools.drg_auto_RTs and DRG_RD in commonTools.drg_auto_RDs): + if (DRG_RT in ociCommonTools.drg_auto_RTs and DRG_RD in ociCommonTools.drg_auto_RDs): continue # Dont create any oute table or route distribution name if OCID is goven in DRG RT Name if ("ocid1.drgroutetable.oc" in DRG_RT): @@ -240,8 +241,8 @@ def create_terraform_drg_route(inputfile, outdir, service_dir, prefix, ct, non_g columnvalue = commonTools.check_columnvalue(columnvalue) # Process Freeform and Defined Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "DRG Name": drg_name = columnvalue @@ -285,10 +286,10 @@ def create_terraform_drg_route(inputfile, outdir, service_dir, prefix, ct, non_g region_rt_name = "#"+region.lower() + "_" + tempStr['drg_rt_tf_name']+"#" tempStr['region_rt_name'] = region_rt_name - if (DRG_RT != 'nan' and DRG_RT not in commonTools.drg_auto_RTs and region_rt_name not in common_rt): + if (DRG_RT != 'nan' and DRG_RT not in ociCommonTools.drg_auto_RTs and region_rt_name not in common_rt): drg_rt[region] = drg_rt[region] + drg_rt_template.render(tempStr) - if (DRG_RD.lower() != 'nan' and DRG_RD not in commonTools.drg_auto_RDs): + if (DRG_RD.lower() != 'nan' and DRG_RD not in ociCommonTools.drg_auto_RDs): drg_rd[region] = drg_rd[region] + drg_rd_template.render(tempStr) if region_rt_name not in common_rt: @@ -1122,8 +1123,8 @@ def processSubnet(tempStr): tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Freeform and Defined Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'Availability Domain(AD1|AD2|AD3|Regional)': columnname = 'availability_domain' diff --git a/cd3_automation_toolkit/Network/BaseNetwork/create_terraform_seclist.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_seclist.py similarity index 97% rename from cd3_automation_toolkit/Network/BaseNetwork/create_terraform_seclist.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_seclist.py index 5ac2b2c55..d12ce4425 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/create_terraform_seclist.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_seclist.py @@ -14,8 +14,11 @@ import os from pathlib import Path from jinja2 import Environment, FileSystemLoader -sys.path.append(os.getcwd() + '/../../..') -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +from ocicloud.python.ociCommonTools import * + +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file, Modify Network AND outdir @@ -244,8 +247,8 @@ def processSubnet(tempStr, service_dir): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process the Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'Compartment Name': compartment_var_name = columnvalue diff --git a/cd3_automation_toolkit/Network/BaseNetwork/create_terraform_subnet_vlan.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_subnet_vlan.py similarity index 97% rename from cd3_automation_toolkit/Network/BaseNetwork/create_terraform_subnet_vlan.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_subnet_vlan.py index 8fdbf0b11..2c1880887 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/create_terraform_subnet_vlan.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/create_terraform_subnet_vlan.py @@ -12,10 +12,12 @@ import sys import re import os -sys.path.append(os.getcwd()+"/../../..") from pathlib import Path from jinja2 import Environment, FileSystemLoader -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +from ocicloud.python.ociCommonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file, prefix AND outdir @@ -217,11 +219,14 @@ def processVlan(tempStr): columnvalue = commonTools.check_columnvalue(columnvalue) # Check for multivalued columns - tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) + if columnname == 'IPv6CIDR Block': + tempdict = {'ipv6cidr_block' : columnvalue} + else: + tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process the Freefrorm and Defined Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'Compartment Name': compartment_var_name = columnvalue diff --git a/cd3_automation_toolkit/Network/BaseNetwork/exportNSG.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/exportNSG.py similarity index 93% rename from cd3_automation_toolkit/Network/BaseNetwork/exportNSG.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/exportNSG.py index 950941c2a..91c556951 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/exportNSG.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/exportNSG.py @@ -5,8 +5,9 @@ from oci.core.virtual_network_client import VirtualNetworkClient import os import subprocess as sp -sys.path.append(os.getcwd()+"/../../..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools def convertNullToNothing(input): EMPTY_STRING = "" @@ -90,7 +91,7 @@ def print_nsgsl(values_for_column_nsgs,vnc,region, comp_name, vcn_name, nsg, nsg icmpcode = nsgsl.icmp_options.code if nsgsl.protocol.lower()!="all": - protocol = str(commonTools().protocol_dict[nsgsl.protocol].lower()) + protocol = str(ociCommonTools().protocol_dict[nsgsl.protocol].lower()) else: protocol="all" for col_header in values_for_column_nsgs.keys(): @@ -127,11 +128,11 @@ def print_nsgsl(values_for_column_nsgs,vnc,region, comp_name, vcn_name, nsg, nsg if str(is_stateless).lower() == 'none': is_stateless = "false" values_for_column_nsgs[col_header].append(str(is_stateless)) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_nsgs = commonTools.export_tags(nsg, col_header, values_for_column_nsgs) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_nsgs = ociCommonTools.export_tags(nsg, col_header, values_for_column_nsgs) else: oci_objs = [nsg,nsgsl] - values_for_column_nsgs = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_nsgs,values_for_column_nsgs) + values_for_column_nsgs = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_nsgs,values_for_column_nsgs) nsg_rule_tf_name = tf_name + "_security_rule" + str(i) tf_resource = f'module.nsg-rules[\\"{nsg_rule_tf_name}\\"].oci_core_network_security_group_security_rule.nsg_rule' @@ -151,11 +152,11 @@ def print_nsg(values_for_column_nsgs,region, comp_name, vcn_name, nsg,state): values_for_column_nsgs[col_header].append(comp_name) elif (col_header == "VCN Name"): values_for_column_nsgs[col_header].append(vcn_name) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_nsgs = commonTools.export_tags(nsg, col_header, values_for_column_nsgs) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_nsgs = ociCommonTools.export_tags(nsg, col_header, values_for_column_nsgs) else: oci_objs = [nsg] - values_for_column_nsgs = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_nsgs,values_for_column_nsgs) + values_for_column_nsgs = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_nsgs,values_for_column_nsgs) tf_resource = f'module.nsgs[\\"{tf_name}\\"].oci_core_network_security_group.network_security_group' if tf_import_cmd and tf_resource not in state["resources"]: importCommands[region.lower()] += f'\n{tf_or_tofu} import "{tf_resource}" {str(nsg.id)}' @@ -197,7 +198,7 @@ def export_nsg(inputfile, outdir, service_dir,config,signer, ct, export_compartm for reg in export_regions: - config.__setitem__("region", commonTools().region_dict[reg]) + config.__setitem__("region", ociCommonTools().region_dict[reg]) state = {'path': f'{outdir}/{reg}/{service_dir}', 'resources': []} try: byteOutput = sp.check_output(tf_state_list, cwd=state["path"],stderr=sp.DEVNULL) diff --git a/cd3_automation_toolkit/Network/BaseNetwork/exportRoutetable.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/exportRoutetable.py similarity index 96% rename from cd3_automation_toolkit/Network/BaseNetwork/exportRoutetable.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/exportRoutetable.py index 4e1e3b9f2..b847ee36b 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/exportRoutetable.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/exportRoutetable.py @@ -5,8 +5,10 @@ from oci.core.virtual_network_client import VirtualNetworkClient import os import subprocess as sp -sys.path.append(os.getcwd()+"/../../..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +from ocicloud.python.ociCommonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools def get_network_entity_name(config,signer,network_identity_id,export_tags): vcn1 = VirtualNetworkClient(config=config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY,signer=signer) @@ -204,8 +206,8 @@ def insert_values(routetable,values_for_column,region,comp_name,name,routerule,e values_for_column[col_header].append(comp_name) elif (col_header == "VCN Name"): values_for_column[col_header].append(name) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(routetable, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(routetable, col_header, values_for_column) elif (routerule != None and col_header == 'Route Destination Object'): network_entity_id = routerule.network_entity_id @@ -226,7 +228,7 @@ def insert_values(routetable,values_for_column,region,comp_name,name,routerule,e values_for_vcninfo['onprem_destinations'].append(routerule.destination) else: oci_objs = [routetable,routerule] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict,values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict,values_for_column) def insert_values_drg(routetable,import_drg_route_distribution_name,values_for_column_drg,region,comp_name,name,routerule,export_tags): @@ -239,8 +241,8 @@ def insert_values_drg(routetable,import_drg_route_distribution_name,values_for_c values_for_column_drg[col_header].append(name) elif (col_header == "Import DRG Route Distribution Name"): values_for_column_drg[col_header].append(import_drg_route_distribution_name) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_drg = commonTools.export_tags(routetable, col_header, values_for_column_drg) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_drg = ociCommonTools.export_tags(routetable, col_header, values_for_column_drg) elif (routerule != None and col_header == 'Next Hop Attachment'): next_hop_attachment_id=routerule.next_hop_drg_attachment_id @@ -253,7 +255,7 @@ def insert_values_drg(routetable,import_drg_route_distribution_name,values_for_c else: oci_objs = [routetable,routerule] - values_for_column_drg = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_drg,values_for_column_drg) + values_for_column_drg = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_drg,values_for_column_drg) @@ -373,7 +375,7 @@ def export_drg_routetable(inputfile, outdir, service_dir,config1,signer1, ct, ex drgv2=parseDRGs(cd3file) for reg in export_regions: - config.__setitem__("region", commonTools().region_dict[reg]) + config.__setitem__("region", ociCommonTools().region_dict[reg]) state = {'path': f'{outdir}/{reg}/{service_dir}', 'resources': []} try: byteOutput = sp.check_output(tf_state_list, cwd=state["path"],stderr=sp.DEVNULL) @@ -473,7 +475,7 @@ def export_drg_routetable(inputfile, outdir, service_dir,config1,signer1, ct, ex drg_rt_name = drg_display_name + "_" + drg_route_table_name drg_rt_tf_name = commonTools.check_tf_variable(drg_rt_name) if tf_import_cmd_drg: - if drg_route_table_name not in commonTools.drg_auto_RTs: + if drg_route_table_name not in ociCommonTools.drg_auto_RTs: tf_resource = f'module.drg-route-tables[\\"{drg_rt_tf_name}\\"].oci_core_drg_route_table.drg_route_table' if tf_resource not in state["resources"]: importCommands_drg[reg] += f'\n{tf_or_tofu} import "{tf_resource}" {drg_route_table_id}' @@ -552,7 +554,7 @@ def export_routetable(inputfile, outdir, service_dir,config1,signer1, ct, export export_compartment_ids.append(ct.ntk_compartment_ids[comp]) for reg in export_regions: - config.__setitem__("region", commonTools().region_dict[reg]) + config.__setitem__("region", ociCommonTools().region_dict[reg]) state = {'path': f'{outdir}/{reg}/{service_dir}', 'resources': []} try: byteOutput = sp.check_output(tf_state_list, cwd=state["path"],stderr=sp.DEVNULL) diff --git a/cd3_automation_toolkit/Network/BaseNetwork/exportSeclist.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/exportSeclist.py similarity index 93% rename from cd3_automation_toolkit/Network/BaseNetwork/exportSeclist.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/exportSeclist.py index a53685440..eeb7466af 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/exportSeclist.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/exportSeclist.py @@ -5,8 +5,10 @@ from oci.core.virtual_network_client import VirtualNetworkClient import os import subprocess as sp -sys.path.append(os.getcwd()+"/../../..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +from ocicloud.python.ociCommonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools def convertNullToNothing(input): EMPTY_STRING = "" @@ -39,10 +41,10 @@ def insert_values(values_for_column,oci_objs, region, comp_name, vcn_name, rulet values_for_column[col_header].append(icmptype) elif (col_header == "ICMPCode"): values_for_column[col_header].append(icmpcode) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(oci_objs[0], col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(oci_objs[0], col_header, values_for_column) else: - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict,values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict,values_for_column) def print_secrules(seclists,region,vcn_name,comp_name,export_tags,state): @@ -193,6 +195,16 @@ def print_secrules(seclists,region,vcn_name,comp_name,export_tags,state): printstr= (dn + ",ingress,icmp," + str(rule.is_stateless) + "," + rule.source + ",,,,,," + type + "," + code+","+desc) insert_values(values_for_column,oci_objs,region, comp_name, vcn_name, 'ingress', 'icmp', '', '','', '', type, code) + elif rule.protocol == "58": + if rule.icmp_options is None: + printstr= (dn + ",ingress,ipv6-icmp," + str(rule.is_stateless) + "," + rule.source + ",,,,,,,,"+desc) + insert_values(values_for_column,oci_objs,region, comp_name, vcn_name, 'ingress', 'ipv6-icmp','', '', '','', '', '') + else: + code = convertNullToNothing(rule.icmp_options.code) + type = convertNullToNothing(rule.icmp_options.type) + printstr= (dn + ",ingress,ipv6-icmp," + str(rule.is_stateless) + "," + rule.source + ",,,,,," + type + "," + code+","+desc) + insert_values(values_for_column,oci_objs,region, comp_name, vcn_name, 'ingress', 'ipv6-icmp', '', '','', '', type, code) + elif rule.protocol == "17": if rule.udp_options is None: printstr= (dn + ",ingress,udp," + str(rule.is_stateless) + "," + rule.source + ",,,,,,,,"+desc) @@ -266,7 +278,7 @@ def export_seclist(inputfile, outdir, service_dir,config,signer, ct, export_comp vcns_check = parseVCNs(cd3file) for reg in export_regions: - config.__setitem__("region", commonTools().region_dict[reg]) + config.__setitem__("region", ociCommonTools().region_dict[reg]) state = {'path': f'{outdir}/{reg}/{service_dir}', 'resources': []} try: byteOutput = sp.check_output(tf_state_list, cwd=state["path"],stderr=sp.DEVNULL) diff --git a/cd3_automation_toolkit/Network/BaseNetwork/export_network_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/export_network_nonGreenField.py similarity index 97% rename from cd3_automation_toolkit/Network/BaseNetwork/export_network_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/export_network_nonGreenField.py index 49fdf86be..49b82754d 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/export_network_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/export_network_nonGreenField.py @@ -11,8 +11,9 @@ import subprocess as sp from pathlib import Path -sys.path.append(os.getcwd() + "/..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools importCommands = {} importCommands_dhcp = {} @@ -93,11 +94,11 @@ def print_drgv2(values_for_column_drgv2, region, comp_name, vcn_info, drg_info, else: statement_val = statement_val + "\n" + "ALL::::" + str(priority) + "\n" values_for_column_drgv2[col_header].append(statement_val) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_drgv2 = commonTools.export_tags(drg_info, col_header, values_for_column_drgv2) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_drgv2 = ociCommonTools.export_tags(drg_info, col_header, values_for_column_drgv2) else: oci_objs = [drg_info, drg_attachment_info, drg_rt_info, import_drg_route_distribution_info] - values_for_column_drgv2 = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_drgv2, + values_for_column_drgv2 = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_drgv2, values_for_column_drgv2) @@ -110,6 +111,12 @@ def print_vcns(values_for_column_vcns, region, comp_name, vnc, vcn_info, drg_att values_for_column_vcns[col_header].append(region) elif (col_header == "Compartment Name"): values_for_column_vcns[col_header].append(comp_name) + elif (col_header == "Is Oracle GUA Allocation Enabled"): + print(vcn_info.ipv6_cidr_blocks) + if vcn_info.ipv6_cidr_blocks and len(vcn_info.ipv6_cidr_blocks) > 0: + values_for_column_vcns[col_header].append("True") + else: + values_for_column_vcns[col_header].append("") elif (col_header == "DRG Required"): if drg_attachment_info != None: if (drg_info == None): @@ -173,11 +180,11 @@ def print_vcns(values_for_column_vcns, region, comp_name, vnc, vcn_info, drg_att values_for_column_vcns[col_header].append(value) elif (col_header == "Hub/Spoke/Peer/None"): values_for_column_vcns[col_header].append("exported") - elif col_header.lower() in commonTools.tagColumns: - values_for_column_vcns = commonTools.export_tags(vcn_info, col_header, values_for_column_vcns) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_vcns = ociCommonTools.export_tags(vcn_info, col_header, values_for_column_vcns) else: oci_objs = [vcn_info, drg_info, igw_info, ngw_info, sgw_info] - values_for_column_vcns = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_vcns, + values_for_column_vcns = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_vcns, values_for_column_vcns) tf_name = commonTools.check_tf_variable(vcn_info.display_name) @@ -221,10 +228,10 @@ def print_dhcp(values_for_column_dhcp, region, comp_name, vcn_name, dhcp_info, s values_for_column_dhcp[col_header].append(search_domain_names_str) elif (col_header == "Custom DNS Server"): values_for_column_dhcp[col_header].append(custom_dns_servers_str) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_dhcp = commonTools.export_tags(dhcp_info, col_header, values_for_column_dhcp) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_dhcp = ociCommonTools.export_tags(dhcp_info, col_header, values_for_column_dhcp) else: - values_for_column_dhcp = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_dhcp, + values_for_column_dhcp = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_dhcp, values_for_column_dhcp) if ("Default DHCP Options for " in dhcp_info.display_name): tf_resource = f'module.default-dhcps[\\"{tf_name}\\"].oci_core_default_dhcp_options.default_dhcp_option' @@ -323,12 +330,12 @@ def print_subnets_vlans(values_for_column_subnets_vlans, region, comp_name, vcn_ values_for_column_subnets_vlans[col_header].append(value) elif subnet_vlan_in_excel == 'VLAN': values_for_column_subnets_vlans[col_header].append("") - elif col_header.lower() in commonTools.tagColumns: - values_for_column_subnets_vlans = commonTools.export_tags(subnet_vlan_info, col_header, + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_subnets_vlans = ociCommonTools.export_tags(subnet_vlan_info, col_header, values_for_column_subnets_vlans) else: oci_objs = [subnet_vlan_info] - values_for_column_subnets_vlans = commonTools.export_extra_columns(oci_objs, col_header, + values_for_column_subnets_vlans = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_subnets_vlans, values_for_column_subnets_vlans) @@ -536,12 +543,12 @@ def get_comp_details(comp_data): statement_val = statement_val + "\n" + "ALL::::" + str(priority) + "\n" values_for_column[col_header].append(statement_val) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(drg_info, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(drg_info, col_header, values_for_column) else: oci_objs = [new_rpc, drg_info, drg_attachment_info, src_drg_rt_dist, src_drg_rt_dist_info] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) # add target region entries to xl.should be ignored during create cal. @@ -586,13 +593,13 @@ def get_comp_details(comp_data): statement_val = statement_val + "\n" + "ALL::::" + str(priority) + "\n" values_for_column[col_header].append(statement_val) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(dest_drg_info, col_header, + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(dest_drg_info, col_header, values_for_column) else: oci_objs = [new_rpc, dest_drg_info, dest_drg_rt_dist, dest_drg_rt_dist_info] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) # print(values_for_column) @@ -811,7 +818,7 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp tf_name = commonTools.check_tf_variable( drg_display_name + "_" + import_drg_route_distribution_info.display_name) if ( - import_drg_route_distribution_info.display_name not in commonTools.drg_auto_RDs and "ocid1.drg.oc" not in drg_display_name): + import_drg_route_distribution_info.display_name not in ociCommonTools.drg_auto_RDs and "ocid1.drg.oc" not in drg_display_name): tf_resource = f'module.drg-route-distributions[\\"{tf_name}\\"].oci_core_drg_route_distribution.drg_route_distribution' if tf_resource not in state["resources"]: importCommands[reg].write( @@ -853,7 +860,7 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp tf_name = commonTools.check_tf_variable( drg_display_name + "_" + import_drg_route_distribution_info.display_name) if ( - import_drg_route_distribution_info.display_name not in commonTools.drg_auto_RDs and write_drg_ocids == False): + import_drg_route_distribution_info.display_name not in ociCommonTools.drg_auto_RDs and write_drg_ocids == False): tf_resource = f'module.drg-route-distributions[\\"{tf_name}\\"].oci_core_drg_route_distribution.drg_route_distribution' if tf_resource not in state["resources"]: importCommands[reg].write( @@ -923,7 +930,7 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp tf_name = commonTools.check_tf_variable( drg_display_name + "_" + import_drg_route_distribution_info.display_name) - if (import_drg_route_distribution_info.display_name not in commonTools.drg_auto_RDs): + if (import_drg_route_distribution_info.display_name not in ociCommonTools.drg_auto_RDs): tf_resource = f'module.drg-route-distributions[\\"{tf_name}\\"].oci_core_drg_route_distribution.drg_route_distribution' if tf_resource not in state["resources"]: importCommands[reg].write( diff --git a/cd3_automation_toolkit/Network/BaseNetwork/export_seclist_to_tf.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/export_seclist_to_tf.py similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/export_seclist_to_tf.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/export_seclist_to_tf.py diff --git a/cd3_automation_toolkit/Network/BaseNetwork/modify_routerules_tf.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/modify_routerules_tf.py similarity index 96% rename from cd3_automation_toolkit/Network/BaseNetwork/modify_routerules_tf.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/modify_routerules_tf.py index 5b92a9e00..2b1d733d6 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/modify_routerules_tf.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/modify_routerules_tf.py @@ -13,9 +13,11 @@ import os from oci.config import DEFAULT_LOCATION from pathlib import Path -sys.path.append(os.getcwd()+"/../../..") -from commonTools import * from jinja2 import Environment, FileSystemLoader +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +from ocicloud.python.ociCommonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Takes in input CD3 excel which contains routerules to be updated for the subnet and updates the routes tf file created using BaseNetwork TF generation. @@ -105,7 +107,7 @@ def modify_terraform_drg_routerules(inputfile, outdir, service_dir,prefix, ct, n continue # Process RTs only for those Import DRG Route Distribution Names which are present in cd3(and have been created via TF) - if (DRG_RD_Name not in drgv2.drg_rds[DRG_Name,region] and DRG_RD_Name.lower()!='nan' and DRG_RD_Name not in commonTools.drg_auto_RDs): + if (DRG_RD_Name not in drgv2.drg_rds[DRG_Name,region] and DRG_RD_Name.lower()!='nan' and DRG_RD_Name not in ociCommonTools.drg_auto_RDs): print("skipping DRG route table: " + str(df.loc[i, 'DRG RT Name']) + " as either its DRG is not part of DRGv2 tab in cd3 or its Import Route Distribution Name is not attached to DRG "+ DRG_Name+" as per DRGv2 tab in cd3") continue @@ -132,8 +134,8 @@ def modify_terraform_drg_routerules(inputfile, outdir, service_dir,prefix, ct, n tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) tempStr.update(tempdict) @@ -151,7 +153,7 @@ def modify_terraform_drg_routerules(inputfile, outdir, service_dir,prefix, ct, n if(columnname == "Import DRG Route Distribution Name"): - if columnvalue in commonTools.drg_auto_RDs: + if columnvalue in ociCommonTools.drg_auto_RDs: drg_rt_dstrb_tf_name = commonTools.check_tf_variable(drg_name + "_" + columnvalue) drg_rt_dstrb_res_name = drg_rt_dstrb_tf_name elif columnvalue!='': @@ -187,7 +189,7 @@ def modify_terraform_drg_routerules(inputfile, outdir, service_dir,prefix, ct, n if(len(rts_done[region])==0): k=1 #Create RT resource only if it is not Auto Generated one - if (DRG_RT not in commonTools.drg_auto_RTs): + if (DRG_RT not in ociCommonTools.drg_auto_RTs): tfStrRT[region] = tfStrRT[region] + routetable_drg.render(tempStr) rts_done[region].append(drg_rt_tf_name) @@ -195,7 +197,7 @@ def modify_terraform_drg_routerules(inputfile, outdir, service_dir,prefix, ct, n rts_done[region].append(drg_rt_tf_name) # Create RT resource only if it is not Auto Generated one - if (DRG_RT not in commonTools.drg_auto_RTs): + if (DRG_RT not in ociCommonTools.drg_auto_RTs): tfStrRT[region] = tfStrRT[region] + routetable_drg.render(tempStr) k=1 @@ -392,7 +394,11 @@ def generate_route_table_string(region_rt_name,region,routetableStr,tempStr,comm for columnname in dfcolumns: # Column value - columnvalue = str(df[columnname][i]).strip() + if (columnname != 'Rule Description'): + columnvalue = str(df[columnname][i]).strip() + else: + columnvalue = str(df[columnname][i]) + # Check for boolean/null in column values columnvalue = commonTools.check_columnvalue(columnvalue) @@ -401,8 +407,8 @@ def generate_route_table_string(region_rt_name,region,routetableStr,tempStr,comm tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) tempStr.update(tempdict) if columnname == 'Compartment Name': @@ -434,7 +440,7 @@ def generate_route_table_string(region_rt_name,region,routetableStr,tempStr,comm tempStr.update(tempdict) if columnname == 'Rule Description': - description = columnvalue.strip() + description = columnvalue if description == 'nan': description = "" tempdict = {'description': description} diff --git a/cd3_automation_toolkit/Network/BaseNetwork/modify_secrules_tf.py b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/modify_secrules_tf.py similarity index 96% rename from cd3_automation_toolkit/Network/BaseNetwork/modify_secrules_tf.py rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/modify_secrules_tf.py index 4ce712ce5..9e5226cd3 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/modify_secrules_tf.py +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/modify_secrules_tf.py @@ -15,8 +15,11 @@ from oci.config import DEFAULT_LOCATION from pathlib import Path from jinja2 import Environment, FileSystemLoader -sys.path.append(os.getcwd() + "/../../..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +from ocicloud.python.ociCommonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools + # Execution of the code begins here def modify_terraform_secrules(inputfile, outdir, service_dir,prefix, ct, non_gf_tenancy): @@ -164,8 +167,8 @@ def generate_security_rules(region_seclist_name,processed_seclist,tfStr,region,t tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'SecList Name': display_name = columnvalue.strip() @@ -221,4 +224,5 @@ def generate_security_rules(region_seclist_name,processed_seclist,tfStr,region,t print(default_outfile + " for default seclist has been created for region " + reg) - os.remove('out.csv') + if os.path.exists('out.csv'): + os.remove('out.csv') diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/custom-dhcp-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/custom-dhcp-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/custom-dhcp-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/custom-dhcp-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/default-route-table-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/default-route-table-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/default-route-table-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/default-route-table-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/default-seclist-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/default-seclist-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/default-seclist-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/default-seclist-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/drg-data-source-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/drg-data-source-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/drg-data-source-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/drg-data-source-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/drg-route-distribution-statement-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/drg-route-distribution-statement-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/drg-route-distribution-statement-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/drg-route-distribution-statement-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/drg-route-distribution-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/drg-route-distribution-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/drg-route-distribution-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/drg-route-distribution-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/drg-route-rule-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/drg-route-rule-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/drg-route-rule-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/drg-route-rule-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/drg-route-table-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/drg-route-table-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/drg-route-table-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/drg-route-table-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-default-dhcp-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-default-dhcp-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-default-dhcp-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-default-dhcp-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-drg-attachments-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-drg-attachments-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-drg-attachments-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-drg-attachments-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-drgs-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-drgs-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-drgs-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-drgs-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-igws-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-igws-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-igws-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-igws-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-lpgs-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-lpgs-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-lpgs-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-lpgs-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-ngws-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-ngws-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-ngws-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-ngws-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-sgws-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-sgws-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-sgws-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-sgws-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-vcns-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-vcns-template similarity index 88% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-vcns-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-vcns-template index cc99e8415..3febe5d07 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-vcns-template +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/major-objects-vcns-template @@ -33,14 +33,20 @@ vcns = { {% endif %} {% if byoipv6cidr_details and byoipv6cidr_details != [] %} - byoipv6cidr_details = [{ - byoipv6range_id = "{{ byoipv6range_id }}" - ipv6cidr_block = "{{ ipv6cidr_block }}" - }] + byoipv6cidr_details = [ + {% for item in byoipv6cidr_details %} + {% set last_index = item.rfind('@') %} + {% if last_index != -1 %} + { + byoipv6range_data = item[:last_index] + ipv6cidr_block = item[last_index+1:] + } + {% endif %} + {% endfor %}] {% endif %} {% if ipv6private_cidr_blocks and ipv6private_cidr_blocks != [] %} - ipv6private_cidr_blocks = [ "{{ ipv6private_cidr_blocks }}" ] + ipv6private_cidr_blocks = {{ ipv6private_cidr_blocks }} {% endif %} {% if is_oracle_gua_allocation_enabled and is_oracle_gua_allocation_enabled != "" %} diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/nsg-rule-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/nsg-rule-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/nsg-rule-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/nsg-rule-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/nsg-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/nsg-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/nsg-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/nsg-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/route-rule-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/route-rule-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/route-rule-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/route-rule-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/route-table-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/route-table-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/route-table-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/route-table-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/sec-rule-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/sec-rule-template similarity index 99% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/sec-rule-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/sec-rule-template index f0a04c83b..9654612a3 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/templates/sec-rule-template +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/sec-rule-template @@ -20,7 +20,7 @@ {% if protocol.lower() == "all" %} all = [] - {% elif protocol.lower() == 'icmp' %} + {% elif 'icmp' in protocol.lower() %} {% if icmptype != '' or icmpcode != '' %} icmp = [{ {% if icmptype != '' %} diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/seclist-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/seclist-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/seclist-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/seclist-template diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/subnet-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/subnet-template similarity index 95% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/subnet-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/subnet-template index 6bafaf1cb..0b2d16aed 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/templates/subnet-template +++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/subnet-template @@ -1,5 +1,5 @@ {% if (skeleton and count == 0) %} -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. # ############################# @@ -54,10 +54,12 @@ subnets = { {% if seclist_ids != '' %} security_list_ids = [{{ seclist_ids }}] + {% else %} + security_list_ids = [] {% endif %} {% if ipv6cidr_block and ipv6cidr_block != '' %} - ipv6cidr_block = {{ ipv6cidr_block }} + ipv6cidr_block = "{{ ipv6cidr_block }}" {% endif %} {# ##Do not modify below this line## #} diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/vlan-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/vlan-template similarity index 100% rename from cd3_automation_toolkit/Network/BaseNetwork/templates/vlan-template rename to cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/vlan-template diff --git a/cd3_automation_toolkit/Network/DNS/__init__.py b/cd3_automation_toolkit/ocicloud/python/network/dns/__init__.py similarity index 100% rename from cd3_automation_toolkit/Network/DNS/__init__.py rename to cd3_automation_toolkit/ocicloud/python/network/dns/__init__.py diff --git a/cd3_automation_toolkit/Network/DNS/create_dns_resolvers.py b/cd3_automation_toolkit/ocicloud/python/network/dns/create_dns_resolvers.py similarity index 97% rename from cd3_automation_toolkit/Network/DNS/create_dns_resolvers.py rename to cd3_automation_toolkit/ocicloud/python/network/dns/create_dns_resolvers.py index 3f7e70fe0..bb743d4be 100644 --- a/cd3_automation_toolkit/Network/DNS/create_dns_resolvers.py +++ b/cd3_automation_toolkit/ocicloud/python/network/dns/create_dns_resolvers.py @@ -5,11 +5,13 @@ # DNS-Resolvers # -import os +import os, sys from jinja2 import Environment, FileSystemLoader from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### @@ -285,8 +287,8 @@ def create_terraform_dns_resolvers(inputfile, outdir, service_dir, prefix, ct): tempdict = {'res_resolver_rules': res_rules} # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) columnname = commonTools.check_column_headers(columnname) tempStr[columnname] = str(columnvalue).strip() diff --git a/cd3_automation_toolkit/Network/DNS/create_dns_rrsets.py b/cd3_automation_toolkit/ocicloud/python/network/dns/create_dns_rrsets.py similarity index 97% rename from cd3_automation_toolkit/Network/DNS/create_dns_rrsets.py rename to cd3_automation_toolkit/ocicloud/python/network/dns/create_dns_rrsets.py index 5fb822620..2f72d1ee9 100644 --- a/cd3_automation_toolkit/Network/DNS/create_dns_rrsets.py +++ b/cd3_automation_toolkit/ocicloud/python/network/dns/create_dns_rrsets.py @@ -5,11 +5,13 @@ # DNS-RRset # -import os +import os, sys from jinja2 import Environment, FileSystemLoader from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### diff --git a/cd3_automation_toolkit/Network/DNS/create_dns_views.py b/cd3_automation_toolkit/ocicloud/python/network/dns/create_dns_views.py similarity index 94% rename from cd3_automation_toolkit/Network/DNS/create_dns_views.py rename to cd3_automation_toolkit/ocicloud/python/network/dns/create_dns_views.py index 4532df824..eaab6f006 100644 --- a/cd3_automation_toolkit/Network/DNS/create_dns_views.py +++ b/cd3_automation_toolkit/ocicloud/python/network/dns/create_dns_views.py @@ -4,11 +4,13 @@ # This script will produce a Terraform file that will be used to set up OCI core components # DNS-Views # -import os +import os, sys from jinja2 import Environment, FileSystemLoader from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### @@ -104,8 +106,8 @@ def create_terraform_dns_views(inputfile, outdir, service_dir, prefix, ct): tempdict = {'view_display_name': display_tf_name} # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) columnname = commonTools.check_column_headers(columnname) tempStr[columnname] = str(columnvalue).strip() diff --git a/cd3_automation_toolkit/Network/DNS/create_dns_zones.py b/cd3_automation_toolkit/ocicloud/python/network/dns/create_dns_zones.py similarity index 94% rename from cd3_automation_toolkit/Network/DNS/create_dns_zones.py rename to cd3_automation_toolkit/ocicloud/python/network/dns/create_dns_zones.py index 54af6b0b7..e09cc3d52 100644 --- a/cd3_automation_toolkit/Network/DNS/create_dns_zones.py +++ b/cd3_automation_toolkit/ocicloud/python/network/dns/create_dns_zones.py @@ -4,11 +4,13 @@ # This script will produce a Terraform file that will be used to set up OCI core components # DNS-Zones # -import os +import os, sys from jinja2 import Environment, FileSystemLoader from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### @@ -99,8 +101,8 @@ def create_terraform_dns_zones(inputfile, outdir, service_dir, prefix, ct): tempdict = {'zone_compartment_name': columnvalue}""" # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'Zone': tempdict = {'display_name': zone_name} diff --git a/cd3_automation_toolkit/Network/DNS/export_dns_resolvers.py b/cd3_automation_toolkit/ocicloud/python/network/dns/export_dns_resolvers.py similarity index 97% rename from cd3_automation_toolkit/Network/DNS/export_dns_resolvers.py rename to cd3_automation_toolkit/ocicloud/python/network/dns/export_dns_resolvers.py index da22d814d..2a3c481a9 100644 --- a/cd3_automation_toolkit/Network/DNS/export_dns_resolvers.py +++ b/cd3_automation_toolkit/ocicloud/python/network/dns/export_dns_resolvers.py @@ -5,15 +5,15 @@ # Export DNS Resolvers # import oci -import os +import os, sys import subprocess as sp -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools + importCommands = {} oci_obj_names = {} - - - # Create map for each endpoint def get_e_map(region, dns_client, vnc_client, ct, resolver, ntk_compartment_name): vcn_name = vnc_client.get_vcn(resolver.attached_vcn_id).data.display_name @@ -133,8 +133,10 @@ def print_resolvers(resolver_tf_name, resolver, values_for_column,state, **value values_for_column[col_header].append(endpoint_value['e_nsgs']) elif col_header == 'Rules': values_for_column[col_header].append(endpoint_value['res_rule_detail']) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(resolver, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(resolver, col_header, values_for_column) + + # Execution of the code begins here def export_dns_resolvers(inputfile, outdir, service_dir, config, signer, ct, export_compartments=[], export_regions=[],export_tags=[]): diff --git a/cd3_automation_toolkit/Network/DNS/export_dns_views_zones_records.py b/cd3_automation_toolkit/ocicloud/python/network/dns/export_dns_views_zones_records.py similarity index 94% rename from cd3_automation_toolkit/Network/DNS/export_dns_views_zones_records.py rename to cd3_automation_toolkit/ocicloud/python/network/dns/export_dns_views_zones_records.py index 961dff9d7..5ad3e1844 100644 --- a/cd3_automation_toolkit/Network/DNS/export_dns_views_zones_records.py +++ b/cd3_automation_toolkit/ocicloud/python/network/dns/export_dns_views_zones_records.py @@ -5,9 +5,11 @@ # Export DNS views-zones-rrsets # import oci -import os +import os, sys import subprocess as sp -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools importCommands = {} oci_obj_names = {} @@ -71,8 +73,8 @@ def print_data(region, ntk_compartment_name, rrset, zone_data, view_data, values values_for_column[col_header].append(rrset['rdata']) elif col_header == 'TTL': values_for_column[col_header].append(rrset['ttl']) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(view_data, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(view_data, col_header, values_for_column) def print_empty_view(region, ntk_compartment_name, view_data, values_for_column,zone_name=""): @@ -94,8 +96,8 @@ def print_empty_view(region, ntk_compartment_name, view_data, values_for_column, values_for_column[col_header].append("") elif col_header == 'TTL': values_for_column[col_header].append("") - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(view_data, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(view_data, col_header, values_for_column) # Execution of the code begins here def export_dns_views_zones_rrsets(inputfile, outdir, service_dir, config, signer, ct, dns_filter, export_compartments=[], export_regions=[],export_tags=[]): @@ -225,8 +227,8 @@ def export_dns_views_zones_rrsets(inputfile, outdir, service_dir, config, signer else: print_empty_view(region, ntk_compartment_name, view_data, values_for_column) - if print_zone==False: - print_empty_view(region, ntk_compartment_name, view_data, values_for_column) + # if print_zone==False: + # print_empty_view(region, ntk_compartment_name, view_data, values_for_column) tf_resource = f'module.dns-views[\\"{view_tf_name}\\"].oci_dns_view.view' if tf_resource not in state["resources"]: diff --git a/cd3_automation_toolkit/Network/DNS/templates/dns-records-template b/cd3_automation_toolkit/ocicloud/python/network/dns/templates/dns-records-template similarity index 100% rename from cd3_automation_toolkit/Network/DNS/templates/dns-records-template rename to cd3_automation_toolkit/ocicloud/python/network/dns/templates/dns-records-template diff --git a/cd3_automation_toolkit/Network/DNS/templates/dns-resolvers-template b/cd3_automation_toolkit/ocicloud/python/network/dns/templates/dns-resolvers-template similarity index 100% rename from cd3_automation_toolkit/Network/DNS/templates/dns-resolvers-template rename to cd3_automation_toolkit/ocicloud/python/network/dns/templates/dns-resolvers-template diff --git a/cd3_automation_toolkit/Network/DNS/templates/dns-views-template b/cd3_automation_toolkit/ocicloud/python/network/dns/templates/dns-views-template similarity index 100% rename from cd3_automation_toolkit/Network/DNS/templates/dns-views-template rename to cd3_automation_toolkit/ocicloud/python/network/dns/templates/dns-views-template diff --git a/cd3_automation_toolkit/Network/DNS/templates/dns-zones-template b/cd3_automation_toolkit/ocicloud/python/network/dns/templates/dns-zones-template similarity index 100% rename from cd3_automation_toolkit/Network/DNS/templates/dns-zones-template rename to cd3_automation_toolkit/ocicloud/python/network/dns/templates/dns-zones-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/__init__.py b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/__init__.py similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/__init__.py rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/__init__.py diff --git a/cd3_automation_toolkit/Network/LoadBalancers/create_backendset_backendservers.py b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_backendset_backendservers.py similarity index 97% rename from cd3_automation_toolkit/Network/LoadBalancers/create_backendset_backendservers.py rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_backendset_backendservers.py index ae6bf8e0a..ca5f95b31 100644 --- a/cd3_automation_toolkit/Network/LoadBalancers/create_backendset_backendservers.py +++ b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_backendset_backendservers.py @@ -11,8 +11,11 @@ import pandas as pd from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir @@ -105,8 +108,8 @@ def create_backendset_backendservers(inputfile, outdir, service_dir, prefix, ct) tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Compartment Name": columnname = "compartment_tf_name" diff --git a/cd3_automation_toolkit/Network/LoadBalancers/create_listener.py b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_listener.py similarity index 96% rename from cd3_automation_toolkit/Network/LoadBalancers/create_listener.py rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_listener.py index 5568b75e9..897346011 100644 --- a/cd3_automation_toolkit/Network/LoadBalancers/create_listener.py +++ b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_listener.py @@ -11,8 +11,11 @@ import pandas as pd from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir @@ -101,8 +104,8 @@ def create_listener(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "LBR Name": lbr_tf_name = commonTools.check_tf_variable(columnvalue) diff --git a/cd3_automation_toolkit/Network/LoadBalancers/create_nlb_backendset_backendservers.py b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_nlb_backendset_backendservers.py similarity index 94% rename from cd3_automation_toolkit/Network/LoadBalancers/create_nlb_backendset_backendservers.py rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_nlb_backendset_backendservers.py index 3a957137b..c304874d4 100644 --- a/cd3_automation_toolkit/Network/LoadBalancers/create_nlb_backendset_backendservers.py +++ b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_nlb_backendset_backendservers.py @@ -9,8 +9,11 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir @@ -81,8 +84,8 @@ def create_nlb_backendset_backendservers(inputfile, outdir, service_dir, prefix, backend_set_tf_name = '' #Check if mandatory field is empty - if (str(df.loc[i,'Backend Set Name']).lower() == 'nan' or str(df.loc[i,'Backend HealthCheck Port']).lower() == 'nan'): - print("\nColumn Backend Set Name or Backend HealthCheck Port cannot be left empty.....Exiting!") + if (str(df.loc[i,'Backend Set Name']).lower() == 'nan'): + print("\nColumn Backend Set Name cannot be left empty.....Exiting!") exit(1) # Fetch data; loop through columns @@ -98,8 +101,8 @@ def create_nlb_backendset_backendservers(inputfile, outdir, service_dir, prefix, tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Compartment Name": columnname = "compartment_tf_name" diff --git a/cd3_automation_toolkit/Network/LoadBalancers/create_path_route_set.py b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_path_route_set.py similarity index 95% rename from cd3_automation_toolkit/Network/LoadBalancers/create_path_route_set.py rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_path_route_set.py index f84290270..3a186d685 100644 --- a/cd3_automation_toolkit/Network/LoadBalancers/create_path_route_set.py +++ b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_path_route_set.py @@ -10,8 +10,11 @@ import pandas as pd from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### @@ -101,8 +104,8 @@ def create_path_route_set(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "LBR Name": lbr_tf_name = commonTools.check_tf_variable(columnvalue) diff --git a/cd3_automation_toolkit/Network/LoadBalancers/create_routing_policy.py b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_routing_policy.py similarity index 94% rename from cd3_automation_toolkit/Network/LoadBalancers/create_routing_policy.py rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_routing_policy.py index 09851538f..b6fac148f 100755 --- a/cd3_automation_toolkit/Network/LoadBalancers/create_routing_policy.py +++ b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_routing_policy.py @@ -11,7 +11,10 @@ import pandas as pd from jinja2 import Environment, FileSystemLoader from pathlib import Path -from commonTools import * +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools @@ -93,8 +96,8 @@ def create_lb_routing_policy(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "LBR Name": lbr_tf_name = commonTools.check_tf_variable(columnvalue) diff --git a/cd3_automation_toolkit/Network/LoadBalancers/create_ruleset.py b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_ruleset.py similarity index 97% rename from cd3_automation_toolkit/Network/LoadBalancers/create_ruleset.py rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_ruleset.py index 55133f6f3..7fa339596 100644 --- a/cd3_automation_toolkit/Network/LoadBalancers/create_ruleset.py +++ b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_ruleset.py @@ -12,8 +12,11 @@ import pandas as pd from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir @@ -190,8 +193,8 @@ def add_rules(df,rs_str,tempStr,control_access): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'Allow Invalid Characters (TRUE|FALSE)': columnname = 'allow_invalid_characters' diff --git a/cd3_automation_toolkit/Network/LoadBalancers/create_terraform_lbr_hostname_certs.py b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_terraform_lbr_hostname_certs.py similarity index 97% rename from cd3_automation_toolkit/Network/LoadBalancers/create_terraform_lbr_hostname_certs.py rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_terraform_lbr_hostname_certs.py index e320c3765..4f405bd8f 100644 --- a/cd3_automation_toolkit/Network/LoadBalancers/create_terraform_lbr_hostname_certs.py +++ b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_terraform_lbr_hostname_certs.py @@ -14,8 +14,11 @@ import pandas as pd from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir @@ -110,8 +113,8 @@ def certificate_templates(dfcert): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "LBR Name": lbr_tf_name = commonTools.check_tf_variable(columnvalue) @@ -221,8 +224,8 @@ def certificate_templates(dfcert): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Is Private(True|False)": columnname = 'is_private' diff --git a/cd3_automation_toolkit/Network/LoadBalancers/create_terraform_nlb_listener.py b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_terraform_nlb_listener.py similarity index 97% rename from cd3_automation_toolkit/Network/LoadBalancers/create_terraform_nlb_listener.py rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_terraform_nlb_listener.py index d6d5406ff..2cb3ea257 100644 --- a/cd3_automation_toolkit/Network/LoadBalancers/create_terraform_nlb_listener.py +++ b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/create_terraform_nlb_listener.py @@ -9,8 +9,11 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir @@ -111,8 +114,8 @@ def create_terraform_nlb_listener(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Compartment Name": columnname = "compartment_tf_name" diff --git a/cd3_automation_toolkit/Network/LoadBalancers/export_lbr_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/export_lbr_nonGreenField.py similarity index 97% rename from cd3_automation_toolkit/Network/LoadBalancers/export_lbr_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/export_lbr_nonGreenField.py index c31fc16b0..129df2e87 100644 --- a/cd3_automation_toolkit/Network/LoadBalancers/export_lbr_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/export_lbr_nonGreenField.py @@ -16,8 +16,9 @@ from oci.core.virtual_network_client import VirtualNetworkClient from oci.load_balancer.load_balancer_client import LoadBalancerClient -sys.path.append(os.getcwd() + "/..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools importCommands = {} oci_obj_names = {} @@ -135,10 +136,10 @@ def insert_values(values_for_column, oci_objs, sheet_dict, region, comp_name, di values_for_column[col_header].append(minimum_bandwidth_in_mbps) elif col_header == 'Maximum Bandwidth In Mbps (Flexible shapes only)': values_for_column[col_header].append(maximum_bandwidth_in_mbps) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(oci_objs[0], col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(oci_objs[0], col_header, values_for_column) else: - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) def print_lbr_hostname_certs(region, ct, outdir, values_for_column_lhc, lbr, LBRs, lbr_compartment_name, export_tags, network, @@ -523,8 +524,8 @@ def print_backendset_backendserver(region, ct, values_for_column_bss, lbr, LBRs, sheet_dict_common, lbr_compartment_name) # Process the Tag Columns - elif headers_lower in commonTools.tagColumns: - values_for_column_bss = commonTools.export_tags(eachlbr, col_headers, values_for_column_bss) + elif headers_lower in ociCommonTools.tagColumns: + values_for_column_bss = ociCommonTools.export_tags(eachlbr, col_headers, values_for_column_bss) elif col_headers == 'SSL Protocols': protocols_list = '' @@ -595,14 +596,14 @@ def print_backendset_backendserver(region, ct, values_for_column_bss, lbr, LBRs, else: oci_objs = [backendset_details, eachlbr, hc, certificate_list] - values_for_column_bss = commonTools.export_extra_columns(oci_objs, col_headers, sheet_dict_bss, + values_for_column_bss = ociCommonTools.export_extra_columns(oci_objs, col_headers, sheet_dict_bss, values_for_column_bss) else: if "Cookie" not in col_headers: # Process the remaining Columns oci_objs = [backendset_details, eachlbr, hc, certificate_list] - values_for_column_bss = commonTools.export_extra_columns(oci_objs, col_headers, sheet_dict_bss, + values_for_column_bss = ociCommonTools.export_extra_columns(oci_objs, col_headers, sheet_dict_bss, values_for_column_bss) return values_for_column_bss @@ -696,8 +697,8 @@ def print_listener(region, ct, values_for_column_lis, LBRs, lbr_compartment_name sheet_dict_common, lbr_compartment_name) # Process the Tag Columns - elif headers_lower in commonTools.tagColumns: - values_for_column_lis = commonTools.export_tags(eachlbr, col_headers, values_for_column_lis) + elif headers_lower in ociCommonTools.tagColumns: + values_for_column_lis = ociCommonTools.export_tags(eachlbr, col_headers, values_for_column_lis) # Process the Listerner Columns elif col_headers in sheet_dict_lis.keys(): @@ -732,12 +733,12 @@ def print_listener(region, ct, values_for_column_lis, LBRs, lbr_compartment_name else: oci_objs = [values, eachlbr, sslcerts] - values_for_column_lis = commonTools.export_extra_columns(oci_objs, col_headers, sheet_dict_lis, + values_for_column_lis = ociCommonTools.export_extra_columns(oci_objs, col_headers, sheet_dict_lis, values_for_column_lis) else: oci_objs = [eachlbr, values, sslcerts] - values_for_column_lis = commonTools.export_extra_columns(oci_objs, col_headers, sheet_dict_lis, + values_for_column_lis = ociCommonTools.export_extra_columns(oci_objs, col_headers, sheet_dict_lis, values_for_column_lis) return values_for_column_lis @@ -869,7 +870,7 @@ def print_rule(region, ct, values_for_column_rule, LBRs, lbr_compartment_name,ex pass else: oci_objs = [eachlbr, eachitem] - values_for_column_rule = commonTools.export_extra_columns(oci_objs, col_headers, + values_for_column_rule = ociCommonTools.export_extra_columns(oci_objs, col_headers, sheet_dict_rule, values_for_column_rule) @@ -933,7 +934,7 @@ def print_prs(region, ct, values_for_column_prs, LBRs, lbr_compartment_name,expo else: # Process the remaining Columns oci_objs = [eachlbr, values, path_routes] - values_for_column_prs = commonTools.export_extra_columns(oci_objs, col_headers, sheet_dict_prs, + values_for_column_prs = ociCommonTools.export_extra_columns(oci_objs, col_headers, sheet_dict_prs, values_for_column_prs) return values_for_column_prs @@ -1015,7 +1016,7 @@ def print_routing_policies(region, ct, values_for_column_rp, LBRs, lbr_compartme else: # Process the remaining Columns oci_objs = [eachlbr, values] - values_for_column_rp = commonTools.export_extra_columns(oci_objs, col_headers, + values_for_column_rp = ociCommonTools.export_extra_columns(oci_objs, col_headers, sheet_dict_rp, values_for_column_rp) diff --git a/cd3_automation_toolkit/Network/LoadBalancers/export_nlb_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/export_nlb_nonGreenField.py similarity index 94% rename from cd3_automation_toolkit/Network/LoadBalancers/export_nlb_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/export_nlb_nonGreenField.py index df038c384..3e63a031d 100644 --- a/cd3_automation_toolkit/Network/LoadBalancers/export_nlb_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/export_nlb_nonGreenField.py @@ -15,8 +15,9 @@ from oci.core.virtual_network_client import VirtualNetworkClient from oci.network_load_balancer import NetworkLoadBalancerClient from oci.core.compute_client import ComputeClient -sys.path.append(os.getcwd()+"/..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools importCommands = {} oci_obj_names = {} @@ -136,9 +137,9 @@ def print_nlb_backendset_backendserver(region, values_for_column_bss,NLBs, nlb_c values_for_column_bss[col_header].append('') # Process the Tag Columns - elif col_header.lower() in commonTools.tagColumns: + elif col_header.lower() in ociCommonTools.tagColumns: if cnt_bss == 1: - values_for_column_bss = commonTools.export_tags(eachnlb, col_header, values_for_column_bss) + values_for_column_bss = ociCommonTools.export_tags(eachnlb, col_header, values_for_column_bss) else: values_for_column_bss[col_header].append('') @@ -156,7 +157,7 @@ def print_nlb_backendset_backendserver(region, values_for_column_bss,NLBs, nlb_c values_for_column_bss[col_header].append(backendsets) else: oci_objs = [eachnlb,backendset_details,hc,dns] - values_for_column_bss = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_bss,values_for_column_bss) + values_for_column_bss = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_bss,values_for_column_bss) return values_for_column_bss @@ -284,14 +285,14 @@ def print_nlb_listener(region, outdir, values_for_column_lis, NLBs, nlb_compartm values_for_column_lis[col_header].append("") # Process the Tag Columns - elif col_header.lower() in commonTools.tagColumns: + elif col_header.lower() in ociCommonTools.tagColumns: if cnt_lsnr == 1: - values_for_column_lis = commonTools.export_tags(eachnlb, col_header, values_for_column_lis) + values_for_column_lis = ociCommonTools.export_tags(eachnlb, col_header, values_for_column_lis) else: values_for_column_lis[col_header].append("") else: oci_objs = [values,eachnlb,ips] - values_for_column_lis = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_lis, values_for_column_lis) + values_for_column_lis = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_lis, values_for_column_lis) if cnt_lsnr == 0: for col_header in values_for_column_lis.keys(): @@ -310,11 +311,11 @@ def print_nlb_listener(region, outdir, values_for_column_lis, NLBs, nlb_compartm elif (col_header == "Is Private(True|False)"): values_for_column_lis[col_header].append(not(is_public)) # Process the Tag Columns - elif col_header.lower() in commonTools.tagColumns: - values_for_column_lis = commonTools.export_tags(eachnlb, col_header, values_for_column_lis) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_lis = ociCommonTools.export_tags(eachnlb, col_header, values_for_column_lis) else: oci_objs = [eachnlb,ips] - values_for_column_lis = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_lis,values_for_column_lis) + values_for_column_lis = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_lis,values_for_column_lis) return values_for_column_lis diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/access-control-rules-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/access-control-rules-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/access-control-rules-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/access-control-rules-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/access-method-rules-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/access-method-rules-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/access-method-rules-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/access-method-rules-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/backend-set-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/backend-set-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/backend-set-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/backend-set-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/backends-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/backends-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/backends-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/backends-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/certificate-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/certificate-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/certificate-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/certificate-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/cipher-suite-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/cipher-suite-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/cipher-suite-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/cipher-suite-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/hostname-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/hostname-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/hostname-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/hostname-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/http-header-rules-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/http-header-rules-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/http-header-rules-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/http-header-rules-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/lb-routing-policy-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/lb-routing-policy-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/lb-routing-policy-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/lb-routing-policy-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/lbr-reserved-ips-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/lbr-reserved-ips-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/lbr-reserved-ips-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/lbr-reserved-ips-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/lbr-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/lbr-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/lbr-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/lbr-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/listener-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/listener-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/listener-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/listener-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/nlb-backend-set-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/nlb-backend-set-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/nlb-backend-set-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/nlb-backend-set-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/nlb-backends-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/nlb-backends-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/nlb-backends-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/nlb-backends-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/nlb-listener-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/nlb-listener-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/nlb-listener-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/nlb-listener-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/nlb-reserved-ips-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/nlb-reserved-ips-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/nlb-reserved-ips-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/nlb-reserved-ips-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/nlb-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/nlb-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/nlb-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/nlb-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/path-route-rules-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/path-route-rules-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/path-route-rules-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/path-route-rules-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/path-route-set-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/path-route-set-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/path-route-set-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/path-route-set-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/request-response-header-rules-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/request-response-header-rules-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/request-response-header-rules-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/request-response-header-rules-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/rule-set-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/rule-set-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/rule-set-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/rule-set-template diff --git a/cd3_automation_toolkit/Network/LoadBalancers/templates/uri-redirect-rules-template b/cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/uri-redirect-rules-template similarity index 100% rename from cd3_automation_toolkit/Network/LoadBalancers/templates/uri-redirect-rules-template rename to cd3_automation_toolkit/ocicloud/python/network/loadbalancers/templates/uri-redirect-rules-template diff --git a/cd3_automation_toolkit/commonTools.py b/cd3_automation_toolkit/ocicloud/python/ociCommonTools.py similarity index 71% rename from cd3_automation_toolkit/commonTools.py rename to cd3_automation_toolkit/ocicloud/python/ociCommonTools.py index 8f35569b4..580ed0000 100644 --- a/cd3_automation_toolkit/commonTools.py +++ b/cd3_automation_toolkit/ocicloud/python/ociCommonTools.py @@ -2,40 +2,19 @@ import os import shutil import datetime -import ssl -import sys -import pathlib -import urllib.request import oci from copy import deepcopy from oci.identity import IdentityClient from oci.config import DEFAULT_LOCATION -from openpyxl import load_workbook -from openpyxl.styles import PatternFill -from openpyxl.styles import Alignment -from openpyxl.styles import Border -from openpyxl.styles import Side -from contextlib import contextmanager -import xml.etree.ElementTree as ET -import collections +from common.python.commonTools import * import re import json as simplejson import warnings -import threading warnings.simplefilter("ignore") -def data_frame(filename,sheetname): - # Read the tab from excel, Drop null values, Reset index - df, col_headers = commonTools.read_cd3(filename, sheetname) - df = df.dropna(how='all') - df = df.reset_index(drop=True) +class ociCommonTools(): - return df - - -class commonTools(): - endNames = {'', '', ''} tagColumns = {'freeform tags', 'freeform_tags', 'defined_tags', 'defined tags', 'vnic defined tags','vnic_defined_tags', 'vnic freeform tags', 'vnic_freeform_tags', 'oke_labels', 'oke labels', 'nodepool defined tags', 'nodepool_defined_tags', 'node defined tags', @@ -111,6 +90,9 @@ def __init__(self): dir=os.getcwd() if ("OCSWorkVM" in os.getcwd() or 'user-scripts' in os.getcwd()): os.chdir("../") + dir = os.getcwd() + if("ocicloud/python" not in dir): + os.chdir("ocicloud/python") regionFileName="OCI_Regions" protocolFileName="OCI_Protocols" excelColumnName="Excel_Columns" @@ -470,7 +452,6 @@ def identity_domain_check(self,config, signer): def get_identity_domain_data(self,config, signer, resource,var_file): - config.__setitem__("region",self.region_dict[self.home_region]) selected_domains_data = {} idc = IdentityClient(config=config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY, signer=signer) @@ -500,23 +481,6 @@ def get_identity_domain_data(self,config, signer, resource,var_file): return self.domain_data - #Check value exported - #If None - replace with "" - #If list, convert to comma sepearted string - def check_exported_value(value): - if value == None: - value = "" - if ("list" in str(type(value))): - str1 = "" - if(value.__len__()==0): - value="" - for v in value: - str1 = v + "," + str1 - if (str1 != "" and str1[-1] == ','): - value = str1[:-1] - - return value - # Export Tag fields - common code - Defined and Freeform Tags # header - individual headers/column name # values_for_column - list of columns from read_cd3 function @@ -608,68 +572,13 @@ def export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column): return values_for_column - # Check CD3 Column headers - def check_column_headers(var_name): - # replace special characters and spaces with '_' and convert to lowercase - # replaces multiple occurrence of '_' to just 1 - var_name = var_name.strip() - var_name = re.sub('[@!#$%^&*<>?/}{~: \n()|-]', '_', var_name).lower() - var_name = re.sub('_+', '_', var_name).lower() - return var_name - - #Check TF variable Name - def check_tf_variable(var_name): - tfname = re.compile('[^a-zA-Z0-9_-]') - tfnamestart = re.compile('[A-Za-z]') - - var_name = tfname.sub("-", var_name) - x = tfnamestart.match(var_name) - # variable name doesnot start with letter; append with c - if (x == None): - var_name = "c" + var_name - return var_name - - # Process ColumnValues - def check_columnvalue(columnvalue): - - if str(columnvalue).lower() == 'true' or str(columnvalue).lower() == 'false': - columnvalue = str(columnvalue).lower() - - if (columnvalue.lower() == 'nan'): - columnvalue = "" - - # replace \ with \\ - if("\\" in columnvalue): - columnvalue = columnvalue.replace("\\", "\\\\") - - # replace " with \" - if("\"" in columnvalue): - columnvalue=columnvalue.replace("\"","\\\"") - - return columnvalue - - # Process column values with :: - def check_multivalues_columnvalue(columnvalue, columnname, tempdict): - columnvalue = str(columnvalue).strip() - columnname = commonTools.check_column_headers(columnname) - if "::" in columnvalue: - if ".Flex" in columnvalue or ".Micro" in columnvalue: - columnname = commonTools.check_column_headers(columnname) - multivalues = columnvalue.split("::") - multivalues = [str(part).strip() for part in multivalues if part] - tempdict = {columnname: multivalues} - elif columnname != 'Compartment Name' and "ipv6" not in columnname.lower(): - columnname = commonTools.check_column_headers(columnname) - multivalues = columnvalue.split("::") - multivalues = [str(part).strip() for part in multivalues ]#if part] - tempdict = {columnname: multivalues} - return tempdict # Split values for tagging def split_tag_values(columnname, columnvalue, tempdict): columnvalue = columnvalue.replace("\n", "") if ";" in columnvalue: # If there are more than one tag; split them by ";" and "=" + columnname = commonTools.check_column_headers(columnname) multivalues = columnvalue.split(";") multivalues = [part.split("=") for part in multivalues if part] @@ -692,208 +601,8 @@ def split_tag_values(columnname, columnvalue, tempdict): tempdict = {columnname: [multivalues]} return tempdict - # Read rows from CD3 - def read_cd3(cd3file, sheet_name): - df = {} - try: - df = pd.read_excel(cd3file, sheet_name=sheet_name, skiprows=1, dtype=object) - - except Exception as e: - if("Events" in str(e) or "Notifications" in str(e)): - print("\nTabs - \"Events\" or \"Notifications\" is missing in the CD3. Please make sure to use the correct input file for Events and Notifications in properties file...Exiting!!") - exit(1) - else: - print("Error occurred while reading the CD3 excel sheet: "+ str(e)) - exit(1) - - yield df - try: - book = load_workbook(cd3file) - sheet = book[sheet_name] - except KeyError as e: - if 'does not exist' in str(e): - print("\nTab - \""+sheet_name+"\" seems to be missing in the CD3. Please make sure to use the right CD3 in properties file.....Exiting!!") - exit(1) - except Exception as e: - print(str(e)) - print("Exiting!!") - exit(1) - - values_for_column = collections.OrderedDict() - # values_for_column={} - for j in range(0, sheet.max_column): - col_name = sheet.cell(row=2, column=j + 1).value - if (type(col_name) == str): - values_for_column[col_name] = [] - yield values_for_column - - #Write exported rows to cd3 - def write_to_cd3(values_for_column, cd3file, sheet_name,append=False): - try: - book = load_workbook(cd3file) - sheet = book[sheet_name] - except Exception as e: - print(str(e)) - print("Exiting!!") - exit(1) - if (sheet_name == "VCN Info"): - onprem_destinations = "" - ngw_destinations = "" - igw_destinations = "" - for destination in values_for_column["onprem_destinations"]: - onprem_destinations=destination+","+onprem_destinations - for destination in values_for_column["ngw_destinations"]: - ngw_destinations = destination + "," + ngw_destinations - for destination in values_for_column["igw_destinations"]: - igw_destinations = destination + "," + igw_destinations - - if (onprem_destinations != "" and onprem_destinations[-1] == ','): - onprem_destinations = onprem_destinations[:-1] - if (ngw_destinations != "" and ngw_destinations[-1] == ','): - ngw_destinations = ngw_destinations[:-1] - if (igw_destinations != "" and igw_destinations[-1] == ','): - igw_destinations = igw_destinations[:-1] - - sheet.cell(3,2).value = onprem_destinations - sheet.cell(4,2).value = ngw_destinations - sheet.cell(5,2).value = igw_destinations - try: - book.save(cd3file) - book.close() - except Exception as e: - print(str(e)) - print("Exiting!!") - exit(1) - return - - - #rows_len=len(rows) - rows_len = len(values_for_column["Region"]) - sheet_max_rows = sheet.max_row - #If no rows exported from OCI, remove the sample data as well - if(rows_len == 0) : - if not append: - print("0 rows exported; Nothing to write to CD3 excel; Tab "+sheet_name +" will be empty in CD3 excel!!") - for i in range(0, sheet.max_row): - for j in range(0, sheet.max_column): - sheet.cell(row=i + 3, column=j + 1).value = "" - try: - book.save(cd3file) - book.close() - except Exception as e: - print(str(e)) - print("Exiting!!") - exit(1) - return - - if append: - for x in range(1, sheet_max_rows): - if sheet['A'][x].value == None: - last_line = x - break - #rows_len +=last_line - large = rows_len - start = last_line+1 - - - else: - start = 3 - if (rows_len > sheet_max_rows): - large = rows_len - else: - large = sheet_max_rows - - df, values_for_column_sheet = commonTools.read_cd3(cd3file, sheet_name) - - #Put Data - j=0 - for i in range(0,large): - for col_name in values_for_column.keys(): - #Check if column name to be populated in present in the sheet. - if col_name not in values_for_column_sheet: - continue - # Data - if(i>=rows_len): - sheet.cell(row=i+start, column=j+1).value = "" - else: - sheet.cell(row=i+start, column=j+1).value = values_for_column[col_name][i] - sheet.cell(row=i+start, column=j+1).alignment = Alignment(wrap_text=True) - j=j+1 - j=0 - - - brdr = Border(left=Side(style='thin'), - right=Side(style='thin'), - top=Side(style='thin'), - bottom=Side(style='thin'), - ) - - for row in sheet.iter_rows(min_row=3): - for cell in row: - cell.border = brdr - - # Add color for exported sec rules and route rules - if (sheet_name == "RouteRulesinOCI" or sheet_name == "SecRulesinOCI" or sheet_name == "DRGRouteRulesinOCI"): - names = [] - # Add color coding to exported rules - for row in sheet.iter_rows(min_row=3): - c = 0 - region = "" - name = "" - for cell in row: - c = c + 1 - if (c == 1): - region = cell.value - continue - elif (c == 4): - name = cell.value - break - - vcn_name = region + "_" + name - if (vcn_name not in names): - names.append(vcn_name) - for cellnew in row: - if (len(names) % 2 == 0): - cellnew.fill = PatternFill(start_color="94AFAF", end_color="94AFAF", fill_type="solid") - cellnew.border = brdr - else: - cellnew.fill = PatternFill(start_color="E5DBBE", end_color="E5DBBE", fill_type="solid") - cellnew.border = brdr - else: - for cellnew in row: - if (len(names) % 2 == 0): - cellnew.fill = PatternFill(start_color="94AFAF", end_color="94AFAF", fill_type="solid") - cellnew.border = brdr - else: - cellnew.fill = PatternFill(start_color="E5DBBE", end_color="E5DBBE", fill_type="solid") - cellnew.border = brdr - try: - book.save(cd3file) - book.close() - except Exception as e: - print(str(e)) - print("Exiting!!") - exit(1) # def backup_file(src_dir, pattern, overwrite): - def backup_file(src_dir, resource, pattern): - dest_dir = str(src_dir) + "/backup_" + resource + "/" + datetime.datetime.now().strftime("%d-%m-%H%M%S").replace('/', '-') - for f in os.listdir(str(src_dir)): - if f.endswith(pattern): - print("Backing up existing " + f + " to " + dest_dir) - if not os.path.exists(dest_dir): - # print("\nCreating backup dir " + dest_dir + "\n") - os.makedirs(dest_dir) - - src = os.path.join(str(src_dir), f) - #dest = os.path.join(dest_dir, f) - # print("backing up ....." + src +" to "+dest) - shutil.move(src, dest_dir) - """if (overwrite == 'yes'): - shutil.move(src, dest_dir) - elif (overwrite == 'no'): - shutil.copyfile(src, dest) - """ # def backup_file(src_dir, pattern, overwrite): def copy_file(src_dir, resource, pattern): @@ -946,7 +655,7 @@ def get_protocol(strprotocol): if str(strprotocol).lower() == "all": return "all" else: - protocol_dict = commonTools().protocol_dict + protocol_dict = ociCommonTools().protocol_dict for k, v in protocol_dict.items(): if (strprotocol).lower() == v.lower(): return k @@ -970,7 +679,7 @@ def create_ingress_rule_string(self, secrule, tempStr, ingress_rule, tempdict2, tempdict2 = {'source_type' : source_type} tempStr.update(tempdict2) - tempdict2 = {'protocol_code': commonTools.get_protocol(protocol),'isstateless':isstateless} + tempdict2 = {'protocol_code': ociCommonTools.get_protocol(protocol),'isstateless':isstateless} tempStr.update(tempdict2) ingress_rule = ingress_rule + secrule.render(tempStr) @@ -997,7 +706,7 @@ def create_egress_rule_string(self, secrule, tempStr, egress_rule, tempdict2, re tempdict2 = {'destination_type': destination_type} tempStr.update(tempdict2) - tempDict2 = {'protocol_code': commonTools.get_protocol(protocol),'isstateless':isstateless} + tempDict2 = {'protocol_code': ociCommonTools.get_protocol(protocol),'isstateless':isstateless} tempStr.update(tempDict2) egress_rule = egress_rule + secrule.render(tempStr) @@ -1213,20 +922,6 @@ def __init__(self, filename): v = v[:-1] self.peering_dict[k] = v -@contextmanager -def section(title='', header=False, padding=117): - separator = '-' if not header else '=' - # Not sure why 117 but thats how it was before. - print(f'{title:{separator}^{padding}}') - yield - if header: - print(separator * padding) - - -def exit_menu(msg, exit_code=1): - print(msg) - exit(exit_code) - class parseVCNInfo(): # all_regions = [] @@ -1294,132 +989,3 @@ def __init__(self, filename): value = df_subnet.loc[i,'Compartment Name'].strip(), df_subnet.loc[i,'VCN Name'].strip(), df_subnet.loc[i,'Display Name'].strip() self.vcn_subnet_map[key] = value -class cd3Services(): - - - #Get OCI Cloud Regions - regions_list = "" - def fetch_regions(self,config,signer): - #config = oci.config.from_file(file_location=configFileName) - idc = IdentityClient(config=config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY,signer=signer) - try: - regions_list = idc.list_regions().data - except Exception as e: - print(e) - if ('NotAuthenticated' in str(e)): - print("\nInvalid Credetials - check your keypair/fingerprint/region...Exiting!!!") - exit(1) - - if ("OCSWorkVM" in os.getcwd() or 'user-scripts' in os.getcwd()): - os.chdir("../") - - tempStr = '#Region:Region_Key\n' - reg_dict = {} - - for reg in regions_list: - cd3key = str(reg.name.split('-',1)[1]).lower() - - if 'dcc' in cd3key: - cd3key = str(cd3key.split('-',1)[1]).lower() - - name = str(reg.name).lower() - reg_dict[cd3key] = name - - keys = [] - new_reg_dict={} - for key,val in reg_dict.items(): - keyy = key.split("-")[0] - if keyy not in keys: - keys.append(keyy) - new_reg_dict[keyy]=val - else: - new_reg_dict[key] = val - - #replace prev - old_val = new_reg_dict[keyy] - old_val_key = str(old_val.split('-', 1)[1]).lower() - if 'dcc' in old_val_key: - old_val_key = str(old_val_key.split('-', 1)[1]).lower() - - new_reg_dict[old_val_key] = old_val - new_reg_dict.pop(keyy) - - for cd3key,name in new_reg_dict.items(): - line = cd3key + ":" + name - tempStr = tempStr + line + '\n' - - with open('OCI_Regions', 'w+') as f: - f.write(tempStr) - f.close() - print("Updated OCI_Regions file !!!\n") - - # Parse XML - Used by OCI Protocols - def parse_xml(source: str) -> ET.Element: - it = ET.iterparse(open(source)) - # strip namespaces - for _, el in it: - if "}" in el.tag: - el.tag = el.tag.split("}", 1)[1] - root = it.root # mypy: ignore - return root - - # Parse Date - Used by OCI Protocols - def parse_date(root_xml: ET.Element) -> datetime: - updated = root_xml.find("updated") - assert updated is not None and isinstance(updated.text, str) - return datetime.datetime.strptime(updated.text, "%Y-%m-%d") - - # write_protocols_file - Used for OCI Protocols - def write_protocols_file(source: str, destination: str) -> datetime: - root = cd3Services.parse_xml(source) - updated = cd3Services.parse_date(root) - destination = str(pathlib.Path.cwd())+"/"+destination - with open(destination,"w+") as dst: - dst.write("#protocol number:protocol name\n") - for r in root.iter("record"): - desc_ = r.find("description") - if desc_ is None or desc_.text is None: - desc = "" - else: - desc = desc_.text - name_ = r.find("name") - value_ = r.find("value") - if (value_ is None - or value_.text is None): - continue - if (name_ is None - or name_.text is None): - name = desc - dst.write(str(value_.text) + ":" + name + "\n") - continue - alias = name_.text.split()[0] - value = int(value_.text) - dst.write(str(value)+":"+alias+"\n") - return updated - - def download(url: str, path: str) -> None: - with open(path, "wb") as dst, urllib.request.urlopen(url) as src: - shutil.copyfileobj(src, dst) - - #Get OCI Protocols - def fetch_protocols(self) -> None: - - ssl._create_default_https_context = ssl._create_unverified_context - PROTOCOLS_URL = "https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xml" - PROTOCOLS_XML = "protocol-numbers.xml" - PROTOCOLS_FILE = "OCI_Protocols" - - ##### main code for oci protocols #### - protocols_xml = str(pathlib.Path.cwd()) + "/" + PROTOCOLS_XML - try: - cd3Services.download(PROTOCOLS_URL, protocols_xml) - except OSError as e: - print("Could not download iana service names and port numbers: {}".format(e), - file=sys.stderr, - ) - sys.exit(1) - cd3Services.write_protocols_file(protocols_xml, PROTOCOLS_FILE) - rem_file = pathlib.Path(protocols_xml) - rem_file.unlink() - - print("Updated OCI_Protocols file !!!\n") \ No newline at end of file diff --git a/cd3_automation_toolkit/SDDC/__init__.py b/cd3_automation_toolkit/ocicloud/python/sddc/__init__.py similarity index 100% rename from cd3_automation_toolkit/SDDC/__init__.py rename to cd3_automation_toolkit/ocicloud/python/sddc/__init__.py diff --git a/cd3_automation_toolkit/SDDC/create_terraform_sddc.py b/cd3_automation_toolkit/ocicloud/python/sddc/create_terraform_sddc.py similarity index 96% rename from cd3_automation_toolkit/SDDC/create_terraform_sddc.py rename to cd3_automation_toolkit/ocicloud/python/sddc/create_terraform_sddc.py index 456f5dbe6..d1f65eaf1 100755 --- a/cd3_automation_toolkit/SDDC/create_terraform_sddc.py +++ b/cd3_automation_toolkit/ocicloud/python/sddc/create_terraform_sddc.py @@ -3,9 +3,11 @@ # This script will produce a Terraform file that will be used to set up OCI core components # SDDC management cluster and workload clusters from pathlib import Path -import os -from commonTools import * +import os, sys from jinja2 import Environment, FileSystemLoader +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools # Function to pre-process the data def data_process(filename, sheetName): @@ -80,8 +82,8 @@ def create_terraform_sddc_cluster(inputfile, outdir, service_dir, prefix, ct, sd columnvalue = commonTools.check_columnvalue(str(df[columnname][i]).strip()) # Check for multivalued columns tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == 'SDDC Name': sddc_name = columnvalue.strip() sddc_display_name,sddc_cluster_display_name = sddc_name.split("::") diff --git a/cd3_automation_toolkit/SDDC/export_sddc_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/sddc/export_sddc_nonGreenField.py similarity index 97% rename from cd3_automation_toolkit/SDDC/export_sddc_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/sddc/export_sddc_nonGreenField.py index 034a9d9ff..3b88fdb4f 100644 --- a/cd3_automation_toolkit/SDDC/export_sddc_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/sddc/export_sddc_nonGreenField.py @@ -11,8 +11,9 @@ from oci.core.blockstorage_client import BlockstorageClient import os import subprocess as sp -sys.path.append(os.getcwd() + "/..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools def get_volume_data(bvol, volume_id, ct): @@ -241,15 +242,15 @@ def export_sddc(inputfile, outdir, service_dir,config,signer, ct, export_compart elif col_header == "Provisioning Net VLAN": vlan_id = sddc_network.provisioning_vlan_id values_for_column_sddc[col_header].append(vnc.get_vlan(vlan_id).data.display_name) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_sddc = commonTools.export_tags( + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_sddc = ociCommonTools.export_tags( sddc if sddc_cluster.vsphere_type == "MANAGEMENT" else sddc_cluster, col_header, values_for_column_sddc) else: oci_objs = [sddc, sddc_init_config, sddc_network, sddc_datastores] if sddc_cluster.vsphere_type == "MANAGEMENT" else [ sddc_cluster, sddc_cluster_data, sddc_network, sddc_datastores] - values_for_column_sddc = commonTools.export_extra_columns(oci_objs, col_header, + values_for_column_sddc = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_sddc, values_for_column_sddc) diff --git a/cd3_automation_toolkit/SDDC/templates/sddc-cluster-template b/cd3_automation_toolkit/ocicloud/python/sddc/templates/sddc-cluster-template similarity index 100% rename from cd3_automation_toolkit/SDDC/templates/sddc-cluster-template rename to cd3_automation_toolkit/ocicloud/python/sddc/templates/sddc-cluster-template diff --git a/cd3_automation_toolkit/SDDC/templates/sddc-template b/cd3_automation_toolkit/ocicloud/python/sddc/templates/sddc-template similarity index 100% rename from cd3_automation_toolkit/SDDC/templates/sddc-template rename to cd3_automation_toolkit/ocicloud/python/sddc/templates/sddc-template diff --git a/cd3_automation_toolkit/ocicloud/python/security/__init__.py b/cd3_automation_toolkit/ocicloud/python/security/__init__.py new file mode 100644 index 000000000..43ce62703 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/python/security/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from .cloudguard import * +from .keyvault import * +from .firewall import * + diff --git a/cd3_automation_toolkit/Security/CloudGuard/__init__.py b/cd3_automation_toolkit/ocicloud/python/security/cloudguard/__init__.py similarity index 100% rename from cd3_automation_toolkit/Security/CloudGuard/__init__.py rename to cd3_automation_toolkit/ocicloud/python/security/cloudguard/__init__.py diff --git a/cd3_automation_toolkit/Security/CloudGuard/enable_terraform_cloudguard.py b/cd3_automation_toolkit/ocicloud/python/security/cloudguard/enable_terraform_cloudguard.py similarity index 95% rename from cd3_automation_toolkit/Security/CloudGuard/enable_terraform_cloudguard.py rename to cd3_automation_toolkit/ocicloud/python/security/cloudguard/enable_terraform_cloudguard.py index 4e3c20f8a..01cb4f089 100644 --- a/cd3_automation_toolkit/Security/CloudGuard/enable_terraform_cloudguard.py +++ b/cd3_automation_toolkit/ocicloud/python/security/cloudguard/enable_terraform_cloudguard.py @@ -8,11 +8,13 @@ # Oracle Consulting # Modified (TF Upgrade): Shruthi Subramanian # -import os +import os, sys from jinja2 import Environment, FileSystemLoader from pathlib import Path from oci.config import DEFAULT_LOCATION -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs- Config file, prefix AND outdir diff --git a/cd3_automation_toolkit/Security/CloudGuard/templates/cloud-guard-config-template b/cd3_automation_toolkit/ocicloud/python/security/cloudguard/templates/cloud-guard-config-template similarity index 100% rename from cd3_automation_toolkit/Security/CloudGuard/templates/cloud-guard-config-template rename to cd3_automation_toolkit/ocicloud/python/security/cloudguard/templates/cloud-guard-config-template diff --git a/cd3_automation_toolkit/Security/CloudGuard/templates/cloud-guard-target-template b/cd3_automation_toolkit/ocicloud/python/security/cloudguard/templates/cloud-guard-target-template similarity index 100% rename from cd3_automation_toolkit/Security/CloudGuard/templates/cloud-guard-target-template rename to cd3_automation_toolkit/ocicloud/python/security/cloudguard/templates/cloud-guard-target-template diff --git a/cd3_automation_toolkit/Security/Firewall/__init__.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/__init__.py similarity index 94% rename from cd3_automation_toolkit/Security/Firewall/__init__.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/__init__.py index 8c79bf942..d4b63f67f 100644 --- a/cd3_automation_toolkit/Security/Firewall/__init__.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/__init__.py @@ -16,3 +16,4 @@ from .cloneexport_firewallpolicy_nonGreenField import cloneexport_firewallpolicy from .delete_firewallpolicy import delete_firewallpolicy from .fwpolicy_create_tunnelinspection import fwpolicy_create_tunnelinspect +from .fwpolicy_create_natrules import fwpolicy_create_natrules diff --git a/cd3_automation_toolkit/Security/Firewall/clone_firewallpolicy.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/clone_firewallpolicy.py similarity index 97% rename from cd3_automation_toolkit/Security/Firewall/clone_firewallpolicy.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/clone_firewallpolicy.py index 12bb64253..94ddeb964 100644 --- a/cd3_automation_toolkit/Security/Firewall/clone_firewallpolicy.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/clone_firewallpolicy.py @@ -11,15 +11,15 @@ import oci import os import datetime -import Security import numpy as np from oci.network_firewall import NetworkFirewallClient from oci.vault import VaultsClient from oci.identity import IdentityClient from oci.network_load_balancer import NetworkLoadBalancerClient -sys.path.append(os.getcwd() + "/..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools importCommands = {} oci_obj_names = {} diff --git a/cd3_automation_toolkit/Security/Firewall/cloneexport_firewallpolicy_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/cloneexport_firewallpolicy_nonGreenField.py similarity index 95% rename from cd3_automation_toolkit/Security/Firewall/cloneexport_firewallpolicy_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/cloneexport_firewallpolicy_nonGreenField.py index f6ea2ec02..acda5a345 100644 --- a/cd3_automation_toolkit/Security/Firewall/cloneexport_firewallpolicy_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/cloneexport_firewallpolicy_nonGreenField.py @@ -17,8 +17,9 @@ from oci.key_management import KmsVaultClient from oci.network_load_balancer import NetworkLoadBalancerClient -sys.path.append(os.getcwd() + "/..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools importCommands = {} oci_obj_names = {} @@ -43,8 +44,8 @@ def print_firewall_policy(region, ct, values_for_column_fwpolicy, fwpolicies, fw elif col_header == 'Policy Name': values_for_column_fwpolicy[col_header].append(fwpolicy_display_name) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwpolicy = commonTools.export_tags(eachfwpolicy, col_header, values_for_column_fwpolicy) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwpolicy = ociCommonTools.export_tags(eachfwpolicy, col_header, values_for_column_fwpolicy) return values_for_column_fwpolicy @@ -85,8 +86,8 @@ def print_firewall_address(region, ct, values_for_column_fwaddress, fwpolicies, values_for_column_fwaddress[col_header].append(add.type) elif col_header == 'Address List': values_for_column_fwaddress[col_header].append(address_detail) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwaddress = commonTools.export_tags(policy, col_header, values_for_column_fwaddress) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwaddress = ociCommonTools.export_tags(policy, col_header, values_for_column_fwaddress) return values_for_column_fwaddress @@ -122,8 +123,8 @@ def print_firewall_urllist(region, ct, values_for_column_fwurllist, fwpolicies, values_for_column_fwurllist[col_header].append(url_display_name) elif col_header == 'URL List': values_for_column_fwurllist[col_header].append(url_detail) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwurllist = commonTools.export_tags(urlpolicy, col_header,values_for_column_fwurllist) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwurllist = ociCommonTools.export_tags(urlpolicy, col_header,values_for_column_fwurllist) return values_for_column_fwurllist @@ -173,8 +174,8 @@ def print_firewall_servicelist(region, ct, values_for_column_fwservicelist, fwpo values_for_column_fwservicelist[col_header].append(service_display_name) elif col_header == 'Services': values_for_column_fwservicelist[col_header].append(service_detail) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwservicelist = commonTools.export_tags(servicelistpolicy, col_header,values_for_column_fwservicelist) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwservicelist = ociCommonTools.export_tags(servicelistpolicy, col_header,values_for_column_fwservicelist) return values_for_column_fwservicelist @@ -220,8 +221,8 @@ def print_firewall_applist(region, ct, values_for_column_fwapplist, fwpolicies, values_for_column_fwapplist[col_header].append(application_display_name) elif col_header == 'Applications': values_for_column_fwapplist[col_header].append(application_detail) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwapplist = commonTools.export_tags(applistpolicy, col_header,values_for_column_fwapplist) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwapplist = ociCommonTools.export_tags(applistpolicy, col_header,values_for_column_fwapplist) return values_for_column_fwapplist @@ -309,8 +310,8 @@ def print_firewall_secrules(region, ct, values_for_column_fwsecrules, fwpolicies values_for_column_fwsecrules[col_header].append(raction) elif col_header == 'Position': values_for_column_fwsecrules[col_header].append(rposition) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwsecrules = commonTools.export_tags(applistpolicy, col_header,values_for_column_fwsecrules) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwsecrules = ociCommonTools.export_tags(applistpolicy, col_header,values_for_column_fwsecrules) return values_for_column_fwsecrules def print_firewall_secret(region, ct, values_for_column_fwsecret, fwpolicies, fwpolicy, vault, compartment, policy_detail,kmsvault): @@ -350,8 +351,8 @@ def print_firewall_secret(region, ct, values_for_column_fwsecret, fwpolicies, fw values_for_column_fwsecret[col_header].append(vault_secret_name) elif col_header == 'Version Number': values_for_column_fwsecret[col_header].append(key_info.version_number) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwsecret = commonTools.export_tags(secretpolicy, col_header,values_for_column_fwsecret) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwsecret = ociCommonTools.export_tags(secretpolicy, col_header,values_for_column_fwsecret) return values_for_column_fwsecret @@ -413,8 +414,8 @@ def print_firewall_decryptprofile(region, ct, values_for_column_fwdecryptprofile elif col_header == 'is untrusted issuer blocked': values_for_column_fwdecryptprofile[col_header].append(key_info1_is_untrusted_issuer_blocked) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwdecryptprofile = commonTools.export_tags(decryptionprofile, col_header,values_for_column_fwdecryptprofile) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwdecryptprofile = ociCommonTools.export_tags(decryptionprofile, col_header,values_for_column_fwdecryptprofile) return values_for_column_fwdecryptprofile def print_firewall_decryptrule(region, ct, values_for_column_fwdecryptrule, fwpolicies, fwpolicy, policy_detail): @@ -473,8 +474,8 @@ def print_firewall_decryptrule(region, ct, values_for_column_fwdecryptrule, fwpo values_for_column_fwdecryptrule[col_header].append(drule_info.action) elif col_header == 'Position': values_for_column_fwdecryptrule[col_header].append(dposition) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwdecryptrule = commonTools.export_tags(applistpolicy, col_header,values_for_column_fwdecryptrule) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwdecryptrule = ociCommonTools.export_tags(applistpolicy, col_header,values_for_column_fwdecryptrule) return values_for_column_fwdecryptrule # Execution of the code begins here def cloneexport_firewallpolicy(inputfile, _outdir, service_dir, config, signer, ct, export_compartments, export_regions, policy_detail): diff --git a/cd3_automation_toolkit/Security/Firewall/delete_firewallpolicy.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/delete_firewallpolicy.py similarity index 95% rename from cd3_automation_toolkit/Security/Firewall/delete_firewallpolicy.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/delete_firewallpolicy.py index c055756cb..ec828f7db 100644 --- a/cd3_automation_toolkit/Security/Firewall/delete_firewallpolicy.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/delete_firewallpolicy.py @@ -10,10 +10,10 @@ import sys import oci import os -import Security from oci.network_firewall import NetworkFirewallClient -sys.path.append(os.getcwd() + "/..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools importCommands = {} oci_obj_names = {} diff --git a/cd3_automation_toolkit/Security/Firewall/export_firewall_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/export_firewall_nonGreenField.py similarity index 92% rename from cd3_automation_toolkit/Security/Firewall/export_firewall_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/export_firewall_nonGreenField.py index cc40abb32..336a9b9cf 100644 --- a/cd3_automation_toolkit/Security/Firewall/export_firewall_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/export_firewall_nonGreenField.py @@ -15,8 +15,9 @@ from oci.network_firewall import NetworkFirewallClient from oci.core.virtual_network_client import VirtualNetworkClient -sys.path.append(os.getcwd() + "/..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools importCommands = {} oci_obj_names = {} @@ -60,7 +61,11 @@ def print_firewall(region, export_tags, ct, values_for_column_fw, fws, fw_compar net_cmp_name = comp_name comp_done_ids.append(net_cmp_id) subnet_detail = vcn_name + "::" + subnet_name + #if eachfw.nat_configuration == null: + enable_private_nat ="" + if eachfw.nat_configuration is not None: + enable_private_nat = eachfw.nat_configuration.must_enable_private_nat # Fetch policy policy_ocid = eachfw.network_firewall_policy_id policy_info = fw.get_network_firewall_policy(policy_ocid).data @@ -105,6 +110,9 @@ def print_firewall(region, export_tags, ct, values_for_column_fw, fws, fw_compar elif col_header == 'NSGs': values_for_column_fw[col_header].append(nsg_detail) + elif col_header == 'Enable Private NAT(True/False)': + values_for_column_fw[col_header].append(enable_private_nat) + elif col_header == 'Availability Domain(AD1|AD2|AD3|Regional)': if (eachfw.availability_domain == None): value_of_ad = "Regional" @@ -112,8 +120,8 @@ def print_firewall(region, export_tags, ct, values_for_column_fw, fws, fw_compar value_of_ad = AD(eachfw.availability_domain) values_for_column_fw[col_header].append(value_of_ad) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fw = commonTools.export_tags(eachfw, col_header, values_for_column_fw) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fw = ociCommonTools.export_tags(eachfw, col_header, values_for_column_fw) return values_for_column_fw diff --git a/cd3_automation_toolkit/Security/Firewall/export_firewallpolicy_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/export_firewallpolicy_nonGreenField.py similarity index 87% rename from cd3_automation_toolkit/Security/Firewall/export_firewallpolicy_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/export_firewallpolicy_nonGreenField.py index 8d0c226b5..ae71be332 100644 --- a/cd3_automation_toolkit/Security/Firewall/export_firewallpolicy_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/export_firewallpolicy_nonGreenField.py @@ -19,10 +19,11 @@ import subprocess as sp from oci.network_load_balancer import NetworkLoadBalancerClient -sys.path.append(os.getcwd() + "/..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools -importCommands,importCommands_nfp,importCommands_nfao,importCommands_ulo,importCommands_slo,importCommands_alo,importCommands_sro,importCommands_mso,importCommands_dpo,importCommands_dro,importCommands_fpo,importCommands_tio = {},{},{},{},{},{},{},{},{},{},{},{} +importCommands,importCommands_nfp,importCommands_nfao,importCommands_ulo,importCommands_slo,importCommands_alo,importCommands_sro,importCommands_mso,importCommands_dpo,importCommands_dro,importCommands_fpo,importCommands_tio,importCommands_nato = {},{},{},{},{},{},{},{},{},{},{},{},{} oci_obj_names = {} @@ -65,8 +66,8 @@ def print_firewall_policy(region, ct, values_for_column_fwpolicy, fwpolicies, fw elif col_header == 'Policy Name': values_for_column_fwpolicy[col_header].append(fwpolicy_display_name) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwpolicy = commonTools.export_tags(eachfwpolicy, col_header, values_for_column_fwpolicy) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwpolicy = ociCommonTools.export_tags(eachfwpolicy, col_header, values_for_column_fwpolicy) return values_for_column_fwpolicy @@ -126,8 +127,8 @@ def print_firewall_address(region, ct, values_for_column_fwaddress, fwpolicies, values_for_column_fwaddress[col_header].append(add.type) elif col_header == 'Address List': values_for_column_fwaddress[col_header].append(address_detail) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwaddress = commonTools.export_tags(policy, col_header, values_for_column_fwaddress) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwaddress = ociCommonTools.export_tags(policy, col_header, values_for_column_fwaddress) return values_for_column_fwaddress @@ -182,8 +183,8 @@ def print_firewall_urllist(region, ct, values_for_column_fwurllist, fwpolicies, values_for_column_fwurllist[col_header].append(url_display_name) elif col_header == 'URL List': values_for_column_fwurllist[col_header].append(url_detail) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwurllist = commonTools.export_tags(urlpolicy, col_header,values_for_column_fwurllist) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwurllist = ociCommonTools.export_tags(urlpolicy, col_header,values_for_column_fwurllist) return values_for_column_fwurllist @@ -259,8 +260,8 @@ def print_firewall_servicelist(region, ct, values_for_column_fwservicelist, fwpo values_for_column_fwservicelist[col_header].append(service_display_name) elif col_header == 'Services': values_for_column_fwservicelist[col_header].append(service_detail) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwservicelist = commonTools.export_tags(servicelistpolicy, col_header,values_for_column_fwservicelist) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwservicelist = ociCommonTools.export_tags(servicelistpolicy, col_header,values_for_column_fwservicelist) ## Fetch services without Lists fwservices = oci.pagination.list_call_get_all_results(fwclient.list_services,servicelistpolicy_id) @@ -299,8 +300,8 @@ def print_firewall_servicelist(region, ct, values_for_column_fwservicelist, fwpo values_for_column_fwservicelist[col_header].append("") elif col_header == 'Services': values_for_column_fwservicelist[col_header].append(service_detail) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwservicelist = commonTools.export_tags(servicelistpolicy, col_header,values_for_column_fwservicelist) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwservicelist = ociCommonTools.export_tags(servicelistpolicy, col_header,values_for_column_fwservicelist) return values_for_column_fwservicelist @@ -374,8 +375,8 @@ def print_firewall_applist(region, ct, values_for_column_fwapplist, fwpolicies, values_for_column_fwapplist[col_header].append(application_display_name) elif col_header == 'Applications': values_for_column_fwapplist[col_header].append(application_detail) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwapplist = commonTools.export_tags(applistpolicy, col_header,values_for_column_fwapplist) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwapplist = ociCommonTools.export_tags(applistpolicy, col_header,values_for_column_fwapplist) ## Fetch apps without Lists fwapps = oci.pagination.list_call_get_all_results(fwclient.list_applications, applistpolicy_id) @@ -411,8 +412,8 @@ def print_firewall_applist(region, ct, values_for_column_fwapplist, fwpolicies, values_for_column_fwapplist[col_header].append("") elif col_header == 'Applications': values_for_column_fwapplist[col_header].append(application_detail) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwapplist = commonTools.export_tags(applistpolicy, col_header, + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwapplist = ociCommonTools.export_tags(applistpolicy, col_header, values_for_column_fwapplist) return values_for_column_fwapplist @@ -522,8 +523,8 @@ def print_firewall_secrules(region, ct, values_for_column_fwsecrules, fwpolicies values_for_column_fwsecrules[col_header].append(raction) elif col_header == 'Position': values_for_column_fwsecrules[col_header].append(rposition) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwsecrules = commonTools.export_tags(secrulespolicy, col_header,values_for_column_fwsecrules) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwsecrules = ociCommonTools.export_tags(secrulespolicy, col_header,values_for_column_fwsecrules) return values_for_column_fwsecrules def print_firewall_secret(region, ct, values_for_column_fwsecret, fwpolicies, fwclient, vault, compartment, export_tags,kmsvault,state): @@ -584,8 +585,8 @@ def print_firewall_secret(region, ct, values_for_column_fwsecret, fwpolicies, fw values_for_column_fwsecret[col_header].append(vault_secret_name) elif col_header == 'Version Number': values_for_column_fwsecret[col_header].append(key_info.version_number) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwsecret = commonTools.export_tags(secretpolicy, col_header,values_for_column_fwsecret) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwsecret = ociCommonTools.export_tags(secretpolicy, col_header,values_for_column_fwsecret) return values_for_column_fwsecret @@ -668,8 +669,8 @@ def print_firewall_decryptprofile(region, ct, values_for_column_fwdecryptprofile elif col_header == 'is untrusted issuer blocked': values_for_column_fwdecryptprofile[col_header].append(key_info1_is_untrusted_issuer_blocked) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwdecryptprofile = commonTools.export_tags(decryptionprofile, col_header,values_for_column_fwdecryptprofile) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwdecryptprofile = ociCommonTools.export_tags(decryptionprofile, col_header,values_for_column_fwdecryptprofile) return values_for_column_fwdecryptprofile def print_firewall_decryptrule(region, ct, values_for_column_fwdecryptrule, fwpolicies, fwclient,export_tags,state): @@ -749,8 +750,8 @@ def print_firewall_decryptrule(region, ct, values_for_column_fwdecryptrule, fwpo values_for_column_fwdecryptrule[col_header].append(drule_info.action) elif col_header == 'Position': values_for_column_fwdecryptrule[col_header].append(dposition) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwdecryptrule = commonTools.export_tags(decryptrulepolicy, col_header,values_for_column_fwdecryptrule) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwdecryptrule = ociCommonTools.export_tags(decryptrulepolicy, col_header,values_for_column_fwdecryptrule) return values_for_column_fwdecryptrule def print_firewall_tunnelinspect(region, ct, values_for_column_fwtunnelinspect, fwpolicies, fwclient,export_tags,state): @@ -827,17 +828,102 @@ def print_firewall_tunnelinspect(region, ct, values_for_column_fwtunnelinspect, values_for_column_fwtunnelinspect[col_header].append(tirule_info.action) elif col_header == 'Position': values_for_column_fwtunnelinspect[col_header].append(dposition) - elif col_header.lower() in commonTools.tagColumns: - values_for_column_fwtunnelinspect = commonTools.export_tags(tunnelinspectrulepolicy, col_header,values_for_column_fwtunnelinspect) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwtunnelinspect = ociCommonTools.export_tags(tunnelinspectrulepolicy, col_header,values_for_column_fwtunnelinspect) return values_for_column_fwtunnelinspect +def print_firewall_nat(region, ct, values_for_column_fwnat, fwpolicies, fwclient, export_tags, state): + if not clone: + print("Exporting Nat rules details " + region) + for natrulepolicy in fwpolicies: + + # Tags filter + defined_tags = natrulepolicy.defined_tags + tags_list = [] + if defined_tags: + for tkey, tval in defined_tags.items(): + for kk, vv in tval.items(): + tag = tkey + "." + kk + "=" + vv + tags_list.append(tag) + + if export_tags == []: + check = True + else: + check = any(e in tags_list for e in export_tags) + # None of Tags from export_tags exist on this instance; Dont export this instance + if check == False: + continue + + natrulepolicy_id = natrulepolicy.id + natrulepolicy_display_name = natrulepolicy.display_name + if clone: + natrulepolicy_display_name = target_pol[src_pol.index(natrulepolicy_display_name)] + natrulepolicy_tf_name = commonTools.check_tf_variable(natrulepolicy_display_name) + fwnatrules = oci.pagination.list_call_get_all_results(fwclient.list_nat_rules, natrulepolicy_id) + natrule_info = fwnatrules.data + for nirules in natrule_info: + nirule_info = fwclient.get_nat_rule(nirules.parent_resource_id, nirules.name).data + nirules_display_name = nirules.name + nirules_tf_name = commonTools.check_tf_variable(nirules_display_name) + tf_resource = f'module.nat_rules[\\"{str(natrulepolicy_tf_name)}_{str(nirules_tf_name)}\\"].oci_network_firewall_network_firewall_policy_nat_rule.network_firewall_policy_nat_rule' + if not clone and tf_resource not in state["resources"]: + importCommands_nato[reg] += f'\n{tf_or_tofu} import "{tf_resource}" networkFirewallPolicies/{natrulepolicy_id}/natRules/{nirules_display_name}' + + rsrc_detail = "" + rdst_detail = "" + if nirule_info.condition.source_address != None: + for rsrc in nirule_info.condition.source_address: + rsrc_detail = rsrc_detail + "," + rsrc + if (rsrc_detail != ""): + rsrc_detail = rsrc_detail[1:] + if nirule_info.condition.destination_address != None: + for rdst in nirule_info.condition.destination_address: + rdst_detail = rdst_detail + "," + rdst + if (rdst_detail != ""): + rdst_detail = rdst_detail[1:] + + if nirule_info.position.after_rule == None and nirule_info.position.before_rule == None: + dposition = None + elif nirule_info.position.after_rule == None: + dposition = None + elif nirule_info.position.before_rule == None: + dposition = "after_rule::" + nirule_info.position.after_rule + else: + dposition = "after_rule::" + nirule_info.position.after_rule + + for col_header in values_for_column_fwnat: + if col_header == 'Region': + values_for_column_fwnat[col_header].append(region) + elif col_header == 'Firewall Policy': + values_for_column_fwnat[col_header].append(natrulepolicy_display_name) + elif col_header == 'Rule Name': + values_for_column_fwnat[col_header].append(nirules_display_name) + elif col_header == 'Source Address': + values_for_column_fwnat[col_header].append(rsrc_detail) + elif col_header == 'Destination Address': + values_for_column_fwnat[col_header].append(rdst_detail) + elif col_header == 'Action': + values_for_column_fwnat[col_header].append(nirule_info.action) + elif col_header == 'Type': + values_for_column_fwnat[col_header].append(nirule_info.type) + elif col_header == 'Service': + values_for_column_fwnat[col_header].append(nirule_info.condition.service) + elif col_header == 'Description': + values_for_column_fwnat[col_header].append(nirule_info.description) + elif col_header == 'Position': + values_for_column_fwnat[col_header].append(dposition) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column_fwnat = ociCommonTools.export_tags(natrulepolicy, col_header, + values_for_column_fwnat) + return values_for_column_fwnat + # Execution of the code begins here -def export_firewallpolicy(inputfile, _outdir, service_dir, config, signer, ct, export_compartments, export_regions, export_tags, export_policies,target_policies=[],attached_policy_only="",clone_policy=False): +def export_firewallpolicy(inputfile, _outdir, service_dir, config, signer, ct, export_compartments, export_regions=[], export_tags=[], export_policies=[],target_policies=[],attached_policy_only="",clone_policy=False): global tf_import_cmd global sheet_dict - global importCommands,importCommands_nfp,importCommands_nfao,importCommands_ulo,importCommands_slo,importCommands_alo,importCommands_sro,importCommands_mso,importCommands_dpo,importCommands_dro,importCommands_fpo,importCommands_tio + global importCommands,importCommands_nfp,importCommands_nfao,importCommands_ulo,importCommands_slo,importCommands_alo,importCommands_sro,importCommands_mso,importCommands_dpo,importCommands_dro,importCommands_fpo,importCommands_tio,importCommands_nato global values_for_vcninfo global cd3file global reg @@ -890,6 +976,7 @@ def export_firewallpolicy(inputfile, _outdir, service_dir, config, signer, ct, e df, values_for_column_fwdecryptprofile = commonTools.read_cd3(cd3file, "Firewall-Policy-DecryptProfile") df, values_for_column_fwdecryptrule = commonTools.read_cd3(cd3file, "Firewall-Policy-DecryptRule") df, values_for_column_fwtunnelinspect = commonTools.read_cd3(cd3file, "Firewall-Policy-TunnelInspect") + df, values_for_column_fwnat = commonTools.read_cd3(cd3file, "Firewall-Policy-NatRules") # Get dict for columns from Excel_Columns #sheet_dict_fwpolicy = ct.sheet_dict[sheetname] #sheet_dict_fwaddress = ct.sheet_dict["Firewall-Policy-Address"] @@ -906,8 +993,7 @@ def export_firewallpolicy(inputfile, _outdir, service_dir, config, signer, ct, e commonTools.backup_file(outdir + "/" + reg + "/" + service_dir, resource, "import_commands_firewallpolicy.sh") importCommands[reg], importCommands_nfp[reg], importCommands_nfao[reg], importCommands_ulo[reg], \ importCommands_slo[reg], importCommands_alo[reg], importCommands_sro[reg], importCommands_mso[reg], \ - importCommands_dpo[reg], importCommands_dro[reg], importCommands_fpo[ - reg], importCommands_tio[reg]= "", "", "", "", "", "", "", "", "", "", "", "" + importCommands_dpo[reg], importCommands_dro[reg], importCommands_fpo[reg], importCommands_tio[reg], importCommands_nato[reg] = "", "", "", "", "", "", "", "", "", "", "", "","" # Fetch Network firewall Policy Details @@ -980,6 +1066,7 @@ def export_firewallpolicy(inputfile, _outdir, service_dir, config, signer, ct, e values_for_column_fwdecryptprofile = print_firewall_decryptprofile(region, ct,values_for_column_fwdecryptprofile,fwpolicies, fwclient,export_tags,state) values_for_column_fwdecryptrule = print_firewall_decryptrule(region, ct, values_for_column_fwdecryptrule,fwpolicies, fwclient,export_tags,state) values_for_column_fwtunnelinspect = print_firewall_tunnelinspect(region, ct, values_for_column_fwtunnelinspect,fwpolicies,fwclient,export_tags,state) + values_for_column_fwnat = print_firewall_nat(region, ct, values_for_column_fwnat,fwpolicies, fwclient, export_tags, state) if clone: commonTools.write_to_cd3(values_for_column_fwpolicy, cd3file, "Firewall-Policy",append=True) commonTools.write_to_cd3(values_for_column_fwaddress, cd3file, "Firewall-Policy-AddressList",append=True) @@ -991,6 +1078,7 @@ def export_firewallpolicy(inputfile, _outdir, service_dir, config, signer, ct, e commonTools.write_to_cd3(values_for_column_fwdecryptprofile, cd3file, "Firewall-Policy-DecryptProfile",append=True) commonTools.write_to_cd3(values_for_column_fwdecryptrule, cd3file, "Firewall-Policy-DecryptRule",append=True) commonTools.write_to_cd3(values_for_column_fwtunnelinspect, cd3file, "Firewall-Policy-TunnelInspect", append=True) + commonTools.write_to_cd3(values_for_column_fwnat, cd3file, "Firewall-Policy-NatRules", append=True) else: commonTools.write_to_cd3(values_for_column_fwpolicy, cd3file, "Firewall-Policy") commonTools.write_to_cd3(values_for_column_fwaddress, cd3file, "Firewall-Policy-AddressList") @@ -1002,16 +1090,17 @@ def export_firewallpolicy(inputfile, _outdir, service_dir, config, signer, ct, e commonTools.write_to_cd3(values_for_column_fwdecryptprofile, cd3file, "Firewall-Policy-DecryptProfile") commonTools.write_to_cd3(values_for_column_fwdecryptrule, cd3file, "Firewall-Policy-DecryptRule") commonTools.write_to_cd3(values_for_column_fwtunnelinspect, cd3file, "Firewall-Policy-TunnelInspect") + commonTools.write_to_cd3(values_for_column_fwnat, cd3file, "Firewall-Policy-NatRules") print("Firewall Policies exported to CD3\n") # writing data init_commands = f'\n######### Writing import for Network firewall policy Objects #########\n\n#!/bin/bash\n{tf_or_tofu} init' - importCommands_message = ["Policy","Address Objects","url list Objects","service list Objects","application list Objects","Security Rules Objects","Mapped Secret Objects","Decrypt profile Objects","decryption Rules Objects","policy Objects","Tunnel Inspect Objects"] + importCommands_message = ["Policy","Address Objects","url list Objects","service list Objects","application list Objects","Security Rules Objects","Mapped Secret Objects","Decrypt profile Objects","decryption Rules Objects","policy Objects","Tunnel Inspect Objects","Nat Rules Objects"] for reg in export_regions: count = 0 all_importCommands = [importCommands_nfp[reg], importCommands_nfao[reg], importCommands_ulo[reg], importCommands_slo[reg], importCommands_alo[reg], importCommands_sro[reg], importCommands_mso[reg], importCommands_dpo[reg], - importCommands_dro[reg], importCommands_fpo[reg], importCommands_tio[reg]] + importCommands_dro[reg], importCommands_fpo[reg], importCommands_tio[reg], importCommands_nato[reg]] for item in all_importCommands: if item != "": importCommands[reg] += f'\n\n######### Writing import for Network firewall {importCommands_message[count]} #########\n\n' diff --git a/cd3_automation_toolkit/Security/Firewall/fw_create.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fw_create.py similarity index 92% rename from cd3_automation_toolkit/Security/Firewall/fw_create.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/fw_create.py index b1d178d07..2a0d7996f 100644 --- a/cd3_automation_toolkit/Security/Firewall/fw_create.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fw_create.py @@ -9,9 +9,11 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -import os +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir @@ -94,8 +96,8 @@ def fw_create(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Compartment Name": columnname = "compartment_tf_name" @@ -116,6 +118,11 @@ def fw_create(inputfile, outdir, service_dir, prefix, ct): columnvalue = commonTools.check_tf_variable(columnvalue) tempdict = {'subnet_compartment_tf_name': columnvalue} + if columnname == "Enable Private NAT(True/False)": + columnname = "must_enable_private_nat" + columnvalue = columnvalue.lower() + tempdict = {'must_enable_private_nat': columnvalue} + if columnname == 'Subnet Name': subnet_tf_name = str(columnvalue).strip() if subnet_tf_name == 'nan' or subnet_tf_name == '': diff --git a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create.py similarity index 93% rename from cd3_automation_toolkit/Security/Firewall/fwpolicy_create.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create.py index 18f97d618..e28905566 100644 --- a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create.py @@ -9,9 +9,12 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -import os +import os, sys + +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir @@ -87,8 +90,8 @@ def firewallpolicy_create(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Compartment Name": columnname = "compartment_tf_name" diff --git a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_address.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_address.py similarity index 97% rename from cd3_automation_toolkit/Security/Firewall/fwpolicy_create_address.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_address.py index a7e90e0dc..d307b809f 100644 --- a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_address.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_address.py @@ -9,9 +9,11 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -import os +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir diff --git a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_applicationlist.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_applicationlist.py similarity index 97% rename from cd3_automation_toolkit/Security/Firewall/fwpolicy_create_applicationlist.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_applicationlist.py index 441b8a4df..debc77dd6 100644 --- a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_applicationlist.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_applicationlist.py @@ -9,9 +9,12 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -import os +import os, sys + +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir diff --git a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_apps.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_apps.py similarity index 98% rename from cd3_automation_toolkit/Security/Firewall/fwpolicy_create_apps.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_apps.py index b8811c288..f8f56a253 100644 --- a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_apps.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_apps.py @@ -9,9 +9,11 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -import os +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir diff --git a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_decryptionprofile.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_decryptionprofile.py similarity index 93% rename from cd3_automation_toolkit/Security/Firewall/fwpolicy_create_decryptionprofile.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_decryptionprofile.py index 5e6f61c06..c43f3d93e 100644 --- a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_decryptionprofile.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_decryptionprofile.py @@ -9,9 +9,11 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -import os +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir @@ -87,8 +89,8 @@ def fwpolicy_create_decryptionprofile(inputfile, outdir, service_dir, prefix, ct tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) diff --git a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_decryptrules.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_decryptrules.py similarity index 97% rename from cd3_automation_toolkit/Security/Firewall/fwpolicy_create_decryptrules.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_decryptrules.py index cb5e22e1f..b6c6cde2f 100644 --- a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_decryptrules.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_decryptrules.py @@ -9,9 +9,12 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -import os +import os, sys + +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir diff --git a/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_natrules.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_natrules.py new file mode 100644 index 000000000..95abd59f5 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_natrules.py @@ -0,0 +1,185 @@ +#!/usr/bin/python3 +# Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. +# +# This script will produce a Terraform file that will be used to set up OCI core components +# firewall, Listeners +# +# Author: Suruchi Singla +# Oracle Consulting +# +from oci.config import DEFAULT_LOCATION +from pathlib import Path +from jinja2 import Environment, FileSystemLoader +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools + +###### +# Required Inputs-CD3 excel file, Config file AND outdir +###### + +# Execution of the code begins here +def fwpolicy_create_natrules(inputfile, outdir, service_dir, prefix, ct): + # Load the template file + file_loader = FileSystemLoader(f'{Path(__file__).parent}/templates') + env = Environment(loader=file_loader, keep_trailing_newline=True) + natrules = env.get_template('policy-natrules-template') + + + sheetName = "Firewall-Policy-NatRules" + natrules_auto_tfvars_filename = prefix + "_"+sheetName.lower()+".auto.tfvars" + + filename = inputfile + + outfile = {} + oname = {} + natrules_str = {} + natrules_names = {} + + + # Read cd3 using pandas dataframe + df, col_headers = commonTools.read_cd3(filename, sheetName) + + df = df.dropna(how='all') + df = df.reset_index(drop=True) + + + for reg in ct.all_regions: + natrules_str[reg] = '' + natrules_names[reg] = [] + reg_out_dir = outdir + "/" + reg + "/" + service_dir + resource = sheetName.lower() + commonTools.backup_file(reg_out_dir, resource, natrules_auto_tfvars_filename) + + + # List of the column headers + dfcolumns = df.columns.values.tolist() + + region_seen_so_far = [] + region_list = [] + + for i in df.index: + region = str(df.loc[i, 'Region']) + region = region.strip().lower() + if region.lower() != 'nan' and region in ct.all_regions: + region = region.strip().lower() + if region not in region_seen_so_far: + region_list.append(region) + region_seen_so_far.append(region) + if region in commonTools.endNames: + break + if region != 'nan' and region not in ct.all_regions: + print("\nInvalid Region; It should be one of the regions tenancy is subscribed to...Exiting!!") + exit() + + + + + # temporary dictionaries + tempStr= {} + tempdict= {} + dst_id = '' + src_id = '' + + + + # Fetch data; loop through columns + for columnname in dfcolumns: + + # Column value + columnvalue = str(df[columnname][i]).strip() + + # Check for boolean/null in column values + columnvalue = commonTools.check_columnvalue(columnvalue) + + # Check for multivalued columns + tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) + + + if columnname == "Firewall Policy": + policy_tf_name = commonTools.check_tf_variable(columnvalue) + tempdict = {'policy_tf_name': policy_tf_name} + + if columnname == "Rule Name": + rule_tf_name = commonTools.check_tf_variable(columnvalue) + tempdict = {'rule_tf_name': rule_tf_name,'rule_name':columnvalue} + + if columnname == "Source Address": + if columnvalue != '': + srcaddrs = str(columnvalue).strip().split(",") + if len(srcaddrs) == 1: + for src in srcaddrs: + src_id = "\"" + src.strip() + "\"" + + elif len(srcaddrs) >= 2: + c = 1 + for src in srcaddrs: + data = "\"" + src.strip() + "\"" + + if c == len(srcaddrs): + src_id = src_id + data + else: + src_id = src_id + data + "," + c += 1 + columnvalue = src_id + tempdict = {'src_address': src_id} + + if columnname == "Destination Address": + if columnvalue != '': + dstaddrs = str(columnvalue).strip().split(",") + if len(dstaddrs) == 1: + for dst in dstaddrs: + dst_id = "\"" + dst.strip() + "\"" + + elif len(dstaddrs) >= 2: + c = 1 + for dst in dstaddrs: + data = "\"" + dst.strip() + "\"" + + if c == len(dstaddrs): + dst_id = dst_id + data + else: + dst_id = dst_id + data + "," + c += 1 + columnvalue = dst_id + tempdict = {'dst_address': dst_id} + + if columnname == "Action": + tempdict = {'action': columnvalue} + + if columnname == "Type": + tempdict = {'type': columnvalue} + + if columnname == "Description": + tempdict = {'description': columnvalue} + + if columnname == "Position": + if columnvalue != '': + position = str(columnvalue).strip().split("::") + placement = position[0] + rule_place = position[1] + tempdict = {'placement': placement, 'rule_place': rule_place} + + columnname = commonTools.check_column_headers(columnname) + tempStr[columnname] = str(columnvalue).strip() + tempStr.update(tempdict) + + + natrules_str[region] = natrules_str[region] + natrules.render(tempStr) + + for reg in region_list: + reg_out_dir = outdir + "/" + reg + "/" + service_dir + if not os.path.exists(reg_out_dir): + os.makedirs(reg_out_dir) + outfile[reg] = reg_out_dir + "/" + natrules_auto_tfvars_filename + if natrules_str[reg] != '': + # Generate Final String + src = "##Add New Nat rules for " + reg.lower() + " here##" + natrules_str[reg] = natrules.render(count=0, region=reg).replace(src, natrules_str[reg] + "\n" + src) + natrules_str[reg] = "".join([s for s in natrules_str[reg].strip().splitlines(True) if s.strip("\r\n").strip()]) + natrules_str[reg] = "\n\n" + natrules_str[reg] + oname[reg] = open(outfile[reg], 'a') + oname[reg].write(natrules_str[reg]) + oname[reg].close() + print(outfile[reg] + " containing TF for Firewall Policy Nat rules has been updated for region " + reg) diff --git a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_secret.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_secret.py similarity index 96% rename from cd3_automation_toolkit/Security/Firewall/fwpolicy_create_secret.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_secret.py index cf2247a8d..11e7ac44b 100644 --- a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_secret.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_secret.py @@ -9,9 +9,12 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -import os +import os, sys + +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir @@ -92,7 +95,7 @@ def fwpolicy_create_secret(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: + if columnname.lower() in ociCommonTools.tagColumns: tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Vault Compartment Name": diff --git a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_secrules.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_secrules.py similarity index 98% rename from cd3_automation_toolkit/Security/Firewall/fwpolicy_create_secrules.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_secrules.py index da479500d..763f9a7ec 100644 --- a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_secrules.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_secrules.py @@ -9,9 +9,11 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -import os +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir diff --git a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_service.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_service.py similarity index 98% rename from cd3_automation_toolkit/Security/Firewall/fwpolicy_create_service.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_service.py index aa1d20735..0b65993b4 100644 --- a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_service.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_service.py @@ -9,9 +9,11 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -import os +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir diff --git a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_servicelist.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_servicelist.py similarity index 97% rename from cd3_automation_toolkit/Security/Firewall/fwpolicy_create_servicelist.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_servicelist.py index 5e39aefa9..746392cbb 100644 --- a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_servicelist.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_servicelist.py @@ -9,9 +9,11 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -import os +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir diff --git a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_tunnelinspection.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_tunnelinspection.py similarity index 97% rename from cd3_automation_toolkit/Security/Firewall/fwpolicy_create_tunnelinspection.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_tunnelinspection.py index aa9689dd6..662f218cb 100644 --- a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_tunnelinspection.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_tunnelinspection.py @@ -9,9 +9,11 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -import os +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir diff --git a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_urllist.py b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_urllist.py similarity index 97% rename from cd3_automation_toolkit/Security/Firewall/fwpolicy_create_urllist.py rename to cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_urllist.py index 2d860db4a..3f4c2bb95 100644 --- a/cd3_automation_toolkit/Security/Firewall/fwpolicy_create_urllist.py +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/fwpolicy_create_urllist.py @@ -9,9 +9,11 @@ # from oci.config import DEFAULT_LOCATION from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -import os +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file AND outdir diff --git a/cd3_automation_toolkit/Security/Firewall/templates/firewall-policies-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/firewall-policies-template similarity index 100% rename from cd3_automation_toolkit/Security/Firewall/templates/firewall-policies-template rename to cd3_automation_toolkit/ocicloud/python/security/firewall/templates/firewall-policies-template diff --git a/cd3_automation_toolkit/Security/Firewall/templates/firewalls-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/firewalls-template similarity index 96% rename from cd3_automation_toolkit/Security/Firewall/templates/firewalls-template rename to cd3_automation_toolkit/ocicloud/python/security/firewall/templates/firewalls-template index edd380251..4e2de174d 100644 --- a/cd3_automation_toolkit/Security/Firewall/templates/firewalls-template +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/firewalls-template @@ -36,6 +36,9 @@ firewalls = { {% if nsgs %} nsg_id= [{{ nsgs }}] {% endif %} + {% if must_enable_private_nat %} + must_enable_private_nat= "{{ must_enable_private_nat }}" + {% endif %} {# ##Do not modify below this line## #} {# #} {# ###Section for adding Defined and Freeform Tags### #} diff --git a/cd3_automation_toolkit/Security/Firewall/templates/policy-addresslists-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-addresslists-template similarity index 100% rename from cd3_automation_toolkit/Security/Firewall/templates/policy-addresslists-template rename to cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-addresslists-template diff --git a/cd3_automation_toolkit/Security/Firewall/templates/policy-applicationlists-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-applicationlists-template similarity index 100% rename from cd3_automation_toolkit/Security/Firewall/templates/policy-applicationlists-template rename to cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-applicationlists-template diff --git a/cd3_automation_toolkit/Security/Firewall/templates/policy-apps-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-apps-template similarity index 100% rename from cd3_automation_toolkit/Security/Firewall/templates/policy-apps-template rename to cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-apps-template diff --git a/cd3_automation_toolkit/Security/Firewall/templates/policy-decryptionprofiles-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-decryptionprofiles-template similarity index 100% rename from cd3_automation_toolkit/Security/Firewall/templates/policy-decryptionprofiles-template rename to cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-decryptionprofiles-template diff --git a/cd3_automation_toolkit/Security/Firewall/templates/policy-decryptrules-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-decryptrules-template similarity index 98% rename from cd3_automation_toolkit/Security/Firewall/templates/policy-decryptrules-template rename to cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-decryptrules-template index ffdf17331..2745102d0 100644 --- a/cd3_automation_toolkit/Security/Firewall/templates/policy-decryptrules-template +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-decryptrules-template @@ -8,7 +8,7 @@ # network_firewall_policy_id can be the ocid or the name of the firewall Policy that needs to be attached to the Firewall # action can be NO_DECRYPT or DECRYPT # Sample import command for Firewall Policy Decryption Rules: -# terraform import "module.decryption_rules[\"<>\"].oci_network_firewall_network_firewall_policy_decryption_rule.network_firewall_policy_decryption_rule\" networkFirewallPolicies/<>/decryptionRules/<> +# terraform import "module.decryption_rules[\"<>\"].oci_network_firewall_network_firewall_policy_decryption_rule.network_firewall_policy_decryption_rule\" networkFirewallPolicies/<>/decryptionRules/<> ############################ decryption_rules = { diff --git a/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-natrules-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-natrules-template new file mode 100644 index 000000000..23c22a89e --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-natrules-template @@ -0,0 +1,39 @@ +{% if count == 0 %} +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +############################# +# Firewall Policy NAT Rules +# Firewall Policy NAT Rule - tfvars +# Allowed Values: +# network_firewall_policy_id can be the ocid or the name of the firewall Policy that needs to be attached to the Firewall +# action can only be DIPP_SRC_NAT +# Sample import command for Firewall Policy NAT Rules: +# terraform import "module.nat_rules[\"<>\"].oci_network_firewall_network_firewall_policy_nat_rule.test_network_firewall_policy_nat_rule" networkFirewallPolicies/<>/natRules/<> +############################ + +nat_rules = { + ##Add New Nat rules for {{ region|lower }} here## +} +{% else %} + {% if rule_name != "" and rule_name != "nan" and rule_name != null %} + {{ policy_tf_name }}_{{rule_tf_name}} = { + rule_name = "{{ rule_name }}" + action = "{{ action }}" + type = "{{ type }}" + description = "{{ description }}" + network_firewall_policy_id = "{{ policy_tf_name }}" + condition = [{ + {% if source_address != '' %} + source_address = [{{ src_address }}] + {% endif %} + {% if destination_address != '' %} + destination_address = [{{ dst_address }}] + {% endif %} + service = "{{ service }}" + }] + {% if position != '' %} + {{placement}} = "{{ rule_place }}" + {% endif %} + }, + {% endif %} +{% endif %} diff --git a/cd3_automation_toolkit/Security/Firewall/templates/policy-secrets-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-secrets-template similarity index 100% rename from cd3_automation_toolkit/Security/Firewall/templates/policy-secrets-template rename to cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-secrets-template diff --git a/cd3_automation_toolkit/Security/Firewall/templates/policy-secrules-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-secrules-template similarity index 99% rename from cd3_automation_toolkit/Security/Firewall/templates/policy-secrules-template rename to cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-secrules-template index e9af9b785..ad99cfed8 100644 --- a/cd3_automation_toolkit/Security/Firewall/templates/policy-secrules-template +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-secrules-template @@ -9,7 +9,7 @@ # action can be ALLOW, DROP, REJECT, INSPECT # inspection can be INTRUSION_DETECTION, INTRUSION_PREVENTION # Sample import command for Firewall Policy Security Rule: -# terraform import "module.security_rules[\"<>\"].oci_network_firewall_network_firewall_policy_security_rule.network_firewall_policy_security_rule\" networkFirewallPolicies/<>/securityRules/<> +# terraform import "module.security_rules[\"<>\"].oci_network_firewall_network_firewall_policy_security_rule.network_firewall_policy_security_rule\" networkFirewallPolicies/<>/securityRules/<> ############################ security_rules = { diff --git a/cd3_automation_toolkit/Security/Firewall/templates/policy-servicelists-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-servicelists-template similarity index 100% rename from cd3_automation_toolkit/Security/Firewall/templates/policy-servicelists-template rename to cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-servicelists-template diff --git a/cd3_automation_toolkit/Security/Firewall/templates/policy-services-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-services-template similarity index 100% rename from cd3_automation_toolkit/Security/Firewall/templates/policy-services-template rename to cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-services-template diff --git a/cd3_automation_toolkit/Security/Firewall/templates/policy-tunnelinspect-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-tunnelinspect-template similarity index 81% rename from cd3_automation_toolkit/Security/Firewall/templates/policy-tunnelinspect-template rename to cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-tunnelinspect-template index c6d4b5879..9da03d915 100644 --- a/cd3_automation_toolkit/Security/Firewall/templates/policy-tunnelinspect-template +++ b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-tunnelinspect-template @@ -8,7 +8,7 @@ # network_firewall_policy_id can be the ocid or the name of the firewall Policy that needs to be attached to the Firewall # action can be INSPECT_AND_CAPTURE_LOG or INSPECT # Sample import command for Firewall Policy Tunnel inspection Rules: -# terraform import "module.decryption_rules[\"<>\"].oci_network_firewall_network_firewall_policy_tunnel_inspection_rule.network_firewall_policy_tunnel_inspection_rule\" networkFirewallPolicies/<>/tunnelInspectionRules/<> +# terraform import "module.tunnelinspect_rules[\"<>\"].oci_network_firewall_network_firewall_policy_tunnel_inspection_rule.network_firewall_policy_tunnel_inspection_rule\" networkFirewallPolicies/<>/tunnelInspectionRules/<> ############################ tunnelinspect_rules = { diff --git a/cd3_automation_toolkit/Security/Firewall/templates/policy-urllists-template b/cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-urllists-template similarity index 100% rename from cd3_automation_toolkit/Security/Firewall/templates/policy-urllists-template rename to cd3_automation_toolkit/ocicloud/python/security/firewall/templates/policy-urllists-template diff --git a/cd3_automation_toolkit/Security/KeyVault/__init__.py b/cd3_automation_toolkit/ocicloud/python/security/keyvault/__init__.py similarity index 100% rename from cd3_automation_toolkit/Security/KeyVault/__init__.py rename to cd3_automation_toolkit/ocicloud/python/security/keyvault/__init__.py diff --git a/cd3_automation_toolkit/Security/KeyVault/create_terraform_keyvaults.py b/cd3_automation_toolkit/ocicloud/python/security/keyvault/create_terraform_keyvaults.py similarity index 97% rename from cd3_automation_toolkit/Security/KeyVault/create_terraform_keyvaults.py rename to cd3_automation_toolkit/ocicloud/python/security/keyvault/create_terraform_keyvaults.py index 143066369..43270fd41 100644 --- a/cd3_automation_toolkit/Security/KeyVault/create_terraform_keyvaults.py +++ b/cd3_automation_toolkit/ocicloud/python/security/keyvault/create_terraform_keyvaults.py @@ -12,7 +12,10 @@ from pathlib import Path import math from oci.config import DEFAULT_LOCATION -from commonTools import * +import os, sys +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### @@ -121,7 +124,7 @@ def create_terraform_keyvaults(inputfile, outdir, service_dir, prefix, ct): # Process Defined and Freeform Tags if str(columnname).lower() in ["vault defined tags", "vault freeform tags"]: - tempdict_vault = commonTools.split_tag_values(columnname, columnvalue, tempdict_vault) + tempdict_vault = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict_vault) elif columnname == "Vault Compartment Name": vault_compartment_var_name = columnvalue.strip() @@ -174,7 +177,7 @@ def create_terraform_keyvaults(inputfile, outdir, service_dir, prefix, ct): else: # Process Defined and Freeform Tags for keys if str(columnname).lower() in ["key defined tags", "key freeform tags"]: - tempdict_keys = commonTools.split_tag_values(columnname, columnvalue, tempdict_keys) + tempdict_keys = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict_keys) elif columnname == "Key Compartment Name": if columnvalue != '' and columnvalue.lower() != 'nan': diff --git a/cd3_automation_toolkit/Security/KeyVault/export_keyvaults_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/security/keyvault/export_keyvaults_nonGreenField.py similarity index 96% rename from cd3_automation_toolkit/Security/KeyVault/export_keyvaults_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/security/keyvault/export_keyvaults_nonGreenField.py index a42bc2031..f8f8bf971 100644 --- a/cd3_automation_toolkit/Security/KeyVault/export_keyvaults_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/security/keyvault/export_keyvaults_nonGreenField.py @@ -12,10 +12,11 @@ from oci.key_management import KmsVaultClient import os import subprocess as sp -sys.path.append(os.getcwd() + "/..") -from commonTools import * from oci.exceptions import TransientServiceError +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools # Execution of the code begins here def export_keyvaults(inputfile, outdir, service_dir, config, signer, ct, export_regions=[], export_compartments=[],export_tags=[]): global values_for_column_kms @@ -163,7 +164,7 @@ def export_keyvaults(inputfile, outdir, service_dir, config, signer, ct, export_ else: values_for_column_kms[col_header].append('') elif str(col_header).lower() in ["vault defined tags", "vault freeform tags"]: - values_for_column_kms = commonTools.export_tags(vault, col_header, + values_for_column_kms = ociCommonTools.export_tags(vault, col_header, values_for_column_kms) elif col_header == 'Key Compartment Name': @@ -185,12 +186,12 @@ def export_keyvaults(inputfile, outdir, service_dir, config, signer, ct, export_ values_for_column_kms[col_header].append(get_key_data.auto_key_rotation_details.rotation_interval_in_days if hasattr(get_key_data.auto_key_rotation_details, 'rotation_interval_in_days') else '') elif str(col_header).lower() in ["key defined tags" , "key freeform tags"]: if len(key.defined_tags) != 0: - values_for_column_kms = commonTools.export_tags(key, col_header, values_for_column_kms) + values_for_column_kms = ociCommonTools.export_tags(key, col_header, values_for_column_kms) else: values_for_column_kms[col_header].append('') else: oci_objs = [vault, key, get_key_data, get_vault_data, get_vault_data.replica_details] - values_for_column_kms = commonTools.export_extra_columns(oci_objs, col_header, + values_for_column_kms = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_kms, values_for_column_kms) else: @@ -215,13 +216,13 @@ def export_keyvaults(inputfile, outdir, service_dir, config, signer, ct, export_ values_for_column_kms[col_header].append(get_key_data.auto_key_rotation_details.rotation_interval_in_days if hasattr(get_key_data.auto_key_rotation_details,'rotation_interval_in_days') else '') elif str(col_header).lower() in ["key defined tags", "key freeform tags"]: if len(key.defined_tags) != 0: - values_for_column_kms = commonTools.export_tags(key, col_header, + values_for_column_kms = ociCommonTools.export_tags(key, col_header, values_for_column_kms) else: values_for_column_kms[col_header].append('') else: oci_objs = [key, get_key_data] - values_for_column_kms = commonTools.export_extra_columns(oci_objs, col_header, + values_for_column_kms = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_kms, values_for_column_kms) pass @@ -244,11 +245,11 @@ def export_keyvaults(inputfile, outdir, service_dir, config, signer, ct, export_ else: values_for_column_kms[col_header].append('') elif str(col_header).lower() in ["vault defined tags", "vault freeform tags"]: - values_for_column_kms = commonTools.export_tags(vault, col_header, + values_for_column_kms = ociCommonTools.export_tags(vault, col_header, values_for_column_kms) else: oci_objs = [vault, get_vault_data, get_vault_data.replica_details] - values_for_column_kms = commonTools.export_extra_columns(oci_objs, col_header, + values_for_column_kms = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_kms, values_for_column_kms) diff --git a/cd3_automation_toolkit/Security/KeyVault/templates/keys-template b/cd3_automation_toolkit/ocicloud/python/security/keyvault/templates/keys-template similarity index 100% rename from cd3_automation_toolkit/Security/KeyVault/templates/keys-template rename to cd3_automation_toolkit/ocicloud/python/security/keyvault/templates/keys-template diff --git a/cd3_automation_toolkit/Security/KeyVault/templates/vaults-template b/cd3_automation_toolkit/ocicloud/python/security/keyvault/templates/vaults-template similarity index 100% rename from cd3_automation_toolkit/Security/KeyVault/templates/vaults-template rename to cd3_automation_toolkit/ocicloud/python/security/keyvault/templates/vaults-template diff --git a/cd3_automation_toolkit/ocicloud/python/storage/__init__.py b/cd3_automation_toolkit/ocicloud/python/storage/__init__.py new file mode 100644 index 000000000..6529b967d --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/python/storage/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 + +from .blockvolume import * +from .filesystem import * +from .objectstorage import * \ No newline at end of file diff --git a/cd3_automation_toolkit/Storage/BlockVolume/__init__.py b/cd3_automation_toolkit/ocicloud/python/storage/blockvolume/__init__.py similarity index 100% rename from cd3_automation_toolkit/Storage/BlockVolume/__init__.py rename to cd3_automation_toolkit/ocicloud/python/storage/blockvolume/__init__.py diff --git a/cd3_automation_toolkit/Storage/BlockVolume/create_terraform_block_volumes.py b/cd3_automation_toolkit/ocicloud/python/storage/blockvolume/create_terraform_block_volumes.py similarity index 98% rename from cd3_automation_toolkit/Storage/BlockVolume/create_terraform_block_volumes.py rename to cd3_automation_toolkit/ocicloud/python/storage/blockvolume/create_terraform_block_volumes.py index 2d1f5c489..4cc92b270 100644 --- a/cd3_automation_toolkit/Storage/BlockVolume/create_terraform_block_volumes.py +++ b/cd3_automation_toolkit/ocicloud/python/storage/blockvolume/create_terraform_block_volumes.py @@ -14,8 +14,9 @@ from oci.config import DEFAULT_LOCATION from pathlib import Path from jinja2 import Environment, FileSystemLoader -sys.path.append(os.getcwd() + "/../..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs-CD3 excel file, Config file, prefix AND outdir @@ -110,8 +111,8 @@ def create_terraform_block_volumes(inputfile, outdir, service_dir, prefix,ct): # Process Freeform Tags and Defined Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) if columnname == "Availability Domain(AD1|AD2|AD3)": columnname = "availability_domain" diff --git a/cd3_automation_toolkit/Storage/BlockVolume/export_blockvolumes_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/storage/blockvolume/export_blockvolumes_nonGreenField.py similarity index 97% rename from cd3_automation_toolkit/Storage/BlockVolume/export_blockvolumes_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/storage/blockvolume/export_blockvolumes_nonGreenField.py index c6291fd02..929dbea07 100644 --- a/cd3_automation_toolkit/Storage/BlockVolume/export_blockvolumes_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/storage/blockvolume/export_blockvolumes_nonGreenField.py @@ -10,11 +10,13 @@ import re import oci -import os +import os, sys from oci.core.blockstorage_client import BlockstorageClient from oci.core.compute_client import ComputeClient -from commonTools import * import subprocess as sp +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools importCommands = {} oci_obj_names = {} @@ -208,11 +210,11 @@ def print_blockvolumes(region, BVOLS, bvol, compute, ct, values_for_column, ntk_ values_for_column[col_header].append(autotune_type) elif col_header == 'Max VPUS Per GB': values_for_column[col_header].append(max_vpus_per_gb) - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(blockvols, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(blockvols, col_header, values_for_column) else: oci_objs = [blockvols,attachments] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) # Execution of the code begins here def export_blockvolumes(inputfile, outdir, service_dir, config, signer, ct, export_compartments=[], export_regions=[], export_tags=[],display_names = [], ad_names = []): diff --git a/cd3_automation_toolkit/Storage/BlockVolume/templates/blockvolumes-template b/cd3_automation_toolkit/ocicloud/python/storage/blockvolume/templates/blockvolumes-template similarity index 100% rename from cd3_automation_toolkit/Storage/BlockVolume/templates/blockvolumes-template rename to cd3_automation_toolkit/ocicloud/python/storage/blockvolume/templates/blockvolumes-template diff --git a/cd3_automation_toolkit/Storage/FileSystem/__init__.py b/cd3_automation_toolkit/ocicloud/python/storage/filesystem/__init__.py similarity index 100% rename from cd3_automation_toolkit/Storage/FileSystem/__init__.py rename to cd3_automation_toolkit/ocicloud/python/storage/filesystem/__init__.py diff --git a/cd3_automation_toolkit/Storage/FileSystem/create_terraform_fss.py b/cd3_automation_toolkit/ocicloud/python/storage/filesystem/create_terraform_fss.py similarity index 98% rename from cd3_automation_toolkit/Storage/FileSystem/create_terraform_fss.py rename to cd3_automation_toolkit/ocicloud/python/storage/filesystem/create_terraform_fss.py index e3108eb47..2a44ed284 100644 --- a/cd3_automation_toolkit/Storage/FileSystem/create_terraform_fss.py +++ b/cd3_automation_toolkit/ocicloud/python/storage/filesystem/create_terraform_fss.py @@ -13,8 +13,9 @@ import os from pathlib import Path -sys.path.append(os.getcwd() + "/../..") -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools from jinja2 import Environment, FileSystemLoader @@ -213,8 +214,8 @@ def fss_exports(i, df, tempStr): tempdict = commonTools.check_multivalues_columnvalue(columnvalue, columnname, tempdict) # Process Defined Tags and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) tempStr.update(tempdict) if columnname == 'Compartment Name': diff --git a/cd3_automation_toolkit/Storage/FileSystem/export_fss_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/storage/filesystem/export_fss_nonGreenField.py similarity index 98% rename from cd3_automation_toolkit/Storage/FileSystem/export_fss_nonGreenField.py rename to cd3_automation_toolkit/ocicloud/python/storage/filesystem/export_fss_nonGreenField.py index 640b28d6f..b75de40d2 100644 --- a/cd3_automation_toolkit/Storage/FileSystem/export_fss_nonGreenField.py +++ b/cd3_automation_toolkit/ocicloud/python/storage/filesystem/export_fss_nonGreenField.py @@ -5,11 +5,13 @@ # Oracle Consulting. import oci -import os +import os, sys import re from oci.config import DEFAULT_LOCATION -from commonTools import * import subprocess as sp +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools fs_source_snapshots = {} fss_all_dict = {} @@ -83,11 +85,11 @@ def add_column_data(reg, cname, AD_name, mt_display_name, vplussubnet, mnt_p_ip, values_for_column_fss[col_header].append(tmp_auth) else: values_for_column_fss[col_header].append('') - elif str(col_header).lower() in commonTools.tagColumns: - values_for_column_fss = commonTools.export_tags(fsinfo.data, col_header, values_for_column_fss) + elif str(col_header).lower() in ociCommonTools.tagColumns: + values_for_column_fss = ociCommonTools.export_tags(fsinfo.data, col_header, values_for_column_fss) else: oci_objs = [fsinfo.data, mnt_info1,einfo_path] - values_for_column_fss = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_instances, + values_for_column_fss = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict_instances, values_for_column_fss) diff --git a/cd3_automation_toolkit/Storage/FileSystem/templates/export-options-template b/cd3_automation_toolkit/ocicloud/python/storage/filesystem/templates/export-options-template similarity index 100% rename from cd3_automation_toolkit/Storage/FileSystem/templates/export-options-template rename to cd3_automation_toolkit/ocicloud/python/storage/filesystem/templates/export-options-template diff --git a/cd3_automation_toolkit/Storage/FileSystem/templates/export-resource-template b/cd3_automation_toolkit/ocicloud/python/storage/filesystem/templates/export-resource-template similarity index 100% rename from cd3_automation_toolkit/Storage/FileSystem/templates/export-resource-template rename to cd3_automation_toolkit/ocicloud/python/storage/filesystem/templates/export-resource-template diff --git a/cd3_automation_toolkit/Storage/FileSystem/templates/fss-replication-template b/cd3_automation_toolkit/ocicloud/python/storage/filesystem/templates/fss-replication-template similarity index 100% rename from cd3_automation_toolkit/Storage/FileSystem/templates/fss-replication-template rename to cd3_automation_toolkit/ocicloud/python/storage/filesystem/templates/fss-replication-template diff --git a/cd3_automation_toolkit/Storage/FileSystem/templates/fss-template b/cd3_automation_toolkit/ocicloud/python/storage/filesystem/templates/fss-template similarity index 100% rename from cd3_automation_toolkit/Storage/FileSystem/templates/fss-template rename to cd3_automation_toolkit/ocicloud/python/storage/filesystem/templates/fss-template diff --git a/cd3_automation_toolkit/Storage/FileSystem/templates/mount-target-template b/cd3_automation_toolkit/ocicloud/python/storage/filesystem/templates/mount-target-template similarity index 100% rename from cd3_automation_toolkit/Storage/FileSystem/templates/mount-target-template rename to cd3_automation_toolkit/ocicloud/python/storage/filesystem/templates/mount-target-template diff --git a/cd3_automation_toolkit/Storage/ObjectStorage/__init__.py b/cd3_automation_toolkit/ocicloud/python/storage/objectstorage/__init__.py similarity index 100% rename from cd3_automation_toolkit/Storage/ObjectStorage/__init__.py rename to cd3_automation_toolkit/ocicloud/python/storage/objectstorage/__init__.py diff --git a/cd3_automation_toolkit/Storage/ObjectStorage/create_terraform_oss.py b/cd3_automation_toolkit/ocicloud/python/storage/objectstorage/create_terraform_oss.py similarity index 98% rename from cd3_automation_toolkit/Storage/ObjectStorage/create_terraform_oss.py rename to cd3_automation_toolkit/ocicloud/python/storage/objectstorage/create_terraform_oss.py index 3faa39b5c..37a65d12b 100644 --- a/cd3_automation_toolkit/Storage/ObjectStorage/create_terraform_oss.py +++ b/cd3_automation_toolkit/ocicloud/python/storage/objectstorage/create_terraform_oss.py @@ -8,10 +8,12 @@ # Oracle Consulting # Modified (TF Upgrade):Ranjini Rajendran # -import os +import os, sys from pathlib import Path from jinja2 import Environment, FileSystemLoader -from commonTools import * +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools ###### # Required Inputs- CD3 excel file, Config file, prefix AND outdir @@ -195,8 +197,8 @@ def create_terraform_oss(inputfile, outdir, service_dir, prefix, ct): tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict) # Process Defined and Freeform Tags - if columnname.lower() in commonTools.tagColumns: - tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict) + if columnname.lower() in ociCommonTools.tagColumns: + tempdict = ociCommonTools.split_tag_values(columnname, columnvalue, tempdict) #Compartment name if columnname == "Compartment Name": diff --git a/cd3_automation_toolkit/Storage/ObjectStorage/export_terraform_oss.py b/cd3_automation_toolkit/ocicloud/python/storage/objectstorage/export_terraform_oss.py similarity index 96% rename from cd3_automation_toolkit/Storage/ObjectStorage/export_terraform_oss.py rename to cd3_automation_toolkit/ocicloud/python/storage/objectstorage/export_terraform_oss.py index 02aa205f9..a6a41ff60 100644 --- a/cd3_automation_toolkit/Storage/ObjectStorage/export_terraform_oss.py +++ b/cd3_automation_toolkit/ocicloud/python/storage/objectstorage/export_terraform_oss.py @@ -14,12 +14,12 @@ from oci.object_storage import ObjectStorageClient import os from pathlib import Path -from commonTools import * from jinja2 import Environment, FileSystemLoader -sys.path.append(os.getcwd()+"/..") -from commonTools import * from dateutil import parser import subprocess as sp +sys.path.append(os.getcwd() + "../") +from common.python.commonTools import * +import ocicloud.python.ociCommonTools as ociCommonTools importCommands = {} @@ -69,8 +69,8 @@ def print_buckets(region, outdir, service_dir,state, bucket_data, values_for_col values_for_column[col_header].append('Private') else: values_for_column[col_header].append('Public') - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(bucket_data, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(bucket_data, col_header, values_for_column) elif col_header == 'Retention Rules': values_for_column[col_header].append(retention_rule_data) elif col_header == 'Replication Policy': @@ -112,7 +112,7 @@ def print_buckets(region, outdir, service_dir,state, bucket_data, values_for_col else: oci_objs = [bucket_data] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column) else: for col_header in values_for_column: if col_header == 'Region': @@ -140,15 +140,15 @@ def print_buckets(region, outdir, service_dir,state, bucket_data, values_for_col values_for_column[col_header].append('Private') else: values_for_column[col_header].append('Public') - elif col_header.lower() in commonTools.tagColumns: - values_for_column = commonTools.export_tags(bucket_data, col_header, values_for_column) + elif col_header.lower() in ociCommonTools.tagColumns: + values_for_column = ociCommonTools.export_tags(bucket_data, col_header, values_for_column) elif col_header == 'Retention Rules': values_for_column[col_header].append(retention_rule_data) elif col_header == 'Replication Policy': values_for_column[col_header].append(rp_details) else: oci_objs = [bucket_data] - values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict,values_for_column) + values_for_column = ociCommonTools.export_extra_columns(oci_objs, col_header, sheet_dict,values_for_column) ###### # Required Inputs- CD3 excel file, Config file, prefix AND outdir diff --git a/cd3_automation_toolkit/Storage/ObjectStorage/templates/oss-template b/cd3_automation_toolkit/ocicloud/python/storage/objectstorage/templates/oss-template similarity index 100% rename from cd3_automation_toolkit/Storage/ObjectStorage/templates/oss-template rename to cd3_automation_toolkit/ocicloud/python/storage/objectstorage/templates/oss-template diff --git a/cd3_automation_toolkit/user-scripts/terraform/README.md b/cd3_automation_toolkit/ocicloud/terraform/README.md similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/README.md rename to cd3_automation_toolkit/ocicloud/terraform/README.md diff --git a/cd3_automation_toolkit/ocicloud/terraform/adb.tf b/cd3_automation_toolkit/ocicloud/terraform/adb.tf new file mode 100755 index 000000000..a3daff169 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/terraform/adb.tf @@ -0,0 +1,114 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +############################# +## Module Block - Autonomous database +## Create autonomous database +############################# +data "oci_core_subnets" "oci_subnets_adb" { + # depends_on = [module.subnets] # Uncomment to create Network and FSS together + #for_each = var.adb != null ? var.adb : {} + for_each = { for k, v in var.adb : k => v if v.vcn_name != null } + compartment_id = each.value.subnet_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.subnet_compartment_id)) > 0 ? each.value.subnet_compartment_id : var.compartment_ocids[each.value.subnet_compartment_id]) : var.compartment_ocids[each.value.subnet_compartment_id] + display_name = each.value.subnet_id + vcn_id = length(regexall("ocid1.vcn.oc*", each.value.vcn_name)) > 0 ? each.value.vcn_name : data.oci_core_vcns.oci_vcns_adb[each.key].virtual_networks.*.id[0] +} + +data "oci_core_vcns" "oci_vcns_adb" { + # depends_on = [module.vcns] # Uncomment to create Network and FSS together + #for_each = var.adb != null ? var.adb : {} + for_each = { for k, v in var.adb : k => v if v.vcn_name != null } + compartment_id = each.value.network_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0 ? each.value.network_compartment_id : var.compartment_ocids[each.value.network_compartment_id]) : var.compartment_ocids[each.value.network_compartment_id] + display_name = each.value.vcn_name + state = "Available" +} +/*output "vcns_data" { + value = { for k, v in data.oci_core_vcns.oci_vcns_adb : k => v } +}*/ +locals { + # Get all admin_password values from var.adb + all_passwords = [ + for adb_entry in values(var.adb) : adb_entry.admin_password + ] + + # Remove empty values + non_empty_passwords = compact(local.all_passwords) + + # Keep only ones that start with "ocid1.vaultsecret.oc" + vault_secret_ocids = toset([ + for pw in local.non_empty_passwords : pw + if length(regexall("ocid1.vaultsecret.oc*", pw)) > 0 + ]) + decoded_vault_passwords = { + for k, v in data.oci_secrets_secretbundle.vault_secrets : + k => base64decode(v.secret_bundle_content[0].content) + } +} +data "oci_secrets_secretbundle" "vault_secrets" { + for_each = local.vault_secret_ocids + secret_id = each.value +} + + +module "adb" { + source = "./modules/database/adb" + for_each = var.adb != null ? var.adb : {} + # depends_on = [module.nsgs] + #admin_password = each.value.admin_password + admin_password = ( + length(regexall("ocid1.vaultsecret.oc*", each.value.admin_password)) > 0 ? + local.decoded_vault_passwords[each.value.admin_password] : + each.value.admin_password + ) + compartment_id = each.value.compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id]) : null + are_primary_whitelisted_ips_used = each.value.are_primary_whitelisted_ips_used != null ? each.value.are_primary_whitelisted_ips_used : null + auto_refresh_frequency_in_seconds = each.value.auto_refresh_frequency_in_seconds != null ? each.value.auto_refresh_frequency_in_seconds : null + auto_refresh_point_lag_in_seconds = each.value.auto_refresh_point_lag_in_seconds != null ? each.value.auto_refresh_point_lag_in_seconds : null + autonomous_container_database_id = each.value.autonomous_container_database_id != null ? each.value.autonomous_container_database_id : null + adb_source = each.value.adb_source != null ? each.value.adb_source :null + source_id = each.value.source_id != null ? each.value.source_id : null + autonomous_database_source_backup_id = each.value.autonomous_database_source_backup_id != null ? each.value.autonomous_database_source_backup_id : null + autonomous_database_id = each.value.autonomous_database_id != null ? each.value.autonomous_database_id : null + autonomous_maintenance_schedule_type = each.value.autonomous_maintenance_schedule_type != null ? each.value.autonomous_maintenance_schedule_type : null + backup_retention_period_in_days = each.value.backup_retention_period_in_days != null ? each.value.backup_retention_period_in_days : null + character_set = each.value.character_set + compute_count = each.value.compute_count + compute_model = each.value.compute_model + customer_contacts = each.value.customer_contacts + data_safe_status = each.value.data_safe_status != null ? each.value.data_safe_status : null + data_storage_size_in_gb = each.value.data_storage_size_in_gb != null ? each.value.data_storage_size_in_gb : null + data_storage_size_in_tbs = each.value.data_storage_size_in_tbs != null ? each.value.data_storage_size_in_tbs : null + database_edition = each.value.database_edition != null ? each.value.database_edition: null + db_name = each.value.db_name + db_version = each.value.db_version + db_workload = each.value.db_workload + defined_tags = each.value.defined_tags + display_name = each.value.display_name + kms_key_id = each.value.kms_key_id != null ? each.value.kms_key_id : null + vault_id = each.value.vault_id != null ? each.value.vault_id : null + freeform_tags = each.value.freeform_tags + network_compartment_id = each.value.network_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0 ? each.value.network_compartment_id : var.compartment_ocids[each.value.network_compartment_id]) : null + subnet_id = each.value.subnet_id != null ? (length(regexall("ocid1.subnet.oc*", each.value.subnet_id)) > 0 ? each.value.subnet_id : data.oci_core_subnets.oci_subnets_adb[each.key].subnets.*.id[0]) : null + vcn_id = each.value.vcn_name != null ?(length(regexall("ocid1.vcn.oc*", each.value.vcn_name)) > 0 ? each.value.vcn_name : data.oci_core_vcns.oci_vcns_adb[each.key].virtual_networks.*.id[0]):null + in_memory_percentage = each.value.in_memory_percentage != null ? each.value.in_memory_percentage : null + is_auto_scaling_enabled = each.value.is_auto_scaling_enabled != null ? each.value.is_auto_scaling_enabled : null + is_auto_scaling_for_storage_enabled = each.value.is_auto_scaling_for_storage_enabled != null ? each.value.is_auto_scaling_for_storage_enabled : null + is_backup_retention_locked = each.value.is_backup_retention_locked != null ? each.value.is_backup_retention_locked : null + is_dedicated = each.value.is_dedicated != null ? each.value.is_dedicated : null + is_local_data_guard_enabled = each.value.is_local_data_guard_enabled != null ? each.value.is_local_data_guard_enabled : null + is_mtls_connection_required = each.value.is_mtls_connection_required != null ? each.value.is_mtls_connection_required : null + is_replicate_automatic_backups = each.value.is_replicate_automatic_backups != null ? each.value.is_replicate_automatic_backups : null + license_model = each.value.license_model != null ? each.value.license_model : null + ncharacter_set = each.value.ncharacter_set + network_security_group_ids = each.value.nsg_ids != null ? each.value.nsg_ids : null + ocpu_count = each.value.ocpu_count != null ? each.value.ocpu_count : null + private_endpoint_ip = each.value.private_endpoint_ip != null ? each.value.private_endpoint_ip : null + private_endpoint_label = each.value.private_endpoint_label != null ? each.value.private_endpoint_label : null + refreshable_mode = each.value.refreshable_mode != null ? each.value.refreshable_mode : null + remote_disaster_recovery_type = each.value.remote_disaster_recovery_type != null ? each.value.remote_disaster_recovery_type : null + time_of_auto_refresh_start = each.value.time_of_auto_refresh_start != null ? each.value.time_of_auto_refresh_start : null + timestamp = each.value.timestamp != null ? each.value.timestamp : null + use_latest_available_backup_time_stamp = each.value.use_latest_available_backup_time_stamp != null ? each.value.use_latest_available_backup_time_stamp : null + whitelisted_ips = each.value.whitelisted_ips != null ? each.value.whitelisted_ips : null + +} diff --git a/cd3_automation_toolkit/user-scripts/terraform/backend.tf b/cd3_automation_toolkit/ocicloud/terraform/backend.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/backend.tf rename to cd3_automation_toolkit/ocicloud/terraform/backend.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/block-volume.tf b/cd3_automation_toolkit/ocicloud/terraform/block-volume.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/block-volume.tf rename to cd3_automation_toolkit/ocicloud/terraform/block-volume.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/budget.tf b/cd3_automation_toolkit/ocicloud/terraform/budget.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/budget.tf rename to cd3_automation_toolkit/ocicloud/terraform/budget.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/cloud-guard.tf b/cd3_automation_toolkit/ocicloud/terraform/cloud-guard.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/cloud-guard.tf rename to cd3_automation_toolkit/ocicloud/terraform/cloud-guard.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/database-exacs.tf b/cd3_automation_toolkit/ocicloud/terraform/database-exacs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/database-exacs.tf rename to cd3_automation_toolkit/ocicloud/terraform/database-exacs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/dbsystem-vm-bm.tf b/cd3_automation_toolkit/ocicloud/terraform/dbsystem-vm-bm.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/dbsystem-vm-bm.tf rename to cd3_automation_toolkit/ocicloud/terraform/dbsystem-vm-bm.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/dedicated-vm-host.tf b/cd3_automation_toolkit/ocicloud/terraform/dedicated-vm-host.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/dedicated-vm-host.tf rename to cd3_automation_toolkit/ocicloud/terraform/dedicated-vm-host.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/dns.tf b/cd3_automation_toolkit/ocicloud/terraform/dns.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/dns.tf rename to cd3_automation_toolkit/ocicloud/terraform/dns.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/firewall.tf b/cd3_automation_toolkit/ocicloud/terraform/firewall.tf similarity index 87% rename from cd3_automation_toolkit/user-scripts/terraform/firewall.tf rename to cd3_automation_toolkit/ocicloud/terraform/firewall.tf index e97acf31c..27ebda9fc 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/firewall.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/firewall.tf @@ -12,6 +12,21 @@ data "oci_core_subnets" "firewall_subnets" { display_name = each.value.subnet_id vcn_id = data.oci_core_vcns.firewall_vcns[each.key].virtual_networks.*.id[0] } +data "oci_kms_vaults" "fw_vault" { + for_each = var.secrets + compartment_id = each.value.vault_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.vault_compartment_id)) > 0 ? each.value.vault_compartment_id : var.compartment_ocids[each.value.vault_compartment_id]) : var.compartment_ocids[each.value.vault_compartment_id] + filter { + name = "display_name" + values = [each.value.vault_name] + } +} + +data "oci_vault_secrets" "fw_secret" { + for_each = var.secrets + compartment_id = each.value.vault_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.vault_compartment_id)) > 0 ? each.value.vault_compartment_id : var.compartment_ocids[each.value.vault_compartment_id]) : var.compartment_ocids[each.value.vault_compartment_id] + name = each.value.vault_secret_id + vault_id = data.oci_kms_vaults.fw_vault[each.key].vaults.*.id[0] +} module "firewalls" { source = "./modules/security/firewall/firewall" @@ -28,6 +43,7 @@ module "firewalls" { vcn_name = each.value.vcn_name defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags + must_enable_private_nat = each.value.must_enable_private_nat } @@ -129,9 +145,7 @@ module "secrets" { network_firewall_policy_id = length(regexall("ocid1.networkfirewallpolicy.oc*", each.value.network_firewall_policy_id)) > 0 ? each.value.network_firewall_policy_id : merge(module.policies.*...)[each.value.network_firewall_policy_id]["policy_tf_id"] secret_source = each.value.secret_source secret_type = each.value.secret_type - vault_secret_id = each.value.vault_secret_id - vault_name = each.value.vault_name - compartment_id = each.value.vault_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.vault_compartment_id)) > 0 ? each.value.vault_compartment_id : var.compartment_ocids[each.value.vault_compartment_id]) : var.compartment_ocids[each.value.vault_compartment_id] + vault_secret_id = length(regexall("ocid1.vaultsecret.oc*", each.value.vault_secret_id)) > 0 ? each.value.vault_secret_id : data.oci_vault_secrets.fw_secret[each.key].secrets.*.id[0] version_number = each.value.version_number } @@ -182,6 +196,21 @@ module "tunnelinspect_rules" { protocol = each.value.protocol } +module "nat_rules" { + source = "./modules/security/firewall/nat-rules" + for_each = var.nat_rules != null ? var.nat_rules : {} + depends_on = [module.policies, module.address_lists, module.service_lists] + action = each.value.action + rule_name = each.value.rule_name + description = each.value.description + type = each.value.type + network_firewall_policy_id = length(regexall("ocid1.networkfirewallpolicy.oc*", each.value.network_firewall_policy_id)) > 0 ? each.value.network_firewall_policy_id : merge(module.policies.*...)[each.value.network_firewall_policy_id]["policy_tf_id"] + source_address = each.value.condition[0].source_address != null ? each.value.condition[0].source_address : [] + destination_address = each.value.condition[0].destination_address != null ? each.value.condition[0].destination_address : [] + service = each.value.condition[0].service != null ? each.value.condition[0].service : "" + after_rule = each.value.after_rule + before_rule = each.value.before_rule +} ############################# # Module Block - Network Firewall Logging @@ -237,4 +266,4 @@ module "fw-logs" { output "vcn_logs_id" { value = [ for k,v in merge(module.vcn-logs.*...) : v.log_tf_id] } -*/ \ No newline at end of file +*/ diff --git a/cd3_automation_toolkit/user-scripts/terraform/fss.tf b/cd3_automation_toolkit/ocicloud/terraform/fss.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/fss.tf rename to cd3_automation_toolkit/ocicloud/terraform/fss.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/identity.tf b/cd3_automation_toolkit/ocicloud/terraform/identity.tf similarity index 78% rename from cd3_automation_toolkit/user-scripts/terraform/identity.tf rename to cd3_automation_toolkit/ocicloud/terraform/identity.tf index 110a8d3e6..c0862c0a1 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/identity.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/identity.tf @@ -29,7 +29,7 @@ module "sub-compartments-level1" { depends_on = [module.iam-compartments] # insert the 4 required variables here tenancy_ocid = var.tenancy_ocid - compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(merge(module.iam-compartments.*...)[each.value.parent_compartment_id]["compartment_tf_id"], var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id]) + compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.iam-compartments.*...)[each.value.parent_compartment_id]["compartment_tf_id"]) compartment_name = each.value.name compartment_description = each.value.description enable_delete = each.value.enable_delete @@ -46,7 +46,7 @@ module "sub-compartments-level2" { depends_on = [module.sub-compartments-level1] # insert the 4 required variables here tenancy_ocid = var.tenancy_ocid - compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(merge(module.sub-compartments-level1.*...)[each.value.parent_compartment_id]["compartment_tf_id"], var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id]) + compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.sub-compartments-level1.*...)[each.value.parent_compartment_id]["compartment_tf_id"]) compartment_name = each.value.name compartment_description = each.value.description @@ -64,7 +64,7 @@ module "sub-compartments-level3" { depends_on = [module.sub-compartments-level2] # insert the 4 required variables here tenancy_ocid = var.tenancy_ocid - compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(merge(module.sub-compartments-level2.*...)[each.value.parent_compartment_id]["compartment_tf_id"], var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id]) + compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.sub-compartments-level2.*...)[each.value.parent_compartment_id]["compartment_tf_id"]) compartment_name = each.value.name compartment_description = each.value.description enable_delete = each.value.enable_delete @@ -81,7 +81,7 @@ module "sub-compartments-level4" { depends_on = [module.sub-compartments-level3] # insert the 4 required variables here tenancy_ocid = var.tenancy_ocid - compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(merge(module.sub-compartments-level3.*...)[each.value.parent_compartment_id]["compartment_tf_id"], var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id]) + compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.sub-compartments-level3.*...)[each.value.parent_compartment_id]["compartment_tf_id"]) compartment_name = each.value.name compartment_description = each.value.description enable_delete = each.value.enable_delete @@ -98,7 +98,7 @@ module "sub-compartments-level5" { depends_on = [module.sub-compartments-level4] # insert the 4 required variables here tenancy_ocid = var.tenancy_ocid - compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(merge(module.sub-compartments-level4.*...)[each.value.parent_compartment_id]["compartment_tf_id"], var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id]) + compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.sub-compartments-level4.*...)[each.value.parent_compartment_id]["compartment_tf_id"]) compartment_name = each.value.name compartment_description = each.value.description enable_delete = each.value.enable_delete @@ -282,14 +282,61 @@ module "iam-network-sources" { # Module Block - Identity # Create Identity Domain Groups ############################ +locals { + # get all the domains used for users + users_unique_idcs_endpoints = { + for k, v in var.identity_domain_users : + v.idcs_endpoint => { + idcs_endpoint = v.idcs_endpoint + domain_compartment_id = v.domain_compartment_id + }... + } + # get all the domains used for groups + groups_unique_idcs_endpoints = { + for k, v in var.identity_domain_groups : + v.idcs_endpoint => { + idcs_endpoint = v.idcs_endpoint + domain_compartment_id = v.domain_compartment_id + }... + } + # get unique domains used across users and groups + domains_distinct = { for k, v in merge(local.groups_unique_idcs_endpoints,local.users_unique_idcs_endpoints) : k => distinct(v)[0]... } + + # users in each domain used in groups + domain_users_map = { + for k,v in local.domains_distinct: + k => { + for user in data.oci_identity_domains_users.users[k].users: + user.user_name => user.id + } if contains(keys(local.groups_unique_idcs_endpoints), k) + } + +} +# output "domain_distinct" { +# value = local.domains_distinct +# } +# output "groups_unique_idcs_endpoints" { +# value = local.groups_unique_idcs_endpoints +# } + +# domain data for unique domin used across users and groups data "oci_identity_domains" "iam_domains" { - for_each = merge(var.identity_domain_groups,var.identity_domain_users) + for_each = local.domains_distinct # Required - compartment_id = var.compartment_ocids[each.value.domain_compartment_id] + compartment_id = var.compartment_ocids[each.value[0].domain_compartment_id] # Optional - display_name = each.value.idcs_endpoint + display_name = each.key } +# user data for each used domain +data "oci_identity_domains_users" "users" { + for_each = { for k, v in local.domains_distinct : k => v if contains(keys(local.groups_unique_idcs_endpoints),k) } + idcs_endpoint = data.oci_identity_domains.iam_domains[each.value[0].idcs_endpoint].domains[0].url +} + + # output "user_map" { + # value = local.domain_users_map + # } module "groups" { depends_on = [module.users] @@ -301,10 +348,10 @@ module "groups" { group_description = each.value.group_description != null ? each.value.group_description : null matching_rule = each.value.matching_rule compartment_id = each.value.domain_compartment_id != "root" ? (length(regexall("ocid1.compartment.oc*", each.value.domain_compartment_id)) > 0 ? each.value.domain_compartment_id : var.compartment_ocids[each.value.domain_compartment_id]) : var.tenancy_ocid - identity_domain = data.oci_identity_domains.iam_domains[each.key].domains[0] + identity_domain = data.oci_identity_domains.iam_domains[each.value.idcs_endpoint].domains[0] tenancy_ocid = var.tenancy_ocid members = each.value.members != null ? each.value.members : [] - + domain_users = local.domain_users_map[each.value.idcs_endpoint] #Optional user_can_request_access = each.value.user_can_request_access defined_tags = each.value.defined_tags @@ -328,7 +375,7 @@ module "users" { middle_name = each.value.name.middle_name honorific_prefix = each.value.name.honorific_prefix display_name = each.value.display_name - identity_domain = data.oci_identity_domains.iam_domains[each.key].domains[0] + identity_domain = data.oci_identity_domains.iam_domains[each.value.idcs_endpoint].domains[0] compartment_id = each.value.domain_compartment_id != "root" ? (length(regexall("ocid1.compartment.oc*", each.value.domain_compartment_id)) > 0 ? each.value.domain_compartment_id : var.compartment_ocids[each.value.domain_compartment_id]) : var.tenancy_ocid description = each.value.description email = each.value.email diff --git a/cd3_automation_toolkit/user-scripts/terraform/instance.tf b/cd3_automation_toolkit/ocicloud/terraform/instance.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/instance.tf rename to cd3_automation_toolkit/ocicloud/terraform/instance.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/kms.tf b/cd3_automation_toolkit/ocicloud/terraform/kms.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/kms.tf rename to cd3_automation_toolkit/ocicloud/terraform/kms.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/loadbalancer.tf b/cd3_automation_toolkit/ocicloud/terraform/loadbalancer.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/loadbalancer.tf rename to cd3_automation_toolkit/ocicloud/terraform/loadbalancer.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/managementservices.tf b/cd3_automation_toolkit/ocicloud/terraform/managementservices.tf similarity index 81% rename from cd3_automation_toolkit/user-scripts/terraform/managementservices.tf rename to cd3_automation_toolkit/ocicloud/terraform/managementservices.tf index 3347b3532..156a44b5b 100755 --- a/cd3_automation_toolkit/user-scripts/terraform/managementservices.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/managementservices.tf @@ -9,12 +9,12 @@ module "alarms" { source = "./modules/managementservices/alarm" - depends_on = [module.notifications-topics] + depends_on = [module.notifications] for_each = var.alarms != null ? var.alarms : {} alarm_name = each.value.alarm_name compartment_id = each.value.compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id]) : null - destinations = [for tn in each.value.destinations : (length(regexall("ocid1.onstopic.oc*", tn)) > 0 ? tn : merge(module.notifications-topics.*...)[tn]["topic_tf_id"])] + destinations = [for tn in each.value.destinations : (length(regexall("ocid1.onstopic.oc*", tn)) > 0 ? tn : merge(module.notifications.*...)[tn]["topic_tf_id"])] is_enabled = each.value.is_enabled metric_compartment_id = each.value.metric_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.metric_compartment_id)) > 0 ? each.value.metric_compartment_id : var.compartment_ocids[each.value.metric_compartment_id]) : null namespace = each.value.namespace @@ -43,7 +43,7 @@ output "alarms_id" { module "events" { source = "./modules/managementservices/event" - depends_on = [module.notifications-topics] + depends_on = [module.notifications] for_each = var.events != null ? var.events : {} event_name = each.value.event_name @@ -53,7 +53,7 @@ module "events" { condition = each.value.condition actions = var.events key_name = each.key - topic_name = merge(module.notifications-topics.*...) + topic_name = merge(module.notifications.*...) #Optional defined_tags = each.value.defined_tags @@ -71,28 +71,14 @@ output "events_id" { # Create Notifications ############################ -module "notifications-topics" { - source = "./modules/managementservices/notification-topic" - for_each = var.notifications_topics != null ? var.notifications_topics : {} +module "notifications" { + source = "./modules/managementservices/notification" + for_each = var.notifications != null ? var.notifications : {} compartment_id = each.value.compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id]) : null description = each.value.description topic_name = each.value.topic_name - - #Optional - defined_tags = each.value.defined_tags - freeform_tags = each.value.freeform_tags -} - -module "notifications-subscriptions" { - source = "./modules/managementservices/notification-subscription" - for_each = var.notifications_subscriptions != null ? var.notifications_subscriptions : {} - - depends_on = [module.notifications-topics] - compartment_id = each.value.compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id]) : null - endpoint = each.value.endpoint - protocol = each.value.protocol - topic_id = length(regexall("ocid1.onstopic.oc*", each.value.topic_id)) > 0 ? each.value.topic_id : merge(module.notifications-topics.*...)[each.value.topic_id]["topic_tf_id"] + subscriptions = each.value.subscriptions != null ? each.value.subscriptions : [] #Optional defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags @@ -108,7 +94,18 @@ output "notifications-topics" { ## Module Block - Service Connector ## Create Service Connectors #################################### - +locals { + source_log_group_detail = { + for k,v in var.service_connectors : k => + [ + for name in v.source_details.source_log_group_names : { + compartment_id = length(regexall("ocid1.compartment.oc*", split("@", name)[0])) > 0 ? split("@", name)[0] : var.compartment_ocids[split("@", name)[0]] + log_group_id = split("@", name)[1] + log_id = split("@", name)[2] +} +] +} +} module "service-connectors" { source = "./modules/managementservices/service-connector" @@ -118,7 +115,7 @@ module "service-connectors" { logs_compartment_id = var.tenancy_ocid source_monitoring_details = each.value.source_details.source_kind == "monitoring" ? { for k, v in each.value.source_details.source_monitoring_details : lookup(var.compartment_ocids, k, "not_found") => v } : {} target_monitoring_details = each.value.target_details.target_kind == "monitoring" ? { for k, v in each.value.target_details.target_monitoring_details : lookup(var.compartment_ocids, k, "not_found") => v } : {} - log_group_names = each.value.source_details.source_kind == "logging" ? flatten([for key in each.value.source_details.source_log_group_names : join("@", tolist([lookup(var.compartment_ocids, split("@", key)[0], "null"), split("@", key)[1], split("@", key)[2]]))]) : [] + log_group_names = each.value.source_details.source_kind == "logging" ? local.source_log_group_detail[each.key] : [] display_name = each.value.display_name description = each.value.description source_kind = each.value.source_details.source_kind diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/compute/dedicated-vm-host/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/dedicated-vm-host/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/compute/dedicated-vm-host/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/compute/dedicated-vm-host/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/compute/dedicated-vm-host/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/dedicated-vm-host/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/compute/dedicated-vm-host/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/compute/dedicated-vm-host/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/compute/dedicated-vm-host/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/dedicated-vm-host/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/compute/dedicated-vm-host/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/compute/dedicated-vm-host/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/compute/dedicated-vm-host/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/dedicated-vm-host/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/compute/dedicated-vm-host/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/compute/dedicated-vm-host/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/compute/instance/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/compute/instance/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/compute/instance/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/compute/instance/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/compute/instance/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/compute/instance/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/compute/instance/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/compute/instance/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/compute/instance/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/compute/instance/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget-alert-rule/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget-alert-rule/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget-alert-rule/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget-alert-rule/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget-alert-rule/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget-alert-rule/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget-alert-rule/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget-alert-rule/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget-alert-rule/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget-alert-rule/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget-alert-rule/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget-alert-rule/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget-alert-rule/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget-alert-rule/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget-alert-rule/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget-alert-rule/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/costmanagement/budget/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/costmanagement/budget/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/adb/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/adb/data.tf similarity index 77% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/adb/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/adb/data.tf index b3f74ce95..4618ccd73 100755 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/database/adb/data.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/database/adb/data.tf @@ -10,15 +10,10 @@ locals { nsg_ids = flatten(tolist([for nsg in var.network_security_group_ids : (length(regexall("ocid1.networksecuritygroup.oc*", nsg)) > 0 ? [nsg] : data.oci_core_network_security_groups.network_security_groups_adb[nsg].network_security_groups[*].id)])) } -data "oci_core_vcns" "oci_vcns_adb" { - compartment_id = var.network_compartment_id != null ? var.network_compartment_id : var.compartment_id - display_name = var.vcn_name -} - data "oci_core_network_security_groups" "network_security_groups_adb" { for_each = { for nsg in var.network_security_group_ids : nsg => nsg } compartment_id = var.network_compartment_id != null ? var.network_compartment_id : var.compartment_id display_name = each.value - vcn_id = data.oci_core_vcns.oci_vcns_adb.virtual_networks[0].id + vcn_id = var.vcn_id } diff --git a/cd3_automation_toolkit/ocicloud/terraform/modules/database/adb/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/adb/main.tf new file mode 100644 index 000000000..55ab382de --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/database/adb/main.tf @@ -0,0 +1,118 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +################################ +## Resource Block - Autonomous database +## Create autonomous database +################################ + +resource "oci_database_autonomous_database" "autonomous_database" { + #Required + compartment_id = var.compartment_id + + #Optional + admin_password = var.admin_password + are_primary_whitelisted_ips_used = var.are_primary_whitelisted_ips_used != null ? var.are_primary_whitelisted_ips_used : null + auto_refresh_frequency_in_seconds = var.auto_refresh_frequency_in_seconds != null ? var.auto_refresh_frequency_in_seconds : null + auto_refresh_point_lag_in_seconds = var.auto_refresh_point_lag_in_seconds != null ? var.auto_refresh_point_lag_in_seconds : null + autonomous_container_database_id = var.autonomous_container_database_id != null ? var.autonomous_container_database_id : null + autonomous_database_backup_id = var.autonomous_database_source_backup_id != null ? ((var.adb_source != null && var.adb_source == "BACKUP_FROM_ID")? var.autonomous_database_source_backup_id:null) : null + autonomous_database_id = var.autonomous_database_id != null ? ((var.adb_source != null && var.adb_source == "BACKUP_FROM_TIMESTAMP")? var.autonomous_database_id:null) : null + autonomous_maintenance_schedule_type = var.autonomous_maintenance_schedule_type != null ? var.autonomous_maintenance_schedule_type: null + backup_retention_period_in_days = var.backup_retention_period_in_days != null ? var.backup_retention_period_in_days : null + character_set = var.character_set != null ? var.character_set : null + compute_count = var.compute_count + compute_model = var.compute_model + dynamic "customer_contacts" { + for_each = var.customer_contacts!=null ? (var.customer_contacts[0] != "" ? var.customer_contacts : []) : [] + + content { + email = customer_contacts.value + } + } + data_safe_status = var.data_safe_status != null ? var.data_safe_status: null + data_storage_size_in_gb = var.data_storage_size_in_gb != null ? var.data_storage_size_in_gb: null + data_storage_size_in_tbs = var.data_storage_size_in_tbs != null ? var.data_storage_size_in_tbs : null + database_edition = var.database_edition + db_name = var.db_name + db_version = var.db_version + db_workload = var.db_workload + defined_tags = var.defined_tags + disaster_recovery_type = var.disaster_recovery_type != null ? var.disaster_recovery_type : null + display_name = var.display_name + dynamic "encryption_key" { + for_each = (var.kms_key_id != null && var.vault_id != null) ? [1] : [] + + content { + kms_key_id = var.kms_key_id + vault_id = var.vault_id + } + } + freeform_tags = var.freeform_tags + in_memory_percentage = var.in_memory_percentage != null ? var.in_memory_percentage : null + is_auto_scaling_enabled = var.is_auto_scaling_enabled != null ? var.is_auto_scaling_enabled : null + is_auto_scaling_for_storage_enabled = var.is_auto_scaling_for_storage_enabled != null ? var.is_auto_scaling_for_storage_enabled : null + is_backup_retention_locked = var.is_backup_retention_locked != null ? var.is_backup_retention_locked : null + is_dedicated = var.is_dedicated != null ? var.is_dedicated : null + is_local_data_guard_enabled = var.is_local_data_guard_enabled != null ? var.is_local_data_guard_enabled : null + is_mtls_connection_required = var.is_mtls_connection_required != null ? var.is_mtls_connection_required : null + is_replicate_automatic_backups = var.is_replicate_automatic_backups != null ? var.is_replicate_automatic_backups : null + kms_key_id = var.kms_key_id != null ? var.kms_key_id : null + license_model = var.license_model + ncharacter_set = var.ncharacter_set + nsg_ids = length(var.network_security_group_ids) != 0 ? (local.nsg_ids == [] ? ["INVALID NSG Name"] : local.nsg_ids) : null + ocpu_count = var.ocpu_count != null ? var.ocpu_count : null + private_endpoint_ip = var.private_endpoint_ip != null ? var.private_endpoint_ip : null + private_endpoint_label = var.private_endpoint_label != null? var.private_endpoint_label : null + refreshable_mode = var.refreshable_mode != null ? var.refreshable_mode : null + remote_disaster_recovery_type = var.remote_disaster_recovery_type!= null? var.remote_disaster_recovery_type : null + secret_id = var.secret_id != null ? var.secret_id: null + secret_version_number = var.secret_version_number != null ? var.secret_version_number : null + vault_id = var.vault_id != null ? var.vault_id : null + source = var.adb_source != null ? var.adb_source : null + source_id = var.source_id != null ? var.source_id: null + standby_whitelisted_ips = var.standby_whitelisted_ips != null ? var.standby_whitelisted_ips : null + subnet_id = var.subnet_id + subscription_id = var.subscription_id != null ? var.subscription_id : null + time_of_auto_refresh_start = var.time_of_auto_refresh_start != null ? var.time_of_auto_refresh_start : null + timestamp = var.timestamp != null ? var.timestamp : null + use_latest_available_backup_time_stamp = var.use_latest_available_backup_time_stamp != null ? var.use_latest_available_backup_time_stamp : null + + whitelisted_ips = var.whitelisted_ips != null ?var.whitelisted_ips: null + lifecycle { + ignore_changes = [ + /* source, + admin_password, + character_set, + compute_count, + compute_model, + customer_contacts, + data_storage_size_in_tbs, + disaster_recovery_type, + encryption_key, # This ignores the entire nested block + in_memory_percentage, + is_auto_scaling_enabled, + is_auto_scaling_for_storage_enabled, + is_backup_retention_locked, + is_dedicated, + is_local_data_guard_enabled, + is_mtls_connection_required, + is_replicate_automatic_backups, + kms_key_id, + ocpu_count, + private_endpoint_ip, + private_endpoint_label, + refreshable_mode, + remote_disaster_recovery_type, + secret_id, + secret_version_number, + vault_id, + source_id, + standby_whitelisted_ips, + subscription_id, + time_of_auto_refresh_start, + timestamp, + use_latest_available_backup_time_stamp,*/ + ] + } +} \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/adb/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/adb/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/adb/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/adb/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/adb/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/adb/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/adb/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/adb/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/adb/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/adb/variables.tf similarity index 53% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/adb/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/adb/variables.tf index ef4faad3d..b3019abfc 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/database/adb/variables.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/database/adb/variables.tf @@ -26,11 +26,6 @@ variable "compartment_id" { type = string } -variable "cpu_core_count" { - description = "The number of OCPU cores to be made available to the database" - type = number -} - variable "database_edition" { description = "The database edition of ADB" type = string @@ -105,7 +100,7 @@ variable "subnet_id" { default = "" } -variable "vcn_name" { +variable "vcn_id" { type = string default = "" } @@ -114,3 +109,117 @@ variable "whitelisted_ips" { type = list(string) default = [] } +variable "are_primary_whitelisted_ips_used" { + default = null +} +variable "auto_refresh_frequency_in_seconds" { + default = null +} +variable "auto_refresh_point_lag_in_seconds" { + default = null +} +variable "autonomous_container_database_id" { + default = null +} +variable "adb_source" { + default = null +} +variable "source_id" { + default = null +} +variable "autonomous_database_source_backup_id" { + default = null +} +variable "autonomous_database_id" { + default = null +} + +variable "autonomous_maintenance_schedule_type" { + default = null +} +variable "backup_retention_period_in_days" { + default = null +} +variable "compute_count" { + default = null +} +variable "compute_model" { + default = null +} +variable "data_safe_status" { + default = null +} +variable "data_storage_size_in_gb" { + default = null +} +variable "disaster_recovery_type" { + default = null +} +variable "in_memory_percentage" { + default = null +} +variable "kms_key_id" { + default = null +} +variable "vault_id" { + default = null +} +variable "is_auto_scaling_enabled" { + default = null +} +variable "is_auto_scaling_for_storage_enabled" { + default = null +} +variable "is_backup_retention_locked" { + default = null +} +variable "is_dedicated" { + default = null +} +variable "is_local_data_guard_enabled" { + default = null +} +variable "is_mtls_connection_required" { + default = null +} +variable "is_replicate_automatic_backups" { + default = null +} + +variable "ocpu_count" { + default = null +} +variable "private_endpoint_ip" { + default = null +} +variable "private_endpoint_label" { + default = null +} +variable "refreshable_mode" { + default = null +} +variable "remote_disaster_recovery_type" { + default = null +} +variable "secret_id" { + default = null +} +variable "secret_version_number" { + default = null +} + +variable "standby_whitelisted_ips" { + default = null +} +variable "subscription_id" { + default = null +} +variable "time_of_auto_refresh_start" { + default = null +} +variable "timestamp" { + default = null +} +variable "use_latest_available_backup_time_stamp" { + default = null +} diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/dbsystem-vm-bm/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/dbsystem-vm-bm/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/dbsystem-vm-bm/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/dbsystem-vm-bm/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/dbsystem-vm-bm/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/dbsystem-vm-bm/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/dbsystem-vm-bm/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/dbsystem-vm-bm/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/dbsystem-vm-bm/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/dbsystem-vm-bm/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/dbsystem-vm-bm/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/dbsystem-vm-bm/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/dbsystem-vm-bm/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/dbsystem-vm-bm/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/dbsystem-vm-bm/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/dbsystem-vm-bm/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/dbsystem-vm-bm/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/dbsystem-vm-bm/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/dbsystem-vm-bm/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/dbsystem-vm-bm/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-infra/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-infra/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-infra/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-infra/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-infra/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-infra/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-infra/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-infra/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-infra/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-infra/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-infra/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-infra/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-infra/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-infra/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-infra/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-infra/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-vmcluster/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-vmcluster/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-vmcluster/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-vmcluster/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-vmcluster/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-vmcluster/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-vmcluster/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-vmcluster/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-vmcluster/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-vmcluster/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-vmcluster/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-vmcluster/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-vmcluster/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-vmcluster/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-vmcluster/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-vmcluster/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-vmcluster/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-vmcluster/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/exa-vmcluster/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/exa-vmcluster/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-configuration/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-configuration/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-configuration/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-configuration/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-configuration/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-configuration/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-configuration/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-configuration/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-configuration/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-configuration/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-configuration/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-configuration/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-configuration/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-configuration/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-configuration/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-configuration/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-configuration/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-configuration/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-configuration/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-configuration/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-dbsystem/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-dbsystem/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-dbsystem/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-dbsystem/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-dbsystem/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-dbsystem/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-dbsystem/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-dbsystem/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-dbsystem/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-dbsystem/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-dbsystem/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-dbsystem/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-dbsystem/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-dbsystem/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-dbsystem/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-dbsystem/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-dbsystem/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-dbsystem/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/database/mysql-dbsystem/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/database/mysql-dbsystem/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/quota-policy/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/quota-policy/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/quota-policy/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/quota-policy/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/quota-policy/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/quota-policy/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/quota-policy/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/quota-policy/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/quota-policy/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/quota-policy/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/quota-policy/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/quota-policy/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/quota-policy/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/quota-policy/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/quota-policy/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/quota-policy/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-default/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-default/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-default/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-default/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-default/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-default/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-default/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-default/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-default/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-default/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-default/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-default/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-default/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-default/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-default/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-default/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-key/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-key/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-key/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-key/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-key/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-key/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-key/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-key/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-key/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-key/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-key/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-key/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-key/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-key/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-key/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-key/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-namespace/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-namespace/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-namespace/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-namespace/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-namespace/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-namespace/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-namespace/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-namespace/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-namespace/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-namespace/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-namespace/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-namespace/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-namespace/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-namespace/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/governance/tagging/tag-namespace/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/governance/tagging/tag-namespace/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-compartment/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-compartment/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-compartment/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-compartment/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-compartment/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-compartment/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-compartment/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-compartment/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-compartment/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-compartment/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-compartment/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-compartment/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-compartment/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-compartment/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-compartment/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-compartment/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-group/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-group/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-group/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-group/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-group/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-group/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-group/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-group/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-group/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-group/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-network-sources/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-network-sources/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-network-sources/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-network-sources/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-network-sources/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-network-sources/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-network-sources/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-network-sources/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-network-sources/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-network-sources/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-network-sources/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-network-sources/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-network-sources/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-network-sources/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-network-sources/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-network-sources/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-policy/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-policy/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-policy/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-policy/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-policy/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-policy/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-policy/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-policy/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-policy/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-policy/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-policy/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-policy/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-policy/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-policy/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-policy/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-policy/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-user/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-user/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-user/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-user/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-user/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-user/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-user/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-user/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-user/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-user/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-user/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-user/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-user/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-user/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-user/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-user/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-user/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-user/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-user/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/iam-user/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-group/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-group/data.tf similarity index 85% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-group/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-group/data.tf index bf3cd9ffc..fe2f02adb 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-group/data.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-group/data.tf @@ -20,9 +20,9 @@ data "oci_identity_domains" "iam_domains" { ############################ -data "oci_identity_domains_users" "users" { - idcs_endpoint = var.identity_domain.url -} +# data "oci_identity_domains_users" "users" { +# idcs_endpoint = var.identity_domain.url +# } diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-group/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-group/main.tf similarity index 88% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-group/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-group/main.tf index 79ac0b08c..c2b833350 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-group/main.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-group/main.tf @@ -5,12 +5,12 @@ # Resource Block - Identity # Create Groups ############################ -locals { - user_ids = { - for user in data.oci_identity_domains_users.users.users : - user.user_name => user.id... - } -} +# locals { +# user_ids = { +# for user in data.oci_identity_domains_users.users.users : +# user.user_name => user.id... +# } +# } resource "oci_identity_domains_group" "group" { count = (var.matching_rule == "" || var.matching_rule == null) ? 1 : 0 @@ -25,15 +25,18 @@ resource "oci_identity_domains_group" "group" { "urn:ietf:params:scim:schemas:oracle:idcs:extension:group:Group", ] timeouts {} - urnietfparamsscimschemasoracleidcsextensiongroup_group { - description = var.group_description + dynamic "urnietfparamsscimschemasoracleidcsextensiongroup_group" { + for_each = var.group_description != null ? [1]:[] + content { + description = var.group_description + } } dynamic "members" { for_each = {for k in var.members: k=>k} content { type = "User" - value = local.user_ids[members.value][0] + value = var.domain_users[members.value] } } dynamic "urnietfparamsscimschemasoracleidcsextensionrequestable_group" { @@ -69,6 +72,7 @@ resource "oci_identity_domains_group" "group" { schemas, urnietfparamsscimschemasoracleidcsextension_oci_tags["defined_tags.CreatedOn"], urnietfparamsscimschemasoracleidcsextension_oci_tags["defined_tags.CreatedBy"], +# attribute_sets,attributes,members ] } } @@ -120,6 +124,7 @@ resource "oci_identity_domains_dynamic_resource_group" "dynamic_group" { schemas, urnietfparamsscimschemasoracleidcsextension_oci_tags["defined_tags.CreatedOn"], urnietfparamsscimschemasoracleidcsextension_oci_tags["defined_tags.CreatedBy"], + #attribute_sets,attributes,matching_rule ] } } diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-group/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-group/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-group/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-group/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-group/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-group/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-group/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-group/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-group/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-group/variables.tf similarity index 95% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-group/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-group/variables.tf index c24585f26..c77eba9a1 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-group/variables.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-group/variables.tf @@ -18,6 +18,11 @@ variable "user_id" { default = null } +variable "domain_users" { +description = "all users in the domain" +default = {} +} + variable "members" { description = "List of email ids of the users" type = list(string) diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-user/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-user/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-user/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-user/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-user/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-user/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-user/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-user/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-user/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-user/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-user/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-user/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-user/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-user/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-user/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-user/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-user/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-user/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/identity/identity-domain-user/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/identity/identity-domain-user/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/ip/public-ip-pool/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/ip/public-ip-pool/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/ip/public-ip-pool/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/ip/public-ip-pool/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/ip/public-ip-pool/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/ip/public-ip-pool/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/ip/public-ip-pool/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/ip/public-ip-pool/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/ip/public-ip-pool/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/ip/public-ip-pool/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/ip/public-ip-pool/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/ip/public-ip-pool/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/ip/public-ip-pool/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/ip/public-ip-pool/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/ip/public-ip-pool/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/ip/public-ip-pool/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/ip/reserved-public-ip/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/ip/reserved-public-ip/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/ip/reserved-public-ip/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/ip/reserved-public-ip/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/ip/reserved-public-ip/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/ip/reserved-public-ip/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/ip/reserved-public-ip/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/ip/reserved-public-ip/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/ip/reserved-public-ip/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/ip/reserved-public-ip/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/ip/reserved-public-ip/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/ip/reserved-public-ip/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/ip/reserved-public-ip/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/ip/reserved-public-ip/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/ip/reserved-public-ip/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/ip/reserved-public-ip/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/ip/secondary-private-ip/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/ip/secondary-private-ip/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/ip/secondary-private-ip/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/ip/secondary-private-ip/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/ip/secondary-private-ip/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/ip/secondary-private-ip/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/ip/secondary-private-ip/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/ip/secondary-private-ip/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/ip/secondary-private-ip/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/ip/secondary-private-ip/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/ip/secondary-private-ip/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/ip/secondary-private-ip/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/ip/secondary-private-ip/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/ip/secondary-private-ip/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/ip/secondary-private-ip/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/ip/secondary-private-ip/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend-set/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend-set/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend-set/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend-set/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend-set/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend-set/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend-set/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend-set/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend-set/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend-set/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend-set/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend-set/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend-set/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend-set/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend-set/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend-set/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-backend/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-backend/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-certificate/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-certificate/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-certificate/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-certificate/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-certificate/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-certificate/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-certificate/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-certificate/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-certificate/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-certificate/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-certificate/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-certificate/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-certificate/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-certificate/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-certificate/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-certificate/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-cipher-suite/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-cipher-suite/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-cipher-suite/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-cipher-suite/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-cipher-suite/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-cipher-suite/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-cipher-suite/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-cipher-suite/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-cipher-suite/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-cipher-suite/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-cipher-suite/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-cipher-suite/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-cipher-suite/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-cipher-suite/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-cipher-suite/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-cipher-suite/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-hostname/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-hostname/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-hostname/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-hostname/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-hostname/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-hostname/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-hostname/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-hostname/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-hostname/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-hostname/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-hostname/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-hostname/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-hostname/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-hostname/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-hostname/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-hostname/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-listener/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-listener/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-listener/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-listener/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-listener/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-listener/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-listener/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-listener/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-listener/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-listener/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-listener/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-listener/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-listener/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-listener/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-listener/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-listener/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-load-balancer/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-load-balancer/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-load-balancer/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-load-balancer/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-load-balancer/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-load-balancer/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-load-balancer/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-load-balancer/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-load-balancer/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-load-balancer/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-load-balancer/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-load-balancer/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-load-balancer/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-load-balancer/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-load-balancer/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-load-balancer/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-load-balancer/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-load-balancer/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-load-balancer/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-load-balancer/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-path-route-set/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-path-route-set/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-path-route-set/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-path-route-set/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-path-route-set/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-path-route-set/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-path-route-set/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-path-route-set/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-path-route-set/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-path-route-set/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-path-route-set/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-path-route-set/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-path-route-set/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-path-route-set/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-path-route-set/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-path-route-set/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-routing-policy/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-routing-policy/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-routing-policy/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-routing-policy/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-routing-policy/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-routing-policy/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-routing-policy/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-routing-policy/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-routing-policy/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-routing-policy/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-routing-policy/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-routing-policy/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-routing-policy/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-routing-policy/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-routing-policy/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-routing-policy/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-rule-set/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-rule-set/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-rule-set/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-rule-set/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-rule-set/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-rule-set/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-rule-set/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-rule-set/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-rule-set/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-rule-set/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-rule-set/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-rule-set/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-rule-set/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-rule-set/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/loadbalancer/lb-rule-set/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/loadbalancer/lb-rule-set/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/alarm/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/alarm/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/alarm/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/alarm/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/alarm/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/alarm/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/alarm/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/alarm/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/alarm/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/alarm/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/alarm/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/alarm/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/alarm/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/alarm/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/alarm/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/alarm/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/event/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/event/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/event/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/event/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/event/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/event/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/event/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/event/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/event/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/event/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/event/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/event/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/event/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/event/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/event/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/event/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log-group/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log-group/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log-group/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log-group/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log-group/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log-group/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log-group/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log-group/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log-group/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log-group/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log-group/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log-group/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log-group/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log-group/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log-group/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log-group/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/log/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/log/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-topic/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/notification/main.tf similarity index 54% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-topic/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/notification/main.tf index b471cc175..dcde61369 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-topic/main.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/notification/main.tf @@ -18,3 +18,20 @@ resource "oci_ons_notification_topic" "topic" { freeform_tags = var.freeform_tags } + +resource "oci_ons_subscription" "subscription" { + + count = length(var.subscriptions) + #Required + compartment_id = var.compartment_id + endpoint = var.subscriptions[count.index].endpoint + protocol = var.subscriptions[count.index].protocol + topic_id = oci_ons_notification_topic.topic.id + + #Optional + defined_tags = var.defined_tags + freeform_tags = var.freeform_tags + lifecycle { + ignore_changes = [defined_tags,freeform_tags] + } +} \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-subscription/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/notification/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-subscription/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/notification/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-topic/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/notification/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-topic/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/notification/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-topic/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/notification/variables.tf similarity index 91% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-topic/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/notification/variables.tf index 47d1f8ddb..c1498caf7 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-topic/variables.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/notification/variables.tf @@ -13,6 +13,10 @@ variable "compartment_id" { variable "topic_name" { type = string } +variable "subscriptions" { +type = list(map(any)) +default = [] +} variable "description" { type = string } diff --git a/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/service-connector/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/service-connector/data.tf new file mode 100755 index 000000000..d344ef619 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/service-connector/data.tf @@ -0,0 +1,135 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +#################################### +# Data Block - Service Connector +# Create Service Connector Hub +##################################### + +locals { + log_group_names = var.log_group_names + source_kind = var.source_kind + unique_log_groups = { + for key in distinct([ + for item in var.log_group_names : "${item.compartment_id}@${item.log_group_id}"]) : key => { + compartment_id = split("@", key)[0] + log_group_id = split("@", key)[1] + } + } + + resolved_log_group_ids = [ + for item in var.log_group_names : { + compartment_id = item.compartment_id + log_group_id = length(regexall("Audit", item.log_group_id)) > 0 ? (length(regexall("Audit_In_Subcompartment", item.log_group_id)) > 0 ? "_Audit_Include_Subcompartment" : "_Audit") : data.oci_logging_log_groups.source_log_groups["${item.compartment_id}@${item.log_group_id}"].log_groups[0].id + log_id_name = item.log_id + } + ] + + source_log_group_detail = [ + for item in local.resolved_log_group_ids : { + compartment_id = item.compartment_id + log_group_id = item.log_group_id + log_id = lower(item.log_id_name) == "all" ? null : data.oci_logging_logs.source_logs["${item.log_group_id}@${item.log_id_name}"].logs[0].id + } + ] + + # creating ordered map + n = length(local.source_log_group_detail) + # Number of groups of 10 items + groups_count = ceil(local.n / 10) + prefixes = [ + for i in range(local.groups_count) : join("", [for _ in range(i) : "9"]) + ] + # Now create keys by combining prefix + digit (0-9), flatten the list, but only up to n keys + keys = slice(flatten([ + for prefix in local.prefixes : [ + for digit in range(10) : "${prefix}${digit}" + ] + ]), 0, local.n) + indexed_map = { + for idx in range(local.n) : + local.keys[idx] => local.source_log_group_detail[idx] + } + # local ends here +} + +#output "first_level_logs" { + +#value = local.indexed_map #local.source_log_group_detail + +#} + +data "oci_logging_log_groups" "source_log_groups" { + for_each = local.unique_log_groups + compartment_id = each.value.compartment_id + display_name = each.value.log_group_id +} + +data "oci_logging_logs" "source_logs" { + for_each = { + for item in distinct(local.resolved_log_group_ids) : + "${item.log_group_id}@${item.log_id_name}" => item + if lower(item.log_id_name) != "all" } + log_group_id = each.value.log_group_id + display_name = each.value.log_id_name +} + +data "oci_objectstorage_namespace" "os_namespace" { + compartment_id = var.logs_compartment_id +} + +data "oci_streaming_streams" "source_streams" { + for_each = var.source_stream_id + name = each.value + compartment_id = each.key +} +data "oci_streaming_streams" "target_streams" { + for_each = var.stream_id + name = each.value + compartment_id = each.key +} +data "oci_ons_notification_topics" "target_topics" { + for_each = var.topic_id + name = each.value + compartment_id = each.key +} + +data "oci_log_analytics_log_analytics_log_groups" "target_log_analytics_log_groups" { + for_each = var.destination_log_group_id + #Required + compartment_id = each.key + namespace = data.oci_objectstorage_namespace.os_namespace.namespace + + #Optional + display_name = each.value +} + +data "oci_functions_applications" "applications" { + for_each = toset(var.function_details) + #Required + compartment_id = split("@", each.key)[0] + + #Optional + display_name = split("@", each.key)[1] +} + +data "oci_functions_functions" "functions" { + for_each = toset(var.function_details) + #Required + application_id = data.oci_functions_applications.applications[each.key].applications[0].id + + #Optional + display_name = split("@", each.key)[2] +} + +data "oci_identity_compartments" "compartments" { + for_each = toset(keys(var.source_monitoring_details)) + #Required + compartment_id = var.logs_compartment_id + + #Optional + access_level = "ANY" + compartment_id_in_subtree = true + state = "ACTIVE" + name = each.value +} \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/service-connector/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/service-connector/main.tf similarity index 82% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/service-connector/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/service-connector/main.tf index bea14e2c2..d567c8bc2 100755 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/service-connector/main.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/service-connector/main.tf @@ -38,11 +38,11 @@ resource "oci_sch_service_connector" "service_connector" { } dynamic "log_sources" { - for_each = toset(var.log_group_names) + for_each = local.indexed_map #{for idx,val in local.source_log_group_detail:idx => val} content { - compartment_id = split("@", log_sources.key)[0] - log_group_id = length(regexall("Audit", split("@", log_sources.key)[1])) > 0 ? (length(regexall("Audit_In_Subcompartment", split("@", log_sources.key)[1])) > 0 ? "_Audit_Include_Subcompartment" : "_Audit") : data.oci_logging_log_groups.source_log_groups[log_sources.key].log_groups[0].id - log_id = lower(split("@", log_sources.key)[2]) == "all" ? null : data.oci_logging_logs.source_logs[log_sources.key].logs[0].id + compartment_id = log_sources.value.compartment_id #split("@", log_sources.key)[0] + log_group_id = log_sources.value.log_group_id #length(regexall("Audit", split("@", log_sources.key)[1])) > 0 ? (length(regexall("Audit_In_Subcompartment", split("@", log_sources.key)[1])) > 0 ? "_Audit_Include_Subcompartment" : "_Audit") : data.oci_logging_log_groups.source_log_groups[log_sources.key].log_groups[0].id + log_id = log_sources.value.log_id #lower(split("@", log_sources.key)[2]) == "all" ? null : data.oci_logging_logs.source_logs[log_sources.key].logs[0].id } } stream_id = var.source_kind == "streaming" ? data.oci_streaming_streams.source_streams[one(keys(var.source_stream_id))].streams[0].id : null diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-topic/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/service-connector/oracle_provider_req.tf old mode 100644 new mode 100755 similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-topic/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/service-connector/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/service-connector/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/service-connector/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/service-connector/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/service-connector/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/service-connector/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/service-connector/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/service-connector/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/managementservices/service-connector/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/custom-dhcp/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/custom-dhcp/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/custom-dhcp/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/custom-dhcp/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/service-connector/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/custom-dhcp/oracle_provider_req.tf old mode 100755 new mode 100644 similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/service-connector/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/custom-dhcp/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/custom-dhcp/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/custom-dhcp/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/custom-dhcp/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/custom-dhcp/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/custom-dhcp/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/custom-dhcp/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/custom-dhcp/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/custom-dhcp/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/default-dhcp/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/default-dhcp/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/default-dhcp/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/default-dhcp/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/custom-dhcp/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/default-dhcp/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/custom-dhcp/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/default-dhcp/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/default-dhcp/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/default-dhcp/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/default-dhcp/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/default-dhcp/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/default-dhcp/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/default-dhcp/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/default-dhcp/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/default-dhcp/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/dns_resolver/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/dns_resolver/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/dns_resolver/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/dns_resolver/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/default-dhcp/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/dns_resolver/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/default-dhcp/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/dns_resolver/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/dns_resolver/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/dns_resolver/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/dns_resolver/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/dns_resolver/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/dns_resolver/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/dns_resolver/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/dns_resolver/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/dns_resolver/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/rrset/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/rrset/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/rrset/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/rrset/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/dns_resolver/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/rrset/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/dns_resolver/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/rrset/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/rrset/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/rrset/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/rrset/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/rrset/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/rrset/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/rrset/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/rrset/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/rrset/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/view/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/view/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/view/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/view/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/rrset/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/view/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/rrset/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/view/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/view/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/view/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/view/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/view/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/view/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/view/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/view/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/view/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/zone/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/zone/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/zone/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/zone/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/view/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/zone/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/view/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/zone/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/zone/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/zone/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/zone/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/zone/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/zone/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/zone/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/zone/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/dns/zone/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-attachment/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-attachment/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-attachment/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-attachment/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/zone/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-attachment/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/zone/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-attachment/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-attachment/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-attachment/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-attachment/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-attachment/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-attachment/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-attachment/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-attachment/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-attachment/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution-statement/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution-statement/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution-statement/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution-statement/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-attachment/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution-statement/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-attachment/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution-statement/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution-statement/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution-statement/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution-statement/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution-statement/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution-statement/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution-statement/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution-statement/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution-statement/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution-statement/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution-statement/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-distribution/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-rule/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-rule/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-rule/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-rule/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-rule/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-distribution/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-rule/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-rule/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-rule/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-rule/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-rule/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-rule/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-rule/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-rule/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-rule/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-table/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-table/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-table/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-table/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-rule/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-table/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-rule/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-table/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-table/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-table/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-table/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-table/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-table/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-table/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-table/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg-route-table/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-table/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg-route-table/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/drg/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/drg/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/igw/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/igw/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/igw/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/igw/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/drg/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/igw/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/drg/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/igw/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/igw/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/igw/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/igw/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/igw/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/igw/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/igw/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/igw/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/igw/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/lpg/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/lpg/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/lpg/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/lpg/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/igw/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/lpg/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/igw/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/lpg/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/lpg/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/lpg/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/lpg/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/lpg/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/lpg/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/lpg/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/lpg/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/lpg/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/ngw/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/ngw/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/ngw/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/ngw/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/lpg/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/ngw/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/lpg/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/ngw/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/ngw/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/ngw/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/ngw/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/ngw/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/ngw/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/ngw/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/ngw/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/ngw/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg-rule/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg-rule/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg-rule/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg-rule/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/ngw/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg-rule/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/ngw/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg-rule/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg-rule/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg-rule/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg-rule/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg-rule/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg-rule/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg-rule/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg-rule/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg-rule/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg-rule/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg-rule/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/nsg/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/route-table/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/route-table/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/route-table/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/route-table/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/route-table/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/nsg/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/route-table/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/route-table/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/route-table/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/route-table/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/route-table/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/route-table/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/route-table/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/route-table/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/route-table/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/sec-list/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/sec-list/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/sec-list/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/sec-list/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/route-table/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/sec-list/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/route-table/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/sec-list/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/sec-list/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/sec-list/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/sec-list/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/sec-list/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/sec-list/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/sec-list/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/sec-list/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/sec-list/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/sgw/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/sgw/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/sgw/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/sgw/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/sec-list/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/sgw/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/sec-list/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/sgw/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/sgw/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/sgw/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/sgw/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/sgw/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/sgw/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/sgw/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/sgw/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/sgw/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/subnet/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/subnet/main.tf similarity index 83% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/subnet/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/subnet/main.tf index f822ae0c4..62a33904c 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/network/subnet/main.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/network/subnet/main.tf @@ -20,10 +20,11 @@ resource "oci_core_subnet" "subnet" { display_name = var.display_name dns_label = var.dns_label freeform_tags = var.freeform_tags - ipv6cidr_block = var.ipv6cidr_block + ipv6cidr_blocks = var.ipv6cidr_block != null ? [var.ipv6cidr_block] : null prohibit_internet_ingress = var.prohibit_internet_ingress prohibit_public_ip_on_vnic = var.prohibit_public_ip_on_vnic route_table_id = var.route_table_id - security_list_ids = var.security_list_ids != [] ? [for sl in var.security_list_ids : (length(regexall("ocid1.securitylist.oc*", sl)) > 0 ? sl : (sl == "" ? var.vcn_default_security_list_id : var.custom_security_list_id[sl]["seclist_tf_id"]))] : [] + security_list_ids = var.security_list_ids != [] ? [for sl in var.security_list_ids : (length(regexall("ocid1.securitylist.oc*", sl)) > 0 ? sl : (sl == "" ? var.vcn_default_security_list_id : (contains(keys(var.custom_security_list_id), sl) ? var.custom_security_list_id[sl]["seclist_tf_id"] : sl +)))] : [] } \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/sgw/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/subnet/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/sgw/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/subnet/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/subnet/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/subnet/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/subnet/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/subnet/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/subnet/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/subnet/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/subnet/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/subnet/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/vcn/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/vcn/main.tf similarity index 70% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/vcn/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/vcn/main.tf index 43a488159..723688f67 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/network/vcn/main.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/network/vcn/main.tf @@ -5,7 +5,10 @@ # Resource Block - Network # Create VCNs ############################ - +locals { + is_ipv6enabled = var.is_ipv6enabled == null ?(length(var.byoipv6cidr_details) > 0 || + length(var.ipv6private_cidr_blocks) > 0 || var.is_oracle_gua_allocation_enabled) : var.is_ipv6enabled +} resource "oci_core_vcn" "vcn" { #Required @@ -26,9 +29,11 @@ resource "oci_core_vcn" "vcn" { display_name = var.display_name dns_label = var.dns_label freeform_tags = var.freeform_tags - is_ipv6enabled = var.is_ipv6enabled + is_ipv6enabled = local.is_ipv6enabled + #is_ipv6enabled = false + #is_ipv6enabled = var.is_ipv6enabled ipv6private_cidr_blocks = var.ipv6private_cidr_blocks - is_oracle_gua_allocation_enabled = var.is_oracle_gua_allocation_enabled + is_oracle_gua_allocation_enabled = local.is_ipv6enabled ? var.is_oracle_gua_allocation_enabled : null lifecycle { create_before_destroy = true } diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/subnet/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/vcn/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/subnet/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/vcn/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/vcn/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/vcn/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/vcn/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/vcn/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/vcn/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/vcn/variables.tf similarity index 92% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/vcn/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/vcn/variables.tf index ceeb05d84..e79380d61 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/network/vcn/variables.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/network/vcn/variables.tf @@ -45,11 +45,11 @@ variable "freeform_tags" { variable "is_ipv6enabled" { type = bool - default = false + default = null } variable "ipv6private_cidr_blocks" { - type = list(any) + type = list(string) default = [] } @@ -59,6 +59,6 @@ variable "is_oracle_gua_allocation_enabled" { } variable "byoipv6cidr_details" { - type = list(any) - default = [{}] + type = list(map(string)) + default = [] } diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/vlan/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/vlan/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/vlan/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/vlan/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/vlan/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/vlan/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/vlan/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/vlan/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/vcn/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/vlan/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/vcn/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/vlan/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/vlan/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/vlan/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/vlan/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/vlan/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/vlan/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/network/vlan/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/vlan/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/network/vlan/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backend/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backend/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backend/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backend/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backend/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backend/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backend/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backend/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/vlan/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backend/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/network/vlan/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backend/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backend/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backend/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backend/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backend/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backend/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backend/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backend/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backend/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backendset/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backendset/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backendset/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backendset/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backend/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backendset/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backend/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backendset/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backendset/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backendset/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backendset/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backendset/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backendset/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backendset/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backendset/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-backendset/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-listener/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-listener/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-listener/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-listener/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backendset/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-listener/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-backendset/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-listener/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-listener/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-listener/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-listener/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-listener/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-listener/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-listener/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-listener/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb-listener/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-listener/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb-listener/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/networkloadbalancer/nlb/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/networkloadbalancer/nlb/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/virtual-nodepool/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/virtual-nodepool/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/virtual-nodepool/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/virtual-nodepool/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/virtual-nodepool/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/virtual-nodepool/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/virtual-nodepool/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/virtual-nodepool/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/virtual-nodepool/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/virtual-nodepool/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/virtual-nodepool/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/virtual-nodepool/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/virtual-nodepool/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/virtual-nodepool/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/virtual-nodepool/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/virtual-nodepool/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/virtual-nodepool/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/oke/virtual-nodepool/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc-cluster/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc-cluster/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc-cluster/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc-cluster/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/virtual-nodepool/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc-cluster/oracle_provider_req.tf old mode 100644 new mode 100755 similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/oke/virtual-nodepool/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc-cluster/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc-cluster/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc-cluster/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc-cluster/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc-cluster/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc-cluster/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc-cluster/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc-cluster/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc-cluster/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc-cluster/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc-cluster/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/sddc/sddc/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-configuration/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-configuration/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-configuration/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-configuration/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-configuration/oracle_provider_req.tf old mode 100755 new mode 100644 similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/sddc/sddc/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-configuration/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-configuration/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-configuration/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-configuration/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-configuration/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-configuration/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-configuration/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-configuration/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-configuration/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-target/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-target/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-target/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-target/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-target/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-target/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-target/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-target/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-configuration/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-target/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-configuration/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-target/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-target/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-target/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-target/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-target/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-target/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-target/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-target/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/cloud-guard-target/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/address-list/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/address-list/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/address-list/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/address-list/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/address-list/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/address-list/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/address-list/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/address-list/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-target/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/address-list/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/cloud-guard-target/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/address-list/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/address-list/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/address-list/output.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/address-list/output.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/address-list/output.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/address-list/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/address-list/variable.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/address-list/variable.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/address-list/variable.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application-group/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application-group/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application-group/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application-group/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application-group/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application-group/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application-group/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application-group/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/address-list/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application-group/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/address-list/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application-group/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application-group/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application-group/output.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application-group/output.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application-group/output.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application-group/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application-group/variable.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application-group/variable.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application-group/variable.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application-group/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application-group/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application/output.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application/output.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application/output.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application/variable.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application/variable.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/application/variable.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-profile/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-profile/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-profile/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-profile/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-profile/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-profile/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-profile/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-profile/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-profile/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/application/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-profile/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-profile/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-profile/output.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-profile/output.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-profile/output.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-profile/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-profile/variable.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-profile/variable.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-profile/variable.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-rules/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-rules/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-rules/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-rules/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-rules/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-rules/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-rules/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-rules/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-profile/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-rules/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-profile/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-rules/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-rules/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-rules/output.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-rules/output.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-rules/output.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-rules/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-rules/variable.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-rules/variable.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/decryption-rules/variable.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall-policy/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall-policy/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall-policy/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall-policy/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-rules/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall-policy/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/decryption-rules/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall-policy/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall-policy/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall-policy/output.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall-policy/output.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall-policy/output.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall-policy/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall-policy/variable.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall-policy/variable.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall-policy/variable.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall/main.tf similarity index 90% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall/main.tf index ebfd7ba4e..834e37d5b 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall/main.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall/main.tf @@ -12,6 +12,9 @@ resource "oci_network_firewall_network_firewall" "network_firewall" { network_security_group_ids = var.nsg_id != null ? (local.nsg_id == [] ? ["INVALID NSG Name"] : local.nsg_id) : null defined_tags = var.defined_tags freeform_tags = var.freeform_tags + nat_configuration { + must_enable_private_nat = var.must_enable_private_nat + } lifecycle { ignore_changes = [defined_tags["Oracle-Tags.CreatedOn"], defined_tags["Oracle-Tags.CreatedBy"], defined_tags["SE_Details.SE_Name"]] } diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall-policy/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall-policy/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall/output.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall/output.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall/output.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall/variable.tf similarity index 93% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall/variable.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall/variable.tf index 602ee82ea..e20a3789e 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall/variable.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/firewall/variable.tf @@ -71,11 +71,17 @@ variable "name" { default = null } +variable "must_enable_private_nat" { + type = string + default = "false" +} + variable "defined_tags" { type = map(any) default = {} } + variable "freeform_tags" { type = map(any) default = {} diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/security-rules/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/nat-rules/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/security-rules/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/nat-rules/data.tf diff --git a/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/nat-rules/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/nat-rules/main.tf new file mode 100644 index 000000000..50a3ecfec --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/nat-rules/main.tf @@ -0,0 +1,22 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +resource "oci_network_firewall_network_firewall_policy_nat_rule" "network_firewall_policy_nat_rule" { + lifecycle { + ignore_changes = [position] + } + name = var.rule_name + action = var.action + type = var.type + description = var.description + network_firewall_policy_id = var.network_firewall_policy_id + condition { + destination_address = var.destination_address + source_address = var.source_address + service = var.service + } + position { + after_rule = var.after_rule + before_rule = var.before_rule + } +} diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/nat-rules/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/firewall/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/nat-rules/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/nat-rules/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/nat-rules/output.tf new file mode 100755 index 000000000..a618e41e4 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/nat-rules/output.tf @@ -0,0 +1,6 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +output "nat_rules_tf_id" { + value = oci_network_firewall_network_firewall_policy_nat_rule.network_firewall_policy_nat_rule.id +} \ No newline at end of file diff --git a/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/nat-rules/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/nat-rules/variable.tf new file mode 100644 index 000000000..7963d3131 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/nat-rules/variable.tf @@ -0,0 +1,155 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +variable "compartment_id" { + type = string + default = null +} +variable "subnet_name" { + type = string + default = null +} + +variable "vcn_name" { + type = string + default = null +} +variable "network_firewall_policy_id" { + type = string + default = null +} + +variable "display_name" { + type = string + default = null +} + +variable "ipv4address" { + type = string + default = null +} + +variable "icmp_type" { + type = number + default = null + +} + +variable "icmp_code" { + type = number + default = null +} +variable "minimum_port" { + type = number + default = null +} + +variable "maximum_port" { + type = number + default = null +} + +variable "service_name" { + type = string + default = null +} + +variable "service_type" { + type = string + default = null +} + +variable "region" { + type = string + default = "us-ashburn-1" +} + +variable "type" { + type = string + default = null +} + +variable "name" { + type = string + default = null +} + +variable "policy" { + type = map(any) + default = {} +} + +variable "service_port_ranges" { + type = map(any) + default = {} +} + +variable "key_name" { + type = string + default = null +} + +variable "rule_condition" { + type = map(any) + default = {} +} +variable "rule_position" { + type = map(any) + default = {} +} +variable "key_name1" { + type = string + default = null +} + +variable "key_name2" { + type = string + default = null +} + +variable "rule_name" { + type = string + default = null +} + +variable "action" { + type = string + default = null +} + +variable "service" { + type = string + default = null +} + +variable "description" { + type = string + default = null +} + +variable "destination_address" { + type = list(string) + default = [] +} +variable "source_address" { + type = list(string) + default = [] +} + + +variable "after_rule" { + type = string + default = null +} +variable "before_rule" { + type = string + default = null +} + +variable "protocol" { + type = string + default = null +} + + + diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/secret/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/secret/data.tf similarity index 94% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/secret/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/secret/data.tf index 272cd2a8b..81f4234d3 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/secret/data.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/secret/data.tf @@ -1,7 +1,7 @@ # Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. # -data "oci_kms_vaults" "fw_vault" { +/*data "oci_kms_vaults" "fw_vault" { compartment_id = var.compartment_id != null ? var.compartment_id : var.compartment_id filter { name = "display_name" @@ -18,4 +18,4 @@ data "oci_vault_secrets" "fw_secret" { locals { secret_ocid = tostring(data.oci_vault_secrets.fw_secret.secrets[0].id) -} \ No newline at end of file +}*/ \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/secret/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/secret/main.tf similarity index 92% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/secret/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/secret/main.tf index 8744045b2..e87e848cc 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/secret/main.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/secret/main.tf @@ -7,6 +7,6 @@ resource "oci_network_firewall_network_firewall_policy_mapped_secret" "network_f network_firewall_policy_id = var.network_firewall_policy_id source = var.secret_source type = var.secret_type - vault_secret_id = local.secret_ocid + vault_secret_id = var.vault_secret_id version_number = var.version_number } \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/secret/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/secret/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/secret/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/secret/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/secret/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/secret/output.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/secret/output.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/secret/output.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/secret/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/secret/variable.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/secret/variable.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/secret/variable.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/security-rules/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/security-rules/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/security-rules/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/security-rules/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/security-rules/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/security-rules/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/security-rules/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/security-rules/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/security-rules/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/security-rules/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/security-rules/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/security-rules/output.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/security-rules/output.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/security-rules/output.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/security-rules/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/security-rules/variable.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/security-rules/variable.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/security-rules/variable.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service-list/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service-list/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service-list/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service-list/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service-list/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service-list/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service-list/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service-list/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service-list/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service-list/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service-list/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service-list/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service-list/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service-list/output.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service-list/output.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service-list/output.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service-list/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service-list/variable.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service-list/variable.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service-list/variable.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/tunnel-inspect/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/tunnel-inspect/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service/output.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service/output.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service/output.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service/variable.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/service/variable.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/service/variable.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/url-list/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/tunnel-inspect/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/url-list/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/tunnel-inspect/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/tunnel-inspect/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/tunnel-inspect/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/tunnel-inspect/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/tunnel-inspect/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/tunnel-inspect/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/tunnel-inspect/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/tunnel-inspect/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/tunnel-inspect/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/tunnel-inspect/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/tunnel-inspect/output.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/tunnel-inspect/output.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/tunnel-inspect/output.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/tunnel-inspect/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/tunnel-inspect/variable.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/tunnel-inspect/variable.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/tunnel-inspect/variable.tf diff --git a/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/url-list/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/url-list/data.tf new file mode 100644 index 000000000..84adb07b0 --- /dev/null +++ b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/url-list/data.tf @@ -0,0 +1,11 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +/*locals { + policy_ocid = data.oci_network_firewall_network_firewall_policies.fw-policy.network_firewall_policy_summary_collection[*].id + +} +data "oci_network_firewall_network_firewall_policies" "fw-policy" { + compartment_id = var.compartment_id != null ? (length(regexall("ocid1.compartment.oc*", var.compartment_id)) > 0 ? var.compartment_id : var.compartment_ocids[var.compartment_id]) : var.compartment_ocids[var.compartment_id] + display_name = var.network_firewall_policy_id +*/ \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/url-list/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/url-list/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/url-list/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/url-list/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/url-list/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/url-list/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/url-list/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/url-list/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/url-list/output.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/url-list/output.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/url-list/output.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/url-list/output.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/url-list/variable.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/url-list/variable.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/firewall/url-list/variable.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/firewall/url-list/variable.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/key/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/key/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/key/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/key/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/key/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/key/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/key/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/key/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/key/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/key/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/key/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/key/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/key/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/key/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/key/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/key/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/vault/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/vault/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/vault/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/vault/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/vault/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/vault/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/vault/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/vault/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/vault/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/vault/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/vault/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/vault/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/security/vault/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/security/vault/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/security/vault/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/security/vault/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/block-volume/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/block-volume/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/block-volume/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/block-volume/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/block-volume/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/block-volume/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/block-volume/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/block-volume/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/block-volume/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/block-volume/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/block-volume/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/block-volume/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/block-volume/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/block-volume/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/block-volume/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/block-volume/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/block-volume/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/block-volume/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/block-volume/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/block-volume/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/export-option/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/export-option/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/export-option/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/export-option/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/export-option/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/export-option/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/export-option/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/export-option/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/export-option/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/export-option/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/export-option/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/export-option/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/export-option/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/export-option/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/export-option/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/export-option/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss-replication/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss-replication/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss-replication/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss-replication/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss-replication/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss-replication/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss-replication/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss-replication/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss-replication/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss-replication/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss-replication/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss-replication/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss-replication/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss-replication/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss-replication/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss-replication/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/fss/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/fss/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/mount-target/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/mount-target/data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/mount-target/data.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/mount-target/data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/mount-target/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/mount-target/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/mount-target/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/mount-target/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/mount-target/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/mount-target/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/mount-target/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/mount-target/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/mount-target/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/mount-target/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/mount-target/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/mount-target/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/mount-target/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/mount-target/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/file-storage/mount-target/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/file-storage/mount-target/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/object-storage/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/object-storage/main.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/object-storage/main.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/object-storage/main.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/object-storage/oracle_provider_req.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/object-storage/oracle_provider_req.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/object-storage/oracle_provider_req.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/object-storage/oracle_provider_req.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/object-storage/outputs.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/object-storage/outputs.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/object-storage/outputs.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/object-storage/outputs.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/storage/object-storage/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/storage/object-storage/variables.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/modules/storage/object-storage/variables.tf rename to cd3_automation_toolkit/ocicloud/terraform/modules/storage/object-storage/variables.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/mysql-dbsystem.tf b/cd3_automation_toolkit/ocicloud/terraform/mysql-dbsystem.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/mysql-dbsystem.tf rename to cd3_automation_toolkit/ocicloud/terraform/mysql-dbsystem.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/network.tf b/cd3_automation_toolkit/ocicloud/terraform/network.tf similarity index 93% rename from cd3_automation_toolkit/user-scripts/terraform/network.tf rename to cd3_automation_toolkit/ocicloud/terraform/network.tf index 890d153e7..805c4214f 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/network.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/network.tf @@ -42,6 +42,71 @@ data "oci_core_drg_route_distributions" "drg_route_distributions" { # Module Block - Network # Create VCNs ############################ +locals { + all_byo_details = flatten([ + for vcn_name, vcn in var.vcns : [ + for detail in vcn.byoipv6cidr_details : { + original_value = detail.byoipv6range_data + ipv6cidr_block = detail.ipv6cidr_block + comp_fallback = vcn.comp_name + } + ] if vcn.byoipv6cidr_details != null + ]) + + normalized_byo_details = [ + for item in local.all_byo_details : ( + contains(item.original_value, "@") ? + { + # Parse comp and range_name + comp_raw = try(split(item.original_value, "@")[0], "") + range_name = try(split(item.original_value, "@")[1], "") + compartment_name = (try(split(item.original_value, "@")[0], "") != "") ? try(split(item.original_value, "@")[0], "") : item.comp_fallback + cidr = item.ipv6cidr_block + # Use comp@range as unique key + key = split(item.original_value, "@")[1] + } : + { + # No '@' means it is a range_id directly (assume no comp, range_name unknown) + compartment_name = "" # unknown + range_name = item.original_value # treat entire string as range_id + cidr = item.ipv6cidr_block + key = item.original_value # use range_id as key + } + ) + ] + + # Deduplicate by key + unique_list_map = { + for item in local.normalized_byo_details : item.key => { + compartment_name = item.compartment_name + range_name = item.range_name + cidr = item.cidr + } + } + + unique_list = values(local.unique_list_map) + vcn_byoip = { + for k,v in var.vcns : k => { +byoipv6cidr_details = [ +for item in v.byoipv6cidr_details : { +byoipv6range_id = length(regexall("ocid1.byoiprange.oc*",item.range_name)) > 0?item.range_name:data.oci_core_byoip_ranges.byoip[item.range_name].byoip_range_collection.*.id[0] +ipv6cidr_block = item.cidr +} +] +} if v.byoipv6cidr_details != null +} +} + +data "oci_core_byoip_ranges" "byoip" { + for_each = { + for item in local.unique_list : + item.range_name => item if !contains(item.range_name, "ocid1.byoiprange.oc") + } + + compartment_id = length(regexall("ocid1.compartment.oc*", each.value.compartment_name)) > 0 ? each.value.compartment_name : var.compartment_ocids[each.value.compartment_name] + display_name = each.value.range_name +} + module "vcns" { source = "./modules/network/vcn" @@ -54,13 +119,13 @@ module "vcns" { #Optional cidr_blocks = each.value.cidr_blocks display_name = each.value.display_name - byoipv6cidr_details = each.value.byoipv6cidr_details != null ? each.value.byoipv6cidr_details : [] + byoipv6cidr_details = each.value.byoipv6cidr_details != null ? local.vcn_byoip.byoipv6cidr_details : [] dns_label = (each.value.dns_label == "n") ? null : each.value.dns_label is_ipv6enabled = each.value.is_ipv6enabled # Defaults to false by terraform hashicorp defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags - ipv6private_cidr_blocks = each.value.ipv6private_cidr_blocks - is_oracle_gua_allocation_enabled = each.value.is_oracle_gua_allocation_enabled + ipv6private_cidr_blocks = each.value.ipv6private_cidr_blocks != null ? each.value.ipv6private_cidr_blocks : [] + is_oracle_gua_allocation_enabled = each.value.is_oracle_gua_allocation_enabled != null ? each.value.is_oracle_gua_allocation_enabled : false } diff --git a/cd3_automation_toolkit/user-scripts/terraform/networkloadbalancer.tf b/cd3_automation_toolkit/ocicloud/terraform/networkloadbalancer.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/networkloadbalancer.tf rename to cd3_automation_toolkit/ocicloud/terraform/networkloadbalancer.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/nsg.tf b/cd3_automation_toolkit/ocicloud/terraform/nsg.tf similarity index 93% rename from cd3_automation_toolkit/user-scripts/terraform/nsg.tf rename to cd3_automation_toolkit/ocicloud/terraform/nsg.tf index b9554b7c8..31898e855 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/nsg.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/nsg.tf @@ -21,7 +21,7 @@ module "nsgs" { #Required compartment_id = each.value.compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id]) : null - vcn_id = flatten(data.oci_core_vcns.oci_vcns_nsgs[each.key].virtual_networks.*.id)[0] + vcn_id = length(regexall("ocid1.vcn.oc*", each.value.vcn_name)) > 0 ? each.value.vcn_name :flatten(data.oci_core_vcns.oci_vcns_nsgs[each.key].virtual_networks.*.id)[0] defined_tags = each.value.defined_tags display_name = each.value.display_name freeform_tags = each.value.freeform_tags diff --git a/cd3_automation_toolkit/user-scripts/terraform/object-storage.tf b/cd3_automation_toolkit/ocicloud/terraform/object-storage.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/object-storage.tf rename to cd3_automation_toolkit/ocicloud/terraform/object-storage.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/oci-data.tf b/cd3_automation_toolkit/ocicloud/terraform/oci-data.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/oci-data.tf rename to cd3_automation_toolkit/ocicloud/terraform/oci-data.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/oke.tf b/cd3_automation_toolkit/ocicloud/terraform/oke.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/oke.tf rename to cd3_automation_toolkit/ocicloud/terraform/oke.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/provider.tf b/cd3_automation_toolkit/ocicloud/terraform/provider.tf similarity index 96% rename from cd3_automation_toolkit/user-scripts/terraform/provider.tf rename to cd3_automation_toolkit/ocicloud/terraform/provider.tf index 34c1879f0..ceb8f14cf 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/provider.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/provider.tf @@ -19,7 +19,7 @@ terraform { required_providers { oci = { source = "oracle/oci" - version = "7.8.0" + version = "7.19.0" } } } diff --git a/cd3_automation_toolkit/user-scripts/terraform/quota.tf b/cd3_automation_toolkit/ocicloud/terraform/quota.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/quota.tf rename to cd3_automation_toolkit/ocicloud/terraform/quota.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/scripts/bastion-ssh-key b/cd3_automation_toolkit/ocicloud/terraform/scripts/bastion-ssh-key similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/scripts/bastion-ssh-key rename to cd3_automation_toolkit/ocicloud/terraform/scripts/bastion-ssh-key diff --git a/cd3_automation_toolkit/user-scripts/terraform/scripts/default.sh b/cd3_automation_toolkit/ocicloud/terraform/scripts/default.sh similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/scripts/default.sh rename to cd3_automation_toolkit/ocicloud/terraform/scripts/default.sh diff --git a/cd3_automation_toolkit/user-scripts/terraform/scripts/default.yaml b/cd3_automation_toolkit/ocicloud/terraform/scripts/default.yaml similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/scripts/default.yaml rename to cd3_automation_toolkit/ocicloud/terraform/scripts/default.yaml diff --git a/cd3_automation_toolkit/user-scripts/terraform/scripts/server-ssh-key b/cd3_automation_toolkit/ocicloud/terraform/scripts/server-ssh-key similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/scripts/server-ssh-key rename to cd3_automation_toolkit/ocicloud/terraform/scripts/server-ssh-key diff --git a/cd3_automation_toolkit/user-scripts/terraform/sddc.tf b/cd3_automation_toolkit/ocicloud/terraform/sddc.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/sddc.tf rename to cd3_automation_toolkit/ocicloud/terraform/sddc.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/tagging.tf b/cd3_automation_toolkit/ocicloud/terraform/tagging.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/tagging.tf rename to cd3_automation_toolkit/ocicloud/terraform/tagging.tf diff --git a/cd3_automation_toolkit/user-scripts/terraform/variables_example.tf b/cd3_automation_toolkit/ocicloud/terraform/variables_example.tf similarity index 95% rename from cd3_automation_toolkit/user-scripts/terraform/variables_example.tf rename to cd3_automation_toolkit/ocicloud/terraform/variables_example.tf index f788d1086..c4c8d692a 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/variables_example.tf +++ b/cd3_automation_toolkit/ocicloud/terraform/variables_example.tf @@ -955,26 +955,71 @@ variable "databases" { variable "adb" { type = map(object({ - admin_password = optional(string) - character_set = optional(string) - compartment_id = string - cpu_core_count = optional(number) - database_edition = optional(string) - data_storage_size_in_tbs = optional(number) - customer_contacts = optional(list(string)) - db_name = string - db_version = optional(string) - db_workload = optional(string) - display_name = optional(string) - license_model = optional(string) - ncharacter_set = optional(string) - network_compartment_id = optional(string) - nsg_ids = optional(list(string)) - subnet_id = optional(string) - vcn_name = optional(string) - whitelisted_ips = optional(list(string)) - defined_tags = optional(map(any)) - freeform_tags = optional(map(any)) + admin_password = optional(string) + compartment_id = string + are_primary_whitelisted_ips_used = optional(bool) + auto_refresh_frequency_in_seconds = optional(number) + auto_refresh_point_lag_in_seconds = optional(number) + adb_source = optional(string) + source_id = optional(string) + #source detail used as source_id + autonomous_database_source_backup_id = optional(string) + autonomous_database_id = optional(string) + #storage + is_auto_scaling_for_storage_enabled = optional(bool) + data_storage_size_in_gb = optional(number) + data_storage_size_in_tbs = optional(number) + autonomous_maintenance_schedule_type = optional(string) + character_set = optional(string) + compute_count = number + compute_model = string + ocpu_count = optional(number) + customer_contacts = optional(list(string)) + data_safe_status = optional(string) + database_edition = optional(string) + db_name = string + db_version = optional(string) + db_workload = optional(string) + display_name = optional(string) + is_auto_scaling_enabled = optional(bool) + #Dedicated Exadata Infrastructure + is_dedicated = optional(bool) + autonomous_container_database_id = optional(string) + + # TDE MEK + kms_key_id = optional(string) + # ADB customer managed key + vault_id = optional(string) + # Only to Autonomous Databases on the Exadata Cloud@Customer platform + in_memory_percentage = optional(number) + + is_local_data_guard_enabled = optional(bool) + is_mtls_connection_required = optional(bool) + tde_kms_key_id = optional(string) + license_model = optional(string) + ncharacter_set = optional(string) + private_endpoint_ip = optional(string) + private_endpoint_label = optional(string) + refreshable_mode = optional(string) + time_of_auto_refresh_start = optional(string) + # Network + network_compartment_id = optional(string) + subnet_compartment_id = optional(string) + subnet_id = optional(string) + vcn_name = optional(string) + nsg_ids = optional(list(string)) + #Backup + backup_retention_period_in_days = optional(number) + is_backup_retention_locked = optional(bool) + #DisasterRecoveryConfiguration + is_replicate_automatic_backups = optional(bool) + remote_disaster_recovery_type = optional(string) + ##source=BACKUP_FROM_TIMESTAMP + timestamp = optional(string) + use_latest_available_backup_time_stamp = optional(bool) + whitelisted_ips = optional(list(string)) + defined_tags = optional(map(any)) + freeform_tags = optional(map(any)) })) default = {} } @@ -1721,10 +1766,11 @@ variable "events" { default = {} } -variable "notifications_topics" { +variable "notifications" { type = map(object({ compartment_id = string topic_name = string + subscriptions = optional (list(map(any))) description = optional(string) defined_tags = optional(map(any)) freeform_tags = optional(map(any)) @@ -1732,17 +1778,6 @@ variable "notifications_topics" { default = {} } -variable "notifications_subscriptions" { - type = map(object({ - compartment_id = string - endpoint = string - protocol = string - topic_id = string - defined_tags = optional(map(any)) - freeform_tags = optional(map(any)) - })) - default = {} -} variable "service_connectors" { type = any @@ -2081,6 +2116,7 @@ variable "firewalls" { availability_domain = optional(string) defined_tags = optional(map(any)) freeform_tags = optional(map(any)) + must_enable_private_nat = optional(string) })) default = {} } @@ -2215,11 +2251,8 @@ variable "decryption_rules" { rule_name = string network_firewall_policy_id = string condition = optional(list(object({ - - destination_address = optional(list(string)) - - source_address = optional(list(string)) - + destination_address = optional(list(string)) + source_address = optional(list(string)) }))) decryption_profile = optional(string) secret = optional(string) @@ -2245,6 +2278,24 @@ variable "tunnelinspect_rules" { })) default = {} } + +variable "nat_rules" { + type = map(object({ + action = string + type = string + rule_name = string + network_firewall_policy_id = string + condition = optional(list(object({ + destination_address = optional(list(string)) + source_address = optional(list(string)) + service = string + }))) + description = optional(string) + after_rule = optional(string) + before_rule = optional(string) + })) + default = {} +} ######################### ####### Firewall Logs ######## ######################### diff --git a/cd3_automation_toolkit/user-scripts/terraform/vlan.tf b/cd3_automation_toolkit/ocicloud/terraform/vlan.tf similarity index 100% rename from cd3_automation_toolkit/user-scripts/terraform/vlan.tf rename to cd3_automation_toolkit/ocicloud/terraform/vlan.tf diff --git a/cd3_automation_toolkit/setUpAzure.properties b/cd3_automation_toolkit/setUpAzure.properties new file mode 100644 index 000000000..8391c2dbf --- /dev/null +++ b/cd3_automation_toolkit/setUpAzure.properties @@ -0,0 +1,25 @@ +[Default] + +#Input variables required to run setUpAzure script + +#path to output directory where terraform files will be generated. eg. /cd3user/azure//terraform_files +outdir= + +#prefix for output terraform files eg demo +prefix= + +# Auth Params +subscription_id= + +tenant_id= + +client_id= + +client_secret= + +#path to cd3 excel eg /cd3user/azure//CD3-Customer.xlsx +cd3file= + +#specify create_resources to create new resources in Azure(greenfield workflow) +#specify export_resources to export resources from Azure(non-greenfield workflow) +workflow_type=create_resources \ No newline at end of file diff --git a/cd3_automation_toolkit/setUpCloud.py b/cd3_automation_toolkit/setUpCloud.py new file mode 100644 index 000000000..b32919603 --- /dev/null +++ b/cd3_automation_toolkit/setUpCloud.py @@ -0,0 +1,30 @@ +import sys +import subprocess + + +def main(): + if len(sys.argv) != 3: + print("Usage: python setUpCloud.py ") + print("Example: python setUpCloud.py azure setUpAzure.properties") + print("Example: python setUpCloud.py oci setUpOCI.properties") + return + + cloud_provider = sys.argv[1].lower() + argument = sys.argv[2] + + if cloud_provider == 'oci': + script_name = 'setUpOCI.py' + elif cloud_provider == 'azure': + script_name = 'user-scripts/setUpAzure.py' + else: + print("Invalid cloud provider. Use 'azure' or 'oci'.") + return + + try: + subprocess.run([sys.executable, script_name, argument], check=True) + except subprocess.CalledProcessError as e: + pass + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/cd3_automation_toolkit/setUpOCI.properties b/cd3_automation_toolkit/setUpOCI.properties index 1a0e8e52b..59715cca3 100644 --- a/cd3_automation_toolkit/setUpOCI.properties +++ b/cd3_automation_toolkit/setUpOCI.properties @@ -2,16 +2,16 @@ #Input variables required to run setUpOCI script -#path to output directory where terraform files will be generated. eg. /cd3user/tenancies//terraform_files +#path to output directory where terraform files will be generated. eg. /cd3user/tenancies//terraform_files outdir= -#prefix for output terraform files eg like demo +#prefix for output terraform files eg like demo prefix= # auth mechanism for OCI APIs - api_key,instance_principal,session_token auth_mechanism= -#input config file for Python API communication with OCI eg /cd3user/tenancies//.config_files/_config; +#input config file for Python API communication with OCI eg /cd3user/tenancies//.config_files/_config; config_file= # Leave it blank if you want single outdir or specify outdir_structure_file.properties containing directory structure for OCI services. @@ -20,9 +20,9 @@ outdir_structure_file= # IaC Tool to be configured - Terraform(specify terraform) or OpenTofu(specify tofu) tf_or_tofu= -#path to cd3 excel eg /cd3user/tenancies//CD3-Customer.xlsx +#path to cd3 excel eg /cd3user/tenancies//CD3-Customer.xlsx cd3file= #specify create_resources to create new resources in OCI(greenfield workflow) #specify export_resources to export resources from OCI(non-greenfield workflow) -workflow_type=create_resources \ No newline at end of file +workflow_type=export_resources \ No newline at end of file diff --git a/cd3_automation_toolkit/setUpOCI.py b/cd3_automation_toolkit/setUpOCI.py index 8025eabc9..950d693b5 100644 --- a/cd3_automation_toolkit/setUpOCI.py +++ b/cd3_automation_toolkit/setUpOCI.py @@ -1,24 +1,12 @@ import argparse import configparser import json -import Database -import Identity -import Compute -import ManagementServices -import DeveloperServices -import Security -import cd3Validator -import cd3FirewallValidator -import Storage -import Network -import SDDC -import Governance -import CostManagement -from commonTools import * from collections import namedtuple import requests import subprocess -import datetime,glob,os +import glob +from ocicloud.python import * +from common.python import * def show_firewall_options(options, quit=False, menu=False, extra=None, index=0): # Just add whitespace between number and option. It just makes it look better @@ -244,7 +232,7 @@ def fetch_compartments(outdir, outdir_struct, ct): var_files={} var_data = {} home_region = ct.home_region - print("outdir specified should contain region directories and then variables_.tf file inside the region directories eg /cd3user/tenancies//terraform_files") + print("outdir specified should contain region directories and then variables_.tf file inside the region directories eg /cd3user/tenancies//terraform_files") print("Verifying out directory and Taking backup of existing variables files...Please wait...") print("\nFetching Compartment Info...Please wait...") ct.get_network_compartment_ids(config['tenancy'], "root", config, signer) @@ -391,7 +379,7 @@ def export_identityOptions(prim_options=[]): def export_compartments(inputfile, outdir,config, signer, ct): resource = 'Compartments' - Identity.export_identity(inputfile, outdir, service_dir_identity,resource, config, signer, ct) + export_identity(inputfile, outdir, service_dir_identity,resource, config, signer, ct) options = [Option(None, create_compartments, 'Processing Compartments Tab'), ] execute_options(options) print("\n\nExecute import_commands_compartments.sh script created under home region directory to synch TF with OCI Identity Compartments\n") @@ -399,7 +387,7 @@ def export_compartments(inputfile, outdir,config, signer, ct): def export_policies(inputfile, outdir,config, signer, ct): resource = 'IAM Policies' #compartments = ct.get_compartment_map(var_file, resource) - Identity.export_identity(inputfile, outdir, service_dir_identity,resource, config, signer, ct, export_compartments=compartments) + export_identity(inputfile, outdir, service_dir_identity,resource, config, signer, ct, export_compartments=compartments) options = [Option(None, create_policies, 'Processing Policies Tab'), ] execute_options(options) print("\n\nExecute import_commands_policies.sh script created under home region directory to synch TF with OCI " +resource +"\n") @@ -407,7 +395,7 @@ def export_policies(inputfile, outdir,config, signer, ct): def export_groups(inputfile, outdir,config, signer, ct): resource = 'IAM Groups' selected_domains_data = ct.get_identity_domain_data(config, signer, resource,var_file) - Identity.export_identity(inputfile, outdir, service_dir_identity,resource, config, signer, ct, export_domains=selected_domains_data) + export_identity(inputfile, outdir, service_dir_identity,resource, config, signer, ct, export_domains=selected_domains_data) options = [Option(None, create_groups, 'Processing Groups Tab'), ] execute_options(options) print("\n\nExecute import_commands_groups.sh script created under home region directory to synch TF with OCI " +resource +"\n") @@ -417,16 +405,16 @@ def export_users(inputfile, outdir,config,signer, ct): resource = 'IAM Users' # check if tenancy is identity_domain enabled selected_domains_data = ct.get_identity_domain_data(config, signer, resource,var_file) - Identity.Users.export_users(inputfile, outdir, service_dir_identity, config, signer, ct,export_domains=selected_domains_data) - options = [Option(None, Identity.Users.create_terraform_users, 'Processing Users Tab'), ] + identity.export_users(inputfile, outdir, service_dir_identity, config, signer, ct,export_domains=selected_domains_data) + options = [Option(None, create_terraform_users, 'Processing Users Tab'), ] execute_options(options,inputfile, outdir,service_dir_identity, prefix, ct) print("\n\nExecute import_commands_users.sh script created under home region directory to synch TF with OCI " +resource +"\n") def export_networkSources(inputfile, outdir, config, signer, ct): resource = 'Network Sources' - Identity.NetworkSources.export_networkSources(inputfile, outdir, service_dir_identity, config, signer, ct) - options = [Option(None, Identity.NetworkSources.create_terraform_networkSources, 'Processing NetworkSources Tab'), ] + identity.export_networkSources(inputfile, outdir, service_dir_identity, config, signer, ct) + options = [Option(None, create_terraform_networkSources, 'Processing NetworkSources Tab'), ] execute_options(options, inputfile, outdir, service_dir_identity, prefix, ct) print("\n\nExecute import_commands_networkSources.sh script created under home region directory to synch TF with OCI " +resource +"\n") @@ -442,7 +430,7 @@ def export_governance(prim_options=[]): def export_tags(prim_options=[]): #compartments = ct.get_compartment_map(var_file, 'Tagging Objects') - Governance.export_tags_nongreenfield(inputfile, outdir, service_dir_tagging, config, signer, ct, export_compartments=compartments) + export_tags_nongreenfield(inputfile, outdir, service_dir_tagging, config, signer, ct, export_compartments=compartments) options = [Option(None, create_tags, 'Processing Tags Tab'), ] execute_options(options) print("\n\nExecute import_commands_tags.sh script created under home region directory to synch TF with OCI Tags\n") @@ -450,7 +438,7 @@ def export_tags(prim_options=[]): update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_tagging]) def export_quotas(prim_options=[]): - Governance.export_quotas_nongreenfield(inputfile, outdir, service_dir_quota, config, signer, ct, export_tags=export_tags_list) + export_quotas_nongreenfield(inputfile, outdir, service_dir_quota, config, signer, ct, export_tags=export_tags_list) options = [Option(None, create_quotas, 'Processing Quotas Tab'), ] execute_options(options) print("\n\nExecute import_commands_quotas.sh script created under home region directory to synch TF with OCI Quota\n") @@ -468,7 +456,7 @@ def export_cost_management(prim_options=[]): execute_options(options) def export_budget(prim_options=[]): - CostManagement.export_budgets_nongreenfield(inputfile, outdir, service_dir_budget, config, signer, ct,export_regions,export_tags_list) + export_budgets_nongreenfield(inputfile, outdir, service_dir_budget, config, signer, ct,export_regions,export_tags_list) options = [Option(None, create_budgets, 'Processing Budgets Tab')] execute_options(options) print("\n\nExecute import_commands_budgets.sh script created under each region directory to synch TF with OCI Tags\n") @@ -512,34 +500,34 @@ def export_networking(inputfile, outdir,config, signer, ct, export_regions,expor service_dirs = [] service_dir = outdir_struct #compartments = ct.get_compartment_map(var_file,'Network Objects') - Network.export_networking(inputfile, outdir, service_dir,config, signer, ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) - options = [ Option(None, Network.create_major_objects, 'Processing VCNs and DRGs Tab'), ] + network.export_networking(inputfile, outdir, service_dir,config, signer, ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) + options = [ Option(None, create_major_objects, 'Processing VCNs and DRGs Tab'), ] execute_options(options, inputfile, outdir, service_dir_network, prefix, ct, non_gf_tenancy) - options = [ Option(None, Network.create_rpc_resource, 'Processing RPCs in DRGs Tab'),] + options = [ Option(None, create_rpc_resource, 'Processing RPCs in DRGs Tab'),] execute_options(options, inputfile, outdir, service_dir_network, prefix, auth_mechanism, config_file_path, ct, non_gf_tenancy) options = [ - Option(None, Network.create_terraform_dhcp_options, 'Processing DHCP Tab'), - Option(None, Network.modify_terraform_secrules, 'Processing SecRulesinOCI Tab'), - Option(None, Network.modify_terraform_routerules, 'Processing RouteRulesinOCI Tab'), - Option(None, Network.modify_terraform_drg_routerules, 'Processing DRGRouteRulesinOCI Tab'), + Option(None, create_terraform_dhcp_options, 'Processing DHCP Tab'), + Option(None, modify_terraform_secrules, 'Processing SecRulesinOCI Tab'), + Option(None, modify_terraform_routerules, 'Processing RouteRulesinOCI Tab'), + Option(None, modify_terraform_drg_routerules, 'Processing DRGRouteRulesinOCI Tab'), ] execute_options(options, inputfile, outdir, service_dir_network, prefix, ct, non_gf_tenancy) options = [ - Option(None, Network.create_terraform_drg_route,'Processing DRGs tab for DRG Route Tables and Route Distribution creation'), + Option(None, create_terraform_drg_route,'Processing DRGs tab for DRG Route Tables and Route Distribution creation'), ] execute_options(options, inputfile, outdir, service_dir_network, prefix, ct, non_gf_tenancy, network_connectivity_in_setupoci='', modify_network=False) - options = [ Option(None, Network.create_terraform_subnet_vlan, 'Processing SubnetsVLANs Tab for Subnets'), ] + options = [ Option(None, create_terraform_subnet_vlan, 'Processing SubnetsVLANs Tab for Subnets'), ] execute_options(options, inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy, network_vlan_in_setupoci='network') - options = [ Option(None, Network.create_terraform_subnet_vlan, 'Processing SubnetsVLANs Tab for VLANs'), ] + options = [ Option(None, create_terraform_subnet_vlan, 'Processing SubnetsVLANs Tab for VLANs'), ] execute_options(options, inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy, network_vlan_in_setupoci='vlan') - options = [ Option(None, Network.create_terraform_nsg, 'Processing NSGs Tab'), ] + options = [ Option(None, create_terraform_nsg, 'Processing NSGs Tab'), ] execute_options(options, inputfile, outdir, service_dir_nsg, prefix, ct) print("\n\nExecute import_commands_network_*.sh script created under each region directory to synch TF with OCI Network objects\n") for service in [service_dir_network,service_dir_vlan,service_dir_nsg]: @@ -547,25 +535,25 @@ def export_networking(inputfile, outdir,config, signer, ct, export_regions,expor def export_major_objects(inputfile, outdir, config, signer, ct, export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file,'VCN Major Objects') - Network.export_major_objects(inputfile, outdir, service_dir_network, config, signer, ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) - Network.export_drg_routetable(inputfile, outdir, service_dir_network, config, signer, ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list, _tf_import_cmd=True) + network.export_major_objects(inputfile, outdir, service_dir_network, config, signer, ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) + export_drg_routetable(inputfile, outdir, service_dir_network, config, signer, ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list, _tf_import_cmd=True) options = [ - Option(None, Network.create_major_objects, 'Processing VCNs and DRGs Tab'), + Option(None, create_major_objects, 'Processing VCNs and DRGs Tab'), ] execute_options(options, inputfile, outdir,service_dir_network, prefix, ct, non_gf_tenancy) options = [ - Option(None, Network.create_rpc_resource, 'Processing RPCs in DRGs Tab'), + Option(None, create_rpc_resource, 'Processing RPCs in DRGs Tab'), ] execute_options(options, inputfile, outdir, service_dir_network, prefix, auth_mechanism, config_file_path, ct, non_gf_tenancy) options = [ - Option(None, Network.create_terraform_drg_route,'Processing DRGs tab for DRG Route Tables and Route Distribution creation'), + Option(None, create_terraform_drg_route,'Processing DRGs tab for DRG Route Tables and Route Distribution creation'), ] execute_options(options, inputfile, outdir, service_dir_network, prefix, ct, non_gf_tenancy,network_connectivity_in_setupoci='', modify_network=False) options = [ - Option(None, Network.modify_terraform_drg_routerules, 'Processing DRGRouteRulesinOCI Tab'), + Option(None, modify_terraform_drg_routerules, 'Processing DRGRouteRulesinOCI Tab'), ] execute_options(options, inputfile, outdir, service_dir_network, prefix, ct, non_gf_tenancy) @@ -574,27 +562,27 @@ def export_major_objects(inputfile, outdir, config, signer, ct, export_regions,e def export_dhcp(inputfile, outdir,config,signer,ct,export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file,'DHCP') - Network.export_dhcp(inputfile, outdir, service_dir_network,config, signer, ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) + network.export_dhcp(inputfile, outdir, service_dir_network,config, signer, ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) options = [ - Option(None, Network.create_terraform_dhcp_options, 'Processing DHCP Tab'), + Option(None, create_terraform_dhcp_options, 'Processing DHCP Tab'), ] execute_options(options, inputfile, outdir, service_dir_network,prefix, ct, non_gf_tenancy) print("\n\nExecute import_commands_network_dhcp.sh script created under each region directory to synch TF with OCI Network objects\n") def export_secrules(inputfile, outdir,config,signer,ct,export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file,'SecRulesInOCI') - Network.export_seclist(inputfile, outdir, service_dir_network, config, signer, ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list,_tf_import_cmd=True) + export_seclist(inputfile, outdir, service_dir_network, config, signer, ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list,_tf_import_cmd=True) options = [ - Option(None, Network.modify_terraform_secrules, 'Processing SecRulesinOCI Tab'), + Option(None, modify_terraform_secrules, 'Processing SecRulesinOCI Tab'), ] execute_options(options, inputfile, outdir,service_dir_network, prefix, ct, non_gf_tenancy) print("\n\nExecute import_commands_network_secrules.sh script created under each region directory to synch TF with OCI Network objects\n") def export_routerules(inputfile, outdir,config,signer,ct,export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file,'RouteRulesInOCI') - Network.export_routetable(inputfile, outdir, service_dir_network, config, signer, ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list,_tf_import_cmd=True) + export_routetable(inputfile, outdir, service_dir_network, config, signer, ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list,_tf_import_cmd=True) options = [ - Option(None, Network.modify_terraform_routerules, 'Processing RouteRulesinOCI Tab'), + Option(None, modify_terraform_routerules, 'Processing RouteRulesinOCI Tab'), ] execute_options(options, inputfile, outdir, service_dir_network,prefix, ct, non_gf_tenancy) print("\n\nExecute import_commands_network_routerules.sh script created under each region directory to synch TF with OCI Network objects\n") @@ -603,15 +591,15 @@ def export_routerules(inputfile, outdir,config,signer,ct,export_regions,export_t def export_subnets_vlans(inputfile, outdir,config,signer,ct,export_regions,export_tags_list): service_dir = outdir_struct #compartments = ct.get_compartment_map(var_file,'Subnets') - Network.export_subnets_vlans(inputfile, outdir, service_dir,config, signer, ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) + network.export_subnets_vlans(inputfile, outdir, service_dir,config, signer, ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) options = [ - Option(None, Network.create_terraform_subnet_vlan, 'Processing SubnetsVLANs Tab for Subnets'), + Option(None, create_terraform_subnet_vlan, 'Processing SubnetsVLANs Tab for Subnets'), ] execute_options(options, inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy, network_vlan_in_setupoci='network') options = [ - Option(None, Network.create_terraform_subnet_vlan, 'Processing SubnetsVLANs Tab for VLANs'), + Option(None, create_terraform_subnet_vlan, 'Processing SubnetsVLANs Tab for VLANs'), ] execute_options(options, inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy, network_vlan_in_setupoci='vlan') @@ -622,9 +610,9 @@ def export_subnets_vlans(inputfile, outdir,config,signer,ct,export_regions,expor def export_nsg(inputfile, outdir,config,signer,ct,export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file,'NSGs') - Network.export_nsg(inputfile, outdir,service_dir_nsg, config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list,_tf_import_cmd=True) + network.export_nsg(inputfile, outdir,service_dir_nsg, config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list,_tf_import_cmd=True) options = [ - Option(None, Network.create_terraform_nsg, 'Processing NSGs Tab'), + Option(None, create_terraform_nsg, 'Processing NSGs Tab'), ] execute_options(options, inputfile, outdir, service_dir_nsg,prefix, ct) print("\n\nExecute import_commands_network_nsg.sh script created under each region directory to synch TF with OCI Network objects\n") @@ -650,13 +638,13 @@ def export_firewallpolicy(inputfile, outdir, config, signer, ct, export_regions, policy_name_str = ct.fwl_pol_pattern_filter if ct.fwl_pol_pattern_filter else None policies = list(map(lambda x: x.strip(), policy_name_str.split(','))) if policy_name_str else None - Security.export_firewallpolicy(inputfile, outdir, service_dir_firewall, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list,export_policies=policies) + security.export_firewallpolicy(inputfile, outdir, service_dir_firewall, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list,export_policies=policies) create_firewall_policy(inputfile, outdir, service_dir_firewall, prefix, ct,execute_all=True) print("\n\nExecute import_commands_firewallpolicy.sh script created under each region directory to synch TF with OCI Firewall policy objects\n") def export_firewalls(inputfile, outdir, config, signer, ct, export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file, 'Firewalls') - Security.export_firewall(inputfile, outdir, service_dir_firewall, config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list) + security.export_firewall(inputfile, outdir, service_dir_firewall, config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list) create_firewall(inputfile, outdir, service_dir_firewall, prefix, ct) print("\n\nExecute import_commands_firewall.sh script created under each region directory to synch TF with OCI Firewall policy objects\n") @@ -673,7 +661,7 @@ def export_compute(prim_options=[],export_all=False): def export_dedicatedvmhosts(inputfile, outdir, config, signer, ct, export_regions,export_tags_list,export_all): #compartments = ct.get_compartment_map(var_file,'Dedicated VM Hosts') - Compute.export_dedicatedvmhosts(inputfile, outdir, service_dir_dedicated_vm_host, config, signer, ct, export_compartments=compartments, export_regions=export_regions, export_tags = export_tags_list) + compute.export_dedicatedvmhosts(inputfile, outdir, service_dir_dedicated_vm_host, config, signer, ct, export_compartments=compartments, export_regions=export_regions, export_tags = export_tags_list) #create_compute(prim_options=['Add/Modify/Delete Dedicated VM Hosts']) options = [Option(None, create_dedicatedvmhosts, 'Processing Dedicated VM Hosts Tab'),] execute_options(options) @@ -702,7 +690,7 @@ def export_instances(inputfile, outdir,config,signer, ct, export_regions,export_ display_names = list(map(lambda x: x.strip(), display_name_str.split(','))) if display_name_str else None ad_names = list(map(lambda x: x.strip(), ad_name_str.split(','))) if ad_name_str else None - Compute.export_instances(inputfile, outdir, service_dir_instance,config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags = export_tags_list, display_names = display_names, ad_names = ad_names) + compute.export_instances(inputfile, outdir, service_dir_instance,config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags = export_tags_list, display_names = display_names, ad_names = ad_names) options = [Option(None, create_instances, 'Processing Instances Tab'), ] execute_options(options) print("\n\nExecute import_commands_instances.sh script created under each region directory to synch TF with OCI Instances\n") @@ -739,7 +727,7 @@ def export_block_volumes(inputfile, outdir,config,signer,ct, export_regions,expo display_names = list(map(lambda x: x.strip(), display_name_str.split(','))) if display_name_str else None ad_names = list(map(lambda x: x.strip(), ad_name_str.split(','))) if ad_name_str else None - Storage.export_blockvolumes(inputfile, outdir, service_dir_block_volume, config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags = export_tags_list, display_names = display_names, ad_names = ad_names) + storage.export_blockvolumes(inputfile, outdir, service_dir_block_volume, config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags = export_tags_list, display_names = display_names, ad_names = ad_names) options = [Option(None, create_block_volumes, 'Processing BlockVolumes Tab'), ] execute_options(options) print("\n\nExecute import_commands_blockvolumes.sh script created under each region directory to synch TF with OCI Block Volume Objects\n") @@ -749,7 +737,7 @@ def export_block_volumes(inputfile, outdir,config,signer,ct, export_regions,expo def export_fss(inputfile, outdir,config, signer, ct, export_regions,export_tags_list,export_all): #compartments = ct.get_compartment_map(var_file,'FSS objects') - Storage.export_fss(inputfile, outdir, service_dir_fss, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags = export_tags_list) + storage.export_fss(inputfile, outdir, service_dir_fss, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags = export_tags_list) options = [Option(None, create_fss, 'Processing FSS Tab'), ] execute_options(options) print("\n\nExecute import_commands_fss.sh script created under each region directory to synch TF with OCI FSS objects\n") @@ -759,7 +747,7 @@ def export_fss(inputfile, outdir,config, signer, ct, export_regions,export_tags_ def export_buckets(inputfile, outdir, config, signer, ct, export_regions,export_tags_list,export_all): #compartments = ct.get_compartment_map(var_file, 'Buckets') - Storage.export_buckets(inputfile, outdir, service_dir_object_storage, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags = export_tags_list) + storage.export_buckets(inputfile, outdir, service_dir_object_storage, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags = export_tags_list) options = [Option(None, create_buckets, 'Processing Buckets Tab'), ] execute_options(options) print("\n\nExecute import_commands_buckets.sh script created under each region directory to synch TF with OCI Object Storage Buckets\n") @@ -778,7 +766,7 @@ def export_loadbalancer(prim_options=[]): def export_lbr(inputfile, outdir,config, signer, ct, export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file,'LBR objects') - Network.export_lbr(inputfile, outdir, service_dir_loadbalancer, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) + network.export_lbr(inputfile, outdir, service_dir_loadbalancer, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) options = [Option(None, create_lb, 'Processing LBaaS Tabs'), ] execute_options(options) print("\n\nExecute import_commands_lbr.sh script created under each region directory to synch TF with OCI LBR objects\n") @@ -788,7 +776,7 @@ def export_lbr(inputfile, outdir,config, signer, ct, export_regions,export_tags_ def export_nlb(inputfile, outdir,config,signer, ct, export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file,'NLB objects') - Network.export_nlb(inputfile, outdir, service_dir_networkloadbalancer, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) + network.export_nlb(inputfile, outdir, service_dir_networkloadbalancer, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) options = [Option(None, create_nlb, 'Processing NLB Tabs'), ] execute_options(options) print("\n\nExecute import_commands_nlb.sh script created under each region directory to synch TF with OCI NLB objects\n") @@ -805,7 +793,7 @@ def export_security(prim_options=[]): def export_kms(inputfile, outdir, config, signer, ct, export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file, 'KMS') - Security.export_keyvaults(inputfile, outdir, service_dir_kms, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) + export_keyvaults(inputfile, outdir, service_dir_kms, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) options = [Option(None, create_kms, 'Processing KMS Tab')] execute_options(options) print("\n\nExecute import_commands_kms.sh script created under each region directory to synch TF with OCI Key Vaults\n") @@ -826,7 +814,7 @@ def export_databases(prim_options=[]): def export_dbsystems_vm_bm(inputfile, outdir,config,signer, ct,export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file,'VM and BM DB Systems') - Database.export_dbsystems_vm_bm(inputfile, outdir, service_dir_dbsystem_vm_bm, config,signer,ct, export_compartments=compartments, export_regions= export_regions,export_tags=export_tags_list) + database.export_dbsystems_vm_bm(inputfile, outdir, service_dir_dbsystem_vm_bm, config,signer,ct, export_compartments=compartments, export_regions= export_regions,export_tags=export_tags_list) options = [Option(None, create_dbsystems_vm_bm, 'Processing DBSystems-VM-BM Tab')] execute_options(options) print("\n\nExecute import_commands_dbsystems-vm-bm.sh script created under each region directory to synch TF with DBSystems\n") @@ -836,8 +824,8 @@ def export_dbsystems_vm_bm(inputfile, outdir,config,signer, ct,export_regions,ex def export_exa_infra_vmclusters(inputfile, outdir,config, signer, ct, export_regions, export_tags_list): #compartments = ct.get_compartment_map(var_file,'EXA Infra and EXA VMClusters') - Database.export_exa_infra(inputfile, outdir, service_dir_database_exacs, config,signer,ct, export_compartments=compartments, export_regions= export_regions,export_tags = export_tags_list) - Database.export_exa_vmclusters(inputfile, outdir, service_dir_database_exacs, config,signer,ct, export_compartments=compartments, export_regions= export_regions, export_tags=export_tags_list) + export_exa_infra(inputfile, outdir, service_dir_database_exacs, config,signer,ct, export_compartments=compartments, export_regions= export_regions,export_tags = export_tags_list) + export_exa_vmclusters(inputfile, outdir, service_dir_database_exacs, config,signer,ct, export_compartments=compartments, export_regions= export_regions, export_tags=export_tags_list) options = [Option(None, create_exa_infra_vmclusters, '')] execute_options(options) print("\n\nExecute import_commands_exa-infra.sh and import_commands_exa-vmclusters.sh scripts created under each region directory to synch TF with Exa-Infra and Exa-VMClusters\n") @@ -847,7 +835,7 @@ def export_exa_infra_vmclusters(inputfile, outdir,config, signer, ct, export_reg def export_adbs(inputfile, outdir,config, signer, ct, export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file,'ADBs') - Database.export_adbs(inputfile, outdir, service_dir_adb, config,signer,ct, export_compartments=compartments, export_regions= export_regions, export_tags=export_tags_list) + database.export_adbs(inputfile, outdir, service_dir_adb, config,signer,ct, export_compartments=compartments, export_regions= export_regions, export_tags=export_tags_list) options = [Option(None, create_adb, 'Processing ADB Tab')] execute_options(options) print("\n\nExecute import_commands_adb.sh script created under each region directory to synch TF with OCI ADBs\n") @@ -857,9 +845,9 @@ def export_adbs(inputfile, outdir,config, signer, ct, export_regions,export_tags def export_mysql(inputfile, outdir,config,signer, ct,export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file, 'MySQL DB Systems and Configurations') - Database.export_mysql_db(inputfile, outdir, service_dir_mysql_dbsystem, config, signer, ct, + export_mysql_db(inputfile, outdir, service_dir_mysql_dbsystem, config, signer, ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list) - Database.export_mysql_configuration(inputfile, outdir, service_dir_mysql_dbsystem, config, signer, ct, + export_mysql_configuration(inputfile, outdir, service_dir_mysql_dbsystem, config, signer, ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list) options = [Option(None, create_mysql, '')] execute_options(options,execute_all=True) @@ -883,25 +871,25 @@ def export_management_services(prim_options=[]): def export_notifications(inputfile, outdir, service_dir, config, signer, ct, export_regions, export_tags_list): #compartments = ct.get_compartment_map(var_file,'Notifications') - ManagementServices.export_notifications(inputfile, outdir, service_dir, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) + managementservices.export_notifications(inputfile, outdir, service_dir, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) create_management_services(prim_options=['Add/Modify/Delete Notifications']) print("\n\nExecute import_commands_notifications.sh script created under each region directory to synch TF with OCI Notifications\n") def export_events(inputfile, outdir, service_dir, config, signer, ct, export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file,'Events') - ManagementServices.export_events(inputfile, outdir, service_dir, config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list) + managementservices.export_events(inputfile, outdir, service_dir, config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list) create_management_services(prim_options=['Add/Modify/Delete Events']) print("\n\nExecute import_commands_events.sh script created under each region directory to synch TF with OCI Events\n") def export_alarms(inputfile, outdir, service_dir, config, signer, ct, export_regions, export_tags_list): #compartments = ct.get_compartment_map(var_file,'Alarms') - ManagementServices.export_alarms(inputfile, outdir, service_dir, config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list) + managementservices.export_alarms(inputfile, outdir, service_dir, config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list) create_management_services(prim_options=['Add/Modify/Delete Alarms']) print("\n\nExecute import_commands_alarms.sh script created under each region directory to synch TF with OCI Alarms\n") def export_service_connectors(inputfile, outdir, service_dir, config, signer, ct, export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file,'Service Connectors') - ManagementServices.export_service_connectors(inputfile, outdir, service_dir, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) + managementservices.export_service_connectors(inputfile, outdir, service_dir, config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) create_management_services(prim_options=['Add/Modify/Delete ServiceConnectors']) print("\n\nExecute import_commands_serviceconnectors.sh script created under each region directory to synch TF with OCI Service Connectors\n") @@ -916,7 +904,7 @@ def export_developer_services(prim_options=[]): def export_oke(inputfile, outdir, config,signer, ct, export_regions,export_tags_list): #compartments = ct.get_compartment_map(var_file,'OKE') - DeveloperServices.export_oke(inputfile, outdir, service_dir_oke,config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list) + developerservices.export_oke(inputfile, outdir, service_dir_oke,config,signer,ct, export_compartments=compartments, export_regions=export_regions, export_tags=export_tags_list) options = [Option(None, create_oke, 'Processing OKE Tab')] execute_options(options,inputfile, outdir, prefix, '', '', ct) print("\n\nExecute import_commands_oke.sh script created under each region directory to synch TF with OKE\n") @@ -926,8 +914,8 @@ def export_oke(inputfile, outdir, config,signer, ct, export_regions,export_tags_ def export_sddc(prim_options=[]): #compartments = ct.get_compartment_map(var_file,'SDDCs') - SDDC.export_sddc(inputfile, outdir, service_dir_sddc,config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) - SDDC.create_terraform_sddc(inputfile, outdir, service_dir_sddc, prefix, ct) + sddc.export_sddc(inputfile, outdir, service_dir_sddc,config,signer,ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) + create_terraform_sddc(inputfile, outdir, service_dir_sddc, prefix, ct) print("\n\nExecute import_commands_sddcs.sh script created under each region directory to synch TF with SDDC\n") # Update modified path list update_path_list(regions_path=export_regions, service_dirs=[service_dir_sddc]) @@ -962,12 +950,12 @@ def export_dns_views_zones_rrsets(inputfile, outdir, service_dir, config, signer if ct.default_dns.lower() == "true": dns_filter = "y" dns_filter = dns_filter if dns_filter else None - Network.export_dns_views_zones_rrsets(inputfile, outdir, service_dir, config, signer, ct, dns_filter=dns_filter, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) + network.export_dns_views_zones_rrsets(inputfile, outdir, service_dir, config, signer, ct, dns_filter=dns_filter, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) options = [Option(None, create_dns, 'Processing DNS-Views-Zones-Records Tab')] execute_options(options) def export_dns_resolvers(inputfile, outdir, service_dir, config, signer, ct, export_regions, export_tags_list,export_all): #compartments = ct.get_compartment_map(var_file, 'DNS Resolvers') - Network.export_dns_resolvers(inputfile, outdir, service_dir, config, signer, ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) + network.export_dns_resolvers(inputfile, outdir, service_dir, config, signer, ct, export_compartments=compartments, export_regions=export_regions,export_tags=export_tags_list) options = [Option(None, create_dns_resolvers, 'Processing DNS-Resolvers Tab')] execute_options(options) @@ -993,7 +981,7 @@ def create_compartments(): choices = [Option("Validate Compartments", None, None)] errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct) if errors == False: - Identity.create_terraform_compartments(inputfile, outdir, service_dir_identity, prefix, ct) + create_terraform_compartments(inputfile, outdir, service_dir_identity, prefix, ct) # Update modified path list update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_identity]) else: @@ -1006,7 +994,7 @@ def create_groups(): choices = [Option("Validate Groups", None, None)] errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct) if errors == False: - Identity.create_terraform_groups(inputfile, outdir, service_dir_identity, prefix, ct) + create_terraform_groups(inputfile, outdir, service_dir_identity, prefix, ct) # Update modified path list update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_identity]) else: @@ -1020,7 +1008,7 @@ def create_policies(): errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct) if errors == False: - Identity.create_terraform_policies(inputfile, outdir, service_dir_identity, prefix, ct) + create_terraform_policies(inputfile, outdir, service_dir_identity, prefix, ct) # Update modified path list update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_identity]) else: @@ -1028,13 +1016,13 @@ def create_policies(): exit(1) def create_users(): - Identity.create_terraform_users(inputfile, outdir, service_dir_identity, prefix, ct) + create_terraform_users(inputfile, outdir, service_dir_identity, prefix, ct) # Update modified path list update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_identity]) def create_networksources(): - Identity.NetworkSources.create_terraform_networkSources(inputfile, outdir, service_dir_identity, prefix, ct) + create_terraform_networkSources(inputfile, outdir, service_dir_identity, prefix, ct) # Update modified path list update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_identity]) @@ -1076,7 +1064,7 @@ def create_tags(): errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct) if errors == False: - Governance.create_terraform_tags(inputfile, outdir, service_dir_tagging, prefix, ct) + create_terraform_tags(inputfile, outdir, service_dir_tagging, prefix, ct) # Update modified path list update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_tagging]) else: @@ -1084,7 +1072,7 @@ def create_tags(): exit(1) def create_quotas(): - Governance.create_terraform_quotas(inputfile, outdir, service_dir_quota, prefix, ct) + create_terraform_quotas(inputfile, outdir, service_dir_quota, prefix, ct) # Update modified path list update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_quota]) @@ -1099,13 +1087,13 @@ def create_cost_management(prim_options=[]): def create_budgets(): - errors = True + errors = False if (workflow_type == 'create_resources'): choices = [Option("Validate Budgets", None, None)] errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct) - if errors == True: - CostManagement.create_terraform_budgets(inputfile, outdir, service_dir_budget, prefix, ct) + if errors == False: + create_terraform_budgets(inputfile, outdir, service_dir_budget, prefix, ct) # Update modified path list update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_budget]) else: @@ -1142,7 +1130,7 @@ def create_terraform_network(inputfile, outdir, service_dir, prefix, ct, non_gf errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct) if errors == False: - Network.create_all_tf_objects(inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy=non_gf_tenancy) + create_all_tf_objects(inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy=non_gf_tenancy) else: print("Please correct the errors in CD3 Sheet and try again. Exiting!!!") exit(1) @@ -1155,7 +1143,7 @@ def modify_terraform_network(inputfile, outdir, service_dir, prefix, ct, non_gf errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct) if errors == False: - Network.create_all_tf_objects(inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy=non_gf_tenancy, modify_network=True, ) + create_all_tf_objects(inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy=non_gf_tenancy, modify_network=True, ) else: print("Please correct the errors in CD3 Sheet and try again. Exiting!!!") exit(1) @@ -1164,7 +1152,7 @@ def export_modify_security_rules(inputfile, outdir, service_dir, prefix, ct, non execute_all = False options = [ Option('Export Security Rules (From OCI into SecRulesinOCI sheet)', export_security_rules, 'Exporting Security Rules in OCI'), - Option('Add/Modify/Delete Security Rules (Reads SecRulesinOCI sheet)', Network.modify_terraform_secrules, 'Processing SecRulesinOCI Tab'), + Option('Add/Modify/Delete Security Rules (Reads SecRulesinOCI sheet)', modify_terraform_secrules, 'Processing SecRulesinOCI Tab'), ] if sub_options: options = match_options(options, sub_options) @@ -1184,13 +1172,13 @@ def export_modify_security_rules(inputfile, outdir, service_dir, prefix, ct, non def export_security_rules(inputfile, outdir, service_dir, config, signer, ct, non_gf_tenancy): compartments = ct.get_compartment_map(var_file, 'OCI Security Rules') export_tags_list = get_tags_list('OCI Security Rules') - Network.export_seclist(inputfile, outdir, service_dir, config, signer, ct, export_compartments=compartments, export_regions= export_regions, export_tags=export_tags_list, _tf_import_cmd=False) + export_seclist(inputfile, outdir, service_dir, config, signer, ct, export_compartments=compartments, export_regions= export_regions, export_tags=export_tags_list, _tf_import_cmd=False) def export_modify_route_rules(inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy,sub_options=[]): execute_all = False options = [ Option('Export Route Rules (From OCI into RouteRulesinOCI sheet)', export_route_rules, 'Exporting Route Rules in OCI'), - Option('Add/Modify/Delete Route Rules (Reads RouteRulesinOCI sheet)', Network.modify_terraform_routerules, 'Processing RouteRulesinOCI Tab'), + Option('Add/Modify/Delete Route Rules (Reads RouteRulesinOCI sheet)', modify_terraform_routerules, 'Processing RouteRulesinOCI Tab'), ] if sub_options: options = match_options(options, sub_options) @@ -1210,13 +1198,13 @@ def export_modify_route_rules(inputfile, outdir, service_dir, prefix, ct, non_gf def export_route_rules(inputfile, outdir, service_dir, config, signer, ct, non_gf_tenancy): compartments = ct.get_compartment_map(var_file, 'OCI Route Rules') export_tags_list = get_tags_list('OCI Route Rules') - Network.export_routetable(inputfile, outdir, service_dir, config, signer, ct, export_compartments=compartments, export_regions= export_regions, export_tags=export_tags_list,_tf_import_cmd=False) + export_routetable(inputfile, outdir, service_dir, config, signer, ct, export_compartments=compartments, export_regions= export_regions, export_tags=export_tags_list,_tf_import_cmd=False) def export_modify_drg_route_rules(inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy,sub_options=[]): execute_all = False options = [ Option('Export DRG Route Rules (From OCI into DRGRouteRulesinOCI sheet)', export_drg_route_rules, 'Exporting DRG Route Rules in OCI'), - Option('Add/Modify/Delete DRG Route Rules (Reads DRGRouteRulesinOCI sheet)', Network.modify_terraform_drg_routerules, 'Processing DRGRouteRulesinOCI Tab'), + Option('Add/Modify/Delete DRG Route Rules (Reads DRGRouteRulesinOCI sheet)', modify_terraform_drg_routerules, 'Processing DRGRouteRulesinOCI Tab'), ] if sub_options: options = match_options(options, sub_options) @@ -1236,14 +1224,14 @@ def export_modify_drg_route_rules(inputfile, outdir, service_dir, prefix, ct, no def export_drg_route_rules(inputfile, outdir, service_dir, config, signer, ct, non_gf_tenancy): compartments = ct.get_compartment_map(var_file,'OCI DRG Route Rules') export_tags_list = get_tags_list('OCI DRG Route Rules') - Network.export_drg_routetable(inputfile, outdir, service_dir, config, signer, ct, export_compartments=compartments, export_regions= export_regions, export_tags=export_tags_list,_tf_import_cmd=False) + export_drg_routetable(inputfile, outdir, service_dir, config, signer, ct, export_compartments=compartments, export_regions= export_regions, export_tags=export_tags_list,_tf_import_cmd=False) def export_modify_nsgs(inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy,sub_options=[]): execute_all = False options = [ Option('Export NSGs (From OCI into NSGs sheet)', export_nsgs, 'Exporting NSGs in OCI'), - Option('Add/Modify/Delete NSGs (Reads NSGs sheet)', Network.create_terraform_nsg, 'Processing NSGs Tab'), + Option('Add/Modify/Delete NSGs (Reads NSGs sheet)', create_terraform_nsg, 'Processing NSGs Tab'), ] if sub_options: options = match_options(options, sub_options) @@ -1255,11 +1243,11 @@ def export_modify_nsgs(inputfile, outdir, service_dir, prefix, ct, non_gf_tenanc def export_nsgs(inputfile, outdir, service_dir, prefix, ct): compartments = ct.get_compartment_map(var_file,'OCI NSGs') export_tags_list = get_tags_list('OCI NSGs') - Network.export_nsg(inputfile, outdir, service_dir, config, signer, ct, export_compartments=compartments, export_regions= export_regions, export_tags=export_tags_list, _tf_import_cmd=False) + network.export_nsg(inputfile, outdir, service_dir, config, signer, ct, export_compartments=compartments, export_regions= export_regions, export_tags=export_tags_list, _tf_import_cmd=False) def create_vlans(inputfile, outdir, service_dir, prefix,ct, non_gf_tenancy, network_vlan_in_setupoci='vlan'): - Network.create_terraform_subnet_vlan(inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy=non_gf_tenancy, network_vlan_in_setupoci='vlan',modify_network=True) - Network.create_terraform_route(inputfile, outdir, service_dir_network, prefix, ct, non_gf_tenancy=non_gf_tenancy, network_vlan_in_setupoci='vlan',modify_network=True) + create_terraform_subnet_vlan(inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy=non_gf_tenancy, network_vlan_in_setupoci='vlan',modify_network=True) + create_terraform_route(inputfile, outdir, service_dir_network, prefix, ct, non_gf_tenancy=non_gf_tenancy, network_vlan_in_setupoci='vlan',modify_network=True) def create_drg_connectivity(inputfile, outdir, service_dir, prefix, ct, non_gf_tenancy,network_vlan_in_setupoci='vlan',sub_options=[]): execute_all = False @@ -1275,8 +1263,8 @@ def create_drg_connectivity(inputfile, outdir, service_dir, prefix, ct, non_gf_ execute_options(options, inputfile, outdir, service_dir, service_dir_network, prefix, auth_mechanism, config_file_path, ct, non_gf_tenancy=non_gf_tenancy) def create_rpc(inputfile, outdir, service_dir, service_dir_network, prefix, auth_mechanism, config_file_path, ct, non_gf_tenancy): - Network.create_rpc_resource(inputfile, outdir, service_dir, prefix, auth_mechanism, config_file_path, ct, non_gf_tenancy=non_gf_tenancy) - Network.create_terraform_drg_route(inputfile, outdir, service_dir_network, prefix, non_gf_tenancy=non_gf_tenancy, ct=ct, network_connectivity_in_setupoci='connectivity', modify_network=True) + create_rpc_resource(inputfile, outdir, service_dir, prefix, auth_mechanism, config_file_path, ct, non_gf_tenancy=non_gf_tenancy) + create_terraform_drg_route(inputfile, outdir, service_dir_network, prefix, non_gf_tenancy=non_gf_tenancy, ct=ct, network_connectivity_in_setupoci='connectivity', modify_network=True) def create_compute(prim_options=[]): options = [ @@ -1297,7 +1285,7 @@ def create_instances(): errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct) if errors == False: - Compute.create_terraform_instances(inputfile, outdir, service_dir_instance, prefix, ct) + create_terraform_instances(inputfile, outdir, service_dir_instance, prefix, ct) # Update modified path list update_path_list(regions_path=subscribed_regions, service_dirs=[service_dir_instance]) else: @@ -1305,7 +1293,7 @@ def create_instances(): exit(1) def create_dedicatedvmhosts(): - Compute.create_terraform_dedicatedhosts(inputfile, outdir, service_dir_dedicated_vm_host, prefix, ct) + create_terraform_dedicatedhosts(inputfile, outdir, service_dir_dedicated_vm_host, prefix, ct) # Update modified path list update_path_list(regions_path=subscribed_regions, service_dirs=[service_dir_dedicated_vm_host]) @@ -1331,7 +1319,7 @@ def create_block_volumes(): errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct) if errors == False: - Storage.create_terraform_block_volumes(inputfile, outdir, service_dir_block_volume, prefix, ct) + create_terraform_block_volumes(inputfile, outdir, service_dir_block_volume, prefix, ct) # Update modified path list update_path_list(regions_path=subscribed_regions, service_dirs=[service_dir_block_volume]) else: @@ -1345,7 +1333,7 @@ def create_fss(): errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct) if errors == False: - Storage.create_terraform_fss(inputfile, outdir, service_dir_fss, prefix, ct) + create_terraform_fss(inputfile, outdir, service_dir_fss, prefix, ct) # Update modified path list update_path_list(regions_path=subscribed_regions, service_dirs=[service_dir_fss]) else: @@ -1360,7 +1348,7 @@ def create_buckets(): errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct) if errors == False: - Storage.create_terraform_oss(inputfile, outdir, service_dir_object_storage, prefix, ct) + create_terraform_oss(inputfile, outdir, service_dir_object_storage, prefix, ct) # Update modified path list update_path_list(regions_path=subscribed_regions, service_dirs=[service_dir_object_storage]) else: @@ -1382,12 +1370,12 @@ def create_loadbalancer(execute_all=False,prim_options=[]): def create_lb(): options = [ - Option(None, Network.create_terraform_lbr_hostname_certs, 'Creating LBR'), - Option(None, Network.create_backendset_backendservers, 'Creating Backend Sets and Backend Servers'), - Option(None, Network.create_listener, 'Creating Listeners'), - Option(None, Network.create_path_route_set, 'Creating Path Route Sets'), - Option(None, Network.create_ruleset, 'Creating Rule Sets'), - Option(None, Network.create_lb_routing_policy, 'Creating Routing Policies'), + Option(None, create_terraform_lbr_hostname_certs, 'Creating LBR'), + Option(None, create_backendset_backendservers, 'Creating Backend Sets and Backend Servers'), + Option(None, create_listener, 'Creating Listeners'), + Option(None, create_path_route_set, 'Creating Path Route Sets'), + Option(None, create_ruleset, 'Creating Rule Sets'), + Option(None, create_lb_routing_policy, 'Creating Routing Policies'), ] execute_options(options, inputfile, outdir, service_dir_loadbalancer, prefix, ct) # Update modified path list @@ -1396,8 +1384,8 @@ def create_lb(): def create_nlb(): options = [ - Option(None, Network.create_terraform_nlb_listener, 'Creating NLB and Listeners'), - Option(None, Network.create_nlb_backendset_backendservers, 'Creating NLB Backend Sets and Backend Servers'), + Option(None, create_terraform_nlb_listener, 'Creating NLB and Listeners'), + Option(None, create_nlb_backendset_backendservers, 'Creating NLB Backend Sets and Backend Servers'), ] execute_options(options, inputfile, outdir, service_dir_networkloadbalancer, prefix, ct) # Update modified path list @@ -1418,28 +1406,28 @@ def create_databases(execute_all=False,prim_options=[]): execute_options(options) def create_dbsystems_vm_bm(): - Database.create_terraform_dbsystems_vm_bm(inputfile, outdir, service_dir_dbsystem_vm_bm, prefix, ct) + create_terraform_dbsystems_vm_bm(inputfile, outdir, service_dir_dbsystem_vm_bm, prefix, ct) # Update modified path list update_path_list(regions_path=subscribed_regions, service_dirs=[service_dir_dbsystem_vm_bm]) def create_exa_infra_vmclusters(): - options = [Option(None, Database.create_terraform_exa_infra, 'Processing Exa-Infra Tab'), - Option(None, Database.create_terraform_exa_vmclusters, 'Processing Exa-VM-Clusters Tab')] + options = [Option(None, create_terraform_exa_infra, 'Processing Exa-Infra Tab'), + Option(None, create_terraform_exa_vmclusters, 'Processing Exa-VM-Clusters Tab')] execute_options(options, inputfile, outdir, service_dir_database_exacs, prefix, ct) # Update modified path list update_path_list(regions_path=subscribed_regions, service_dirs=[service_dir_database_exacs]) def create_adb(): - Database.create_terraform_adb(inputfile, outdir, service_dir_adb, prefix, ct) + create_terraform_adb(inputfile, outdir, service_dir_adb, prefix, ct) # Update modified path list update_path_list(regions_path=subscribed_regions, service_dirs=[service_dir_adb]) def create_mysql(execute_all=False,prim_options=[]): options = [ - Option('Add/Modify/Delete MySQL DB Systems', Database.create_terraform_mysql_db, 'Processing MySQL-DBSystems Tab'), - Option('Add/Modify/Delete MySQL Configurations', Database.create_terraform_mysql_configuration, 'Processing MySQL-Configurations Tab'), + Option('Add/Modify/Delete MySQL DB Systems', create_terraform_mysql_db, 'Processing MySQL-DBSystems Tab'), + Option('Add/Modify/Delete MySQL Configurations', create_terraform_mysql_configuration, 'Processing MySQL-Configurations Tab'), ] execute_options(options, inputfile, outdir, service_dir_mysql_dbsystem, prefix, ct) # Update modified path list @@ -1448,10 +1436,10 @@ def create_mysql(execute_all=False,prim_options=[]): def create_management_services(execute_all=False,prim_options=[]): options = [ - Option("Add/Modify/Delete Notifications", ManagementServices.create_terraform_notifications, 'Setting up Notifications'), - Option("Add/Modify/Delete Events", ManagementServices.create_terraform_events, 'Setting up Events'), - Option("Add/Modify/Delete Alarms", ManagementServices.create_terraform_alarms, 'Setting up Alarms'), - Option("Add/Modify/Delete ServiceConnectors", ManagementServices.create_service_connectors, + Option("Add/Modify/Delete Notifications", create_terraform_notifications, 'Setting up Notifications'), + Option("Add/Modify/Delete Events", create_terraform_events, 'Setting up Events'), + Option("Add/Modify/Delete Alarms", create_terraform_alarms, 'Setting up Alarms'), + Option("Add/Modify/Delete ServiceConnectors", create_service_connectors, 'Setting up SCHs'), ] if prim_options: @@ -1484,16 +1472,16 @@ def create_developer_services(execute_all=False,prim_options=[]): def create_rm_stack(inputfile, outdir, prefix, auth_mechanism, config_file, ct): regions = get_region_list(rm = True, vizoci = False) - DeveloperServices.create_resource_manager(outdir,var_file, outdir_struct, prefix, auth_mechanism, config_file, ct, regions) + create_resource_manager(outdir,var_file, outdir_struct, prefix, auth_mechanism, config_file, ct, regions) def create_oke(inputfile, outdir, prefix, dummy1, dummy2, ct): - DeveloperServices.create_terraform_oke(inputfile, outdir, service_dir_oke, prefix, ct) + create_terraform_oke(inputfile, outdir, service_dir_oke, prefix, ct) # Update modified path list update_path_list(regions_path=subscribed_regions, service_dirs=[service_dir_oke]) def create_sddc(prim_options=[]): - SDDC.create_terraform_sddc(inputfile, outdir, service_dir_sddc, prefix, ct) + create_terraform_sddc(inputfile, outdir, service_dir_sddc, prefix, ct) # Update modified path list update_path_list(regions_path=subscribed_regions, service_dirs=[service_dir_sddc]) @@ -1524,20 +1512,20 @@ def create_dns_management(prim_options=[]): def create_dns(): - Network.create_terraform_dns_views(inputfile, outdir, service_dir_dns, prefix, ct) - Network.create_terraform_dns_zones(inputfile, outdir, service_dir_dns, prefix, ct) - Network.create_terraform_dns_rrsets(inputfile, outdir, service_dir_dns, prefix, ct) + create_terraform_dns_views(inputfile, outdir, service_dir_dns, prefix, ct) + create_terraform_dns_zones(inputfile, outdir, service_dir_dns, prefix, ct) + create_terraform_dns_rrsets(inputfile, outdir, service_dir_dns, prefix, ct) def create_dns_resolvers(): - Network.create_terraform_dns_resolvers(inputfile, outdir, service_dir_dns, prefix, ct) + create_terraform_dns_resolvers(inputfile, outdir, service_dir_dns, prefix, ct) def create_logging(prim_options=[]): options = [ - Option('Enable VCN Flow Logs', ManagementServices.enable_service_logging, 'VCN Flow Logs'), - Option('Enable LBaaS Logs', ManagementServices.enable_service_logging, 'LBaaS Logs'), - Option('Enable Object Storage Buckets Logs', ManagementServices.enable_service_logging, 'OSS Logs'), - Option('Enable File Storage Logs', ManagementServices.enable_service_logging, 'File Storage Logs'), - Option('Enable Network Firewall Logs', ManagementServices.enable_service_logging, 'Network Firewall Logs') + Option('Enable VCN Flow Logs', enable_service_logging, 'VCN Flow Logs'), + Option('Enable LBaaS Logs', enable_service_logging, 'LBaaS Logs'), + Option('Enable Object Storage Buckets Logs', enable_service_logging, 'OSS Logs'), + Option('Enable File Storage Logs', enable_service_logging, 'File Storage Logs'), + Option('Enable Network Firewall Logs', enable_service_logging, 'Network Firewall Logs') ] if prim_options: options = match_options(options, prim_options) @@ -1569,7 +1557,7 @@ def create_kms(): choices = [Option("Validate KMS", None, None)] errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct) if errors == False: - Security.create_terraform_keyvaults(inputfile, outdir, service_dir_kms, prefix, ct) + create_terraform_keyvaults(inputfile, outdir, service_dir_kms, prefix, ct) # Update modified path list update_path_list(regions_path=subscribed_regions, service_dirs=[service_dir_kms]) else: @@ -1616,7 +1604,7 @@ def enable_cis_cloudguard(): else: region = ct.cg_region region = region.lower() - Security.enable_cis_cloudguard(outdir, service_dir_cloud_guard, prefix, ct, region) + security.enable_cis_cloudguard(outdir, service_dir_cloud_guard, prefix, ct, region) # Update modified path list update_path_list(regions_path=subscribed_regions, service_dirs=[service_dir_cloud_guard]) @@ -2006,7 +1994,7 @@ def clone_firewall_policy( inputfile, outdir, service_dir, config, signer, ct): print("Source Policies are mandatory for cloning. ") exit(1) target_policies = list(map(lambda x: x.strip(), target_policy_str.split(','))) if target_policy_str else None - Security.export_firewallpolicy(inputfile, outdir, service_dir, config, signer, ct, + security.export_firewallpolicy(inputfile, outdir, service_dir, config, signer, ct, export_compartments=compartments, export_regions=export_regions, export_policies=src_policies,target_policies=target_policies,attached_policy_only=attached_policy_only,clone_policy=True) @@ -2039,29 +2027,31 @@ def delete_firewall_policy(inputfile, outdir, service_dir, config, signer, ct): def create_firewall_policy(inputfile, outdir, service_dir, prefix, ct,execute_all=False,sub_options=[]): options = [ Option('Execute All', None, 'Processing all tabs related to Firewall-Policy'), - Option('Add/Modify/Delete Policy', Security.firewallpolicy_create, 'Processing Firewall-Policy Tab'), - Option('Add/Modify/Delete Service', Security.fwpolicy_create_service, + Option('Add/Modify/Delete Policy', firewallpolicy_create, 'Processing Firewall-Policy Tab'), + Option('Add/Modify/Delete Service', fwpolicy_create_service, 'Processing Firewall-Policy-ServicesList Tab'), - Option('Add/Modify/Delete Service-list', Security.fwpolicy_create_servicelist, + Option('Add/Modify/Delete Service-list', fwpolicy_create_servicelist, 'Processing Firewall-Policy-ServiceList Tab'), - Option('Add/Modify/Delete Application', Security.fwpolicy_create_apps, + Option('Add/Modify/Delete Application', fwpolicy_create_apps, 'Processing Firewall-Policy-ApplicationList Tab'), - Option('Add/Modify/Delete Application-list', Security.fwpolicy_create_applicationlist, + Option('Add/Modify/Delete Application-list', fwpolicy_create_applicationlist, 'Processing Firewall-Policy-ApplicationList Tab'), - Option('Add/Modify/Delete Address-list', Security.fwpolicy_create_address, + Option('Add/Modify/Delete Address-list', fwpolicy_create_address, 'Processing Firewall-Policy-AddressList Tab'), - Option('Add/Modify/Delete Url-list', Security.fwpolicy_create_urllist, + Option('Add/Modify/Delete Url-list', fwpolicy_create_urllist, 'Processing Firewall-Policy-UrlList Tab'), - Option('Add/Modify/Delete Security rules', Security.fwpolicy_create_secrules, + Option('Add/Modify/Delete Security rules', fwpolicy_create_secrules, 'Processing Firewall-Policy-SecRule Tab'), - Option('Add/Modify/Delete Mapped Secrets', Security.fwpolicy_create_secret, + Option('Add/Modify/Delete Mapped Secrets', fwpolicy_create_secret, 'Processing Firewall-Policy-Secret Tab'), - Option('Add/Modify/Delete Decryption Rules', Security.fwpolicy_create_decryptrules, + Option('Add/Modify/Delete Decryption Rules', fwpolicy_create_decryptrules, 'Processing Firewall-Policy-DecryptRule Tab'), - Option('Add/Modify/Delete Decryption Profile', Security.fwpolicy_create_decryptionprofile, + Option('Add/Modify/Delete Decryption Profile', fwpolicy_create_decryptionprofile, 'Processing Firewall-Policy-Decryption Tab'), - Option('Add/Modify/Delete Tunnel Inspection Rules', Security.fwpolicy_create_tunnelinspect, + Option('Add/Modify/Delete Tunnel Inspection Rules', fwpolicy_create_tunnelinspect, 'Processing Firewall-Policy-TunnelInspect Tab'), + Option('Add/Modify/Delete NAT Rules', fwpolicy_create_natrules, + 'Processing Firewall-Policy-NatRules Tab'), ] if sub_options and sub_options != ['']: options = match_options(options, sub_options) @@ -2073,7 +2063,7 @@ def create_firewall_policy(inputfile, outdir, service_dir, prefix, ct,execute_al def create_firewall(inputfile, outdir, service_dir, prefix, ct,sub_options=[]): - Security.fw_create(inputfile, outdir, service_dir, prefix, ct) + fw_create(inputfile, outdir, service_dir, prefix, ct) #Execution starts here @@ -2152,7 +2142,7 @@ def create_firewall(inputfile, outdir, service_dir, prefix, ct,sub_options=[]): ## Authenticate Params ct=None -ct = commonTools() +ct = ociCommonTools() config,signer = ct.authenticate(auth_mechanism, config_file_path) ct.setInputParameters(prefix,outdir,inputfile,tf_or_tofu) @@ -2202,6 +2192,7 @@ def create_firewall(inputfile, outdir, service_dir, prefix, ct,sub_options=[]): # Set service directories as per outdir_structure file # If single outdir, get service names from /cd3user/oci_tools/cd3_automation_toolkit/user-scripts/.outdir_structure_file.properties +#print("Creating out dir variables...\n") if len(outdir_struct.items())==0: single_outdir_config = configparser.RawConfigParser() outdir_config_file = os.path.dirname(os.path.abspath(__file__))+"/user-scripts/.outdir_structure_file.properties" @@ -2215,6 +2206,7 @@ def create_firewall(inputfile, outdir, service_dir, prefix, ct,sub_options=[]): varname = "service_dir_"+str(key.replace("-","_")).strip() exec(varname + "= value",globals()) +#print("Reading varfile for compartments....\n") var_file = (f'{outdir}/{home_region}/{service_dir_identity}/variables_{home_region}.tf').replace('//','/') try: diff --git a/cd3_automation_toolkit/shell_script.sh b/cd3_automation_toolkit/shell_script.sh index 1ef4f8fbd..e5f527b94 100644 --- a/cd3_automation_toolkit/shell_script.sh +++ b/cd3_automation_toolkit/shell_script.sh @@ -13,7 +13,10 @@ sudo dnf install python-pip -y #sudo ln -s /usr/bin/pip3 /usr/bin/pip # Install required Python packages -pip install --user oci-cli==3.62.2 +pip install --user oci-cli==3.66.1 +pip install --user azure.identity +pip install --user azure.mgmt.compute +pip install --user azure.mgmt.oracledatabase pip install --user pycryptodomex==3.10.1 pip install --user regex==2022.10.31 pip install --user numpy==1.26.4 @@ -33,10 +36,10 @@ pip install --user PyYAML==6.0.1 pip install --user ansible==8.7.0 # Add Python3 site-packages to PYTHONPATH -echo "export PYTHONPATH=\${PYTHONPATH}:/root/.local/lib/python3.9/site-packages/:/cd3user/.local/lib/python3.9/site-packages/" >> /cd3user/.bashrc +echo "export PYTHONPATH=\${PYTHONPATH}:/root/.local/lib/python3.9/site-packages/:/home/cd3user/.local/lib/python3.9/site-packages/" >> /home/cd3user/.bashrc # Add Python binaries to PATH -echo "PATH=\$PATH:/cd3user/.local/bin" >> /cd3user/.bashrc +#echo "PATH=\$PATH:/cd3user/.local/bin" >> /cd3user/.bashrc # Download and install Terraform diff --git a/cd3_automation_toolkit/user-scripts/.outdir_structure_file.properties b/cd3_automation_toolkit/user-scripts/.outdir_structure_file.properties new file mode 100644 index 000000000..ca979e620 --- /dev/null +++ b/cd3_automation_toolkit/user-scripts/.outdir_structure_file.properties @@ -0,0 +1,36 @@ +[Default] +# Format +# OCI_Service_Name=Directory_Name +# Do not Modify the OCI Service Names specified on Left Hand Side +# Modify the directory name specified on Right Hand Side +# Directory will be created for that service under directory. Do not provide absolute path. +# Below data shows the segregation of services as per best practices. Please change as per your requirements. +# You will have to run createTenancy.py from scratch if you want to make any changes to the directory structure later. +# It is mandatory to specify the directory name for each service. + +identity=identity +tagging=tagging +network=network +quota=quota +loadbalancer=loadbalancer +networkloadbalancer=loadbalancer +vlan=vlan +nsg=nsg +# Same Directory must be specified for Instances and Block Volumes. +instance=compute +block-volume = compute +dedicated-vm-host=compute +adb=database +mysql-dbsystem=database +dbsystem-vm-bm=database +database-exacs=database +fss=fss +oke=oke +sddc=ocvs +cloud-guard=security +managementservices=managementservices +budget=budget +kms=kms +object-storage=oss +dns=dns +firewall=firewall \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/__init__.py b/cd3_automation_toolkit/user-scripts/__init__.py index bce6b268a..066c83efc 100644 --- a/cd3_automation_toolkit/user-scripts/__init__.py +++ b/cd3_automation_toolkit/user-scripts/__init__.py @@ -1,3 +1,3 @@ #!/usr/bin/env python3 -from .createTenancyConfig import seek_info \ No newline at end of file + diff --git a/cd3_automation_toolkit/user-scripts/connectAzure.py b/cd3_automation_toolkit/user-scripts/connectAzure.py new file mode 100644 index 000000000..b3cff47a4 --- /dev/null +++ b/cd3_automation_toolkit/user-scripts/connectAzure.py @@ -0,0 +1,196 @@ +#!/usr/bin/python3 +# Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. +# +# This script will help in initializing the docker container; creates config and variables files. +# +# Author: Shruthi Subramanian +# + +import argparse +import logging +import os +import shutil +import datetime +import configparser +from azure.identity import ClientSecretCredential +from azure.core.exceptions import ClientAuthenticationError + +def paginate(operation, *args, **kwargs): + while True: + response = operation(*args, **kwargs) + for value in response.data: + yield value + kwargs["page"] = response.next_page + if not response.has_next_page: + break + + + +# Execution of code begins here +parser = argparse.ArgumentParser(description="Connects the Container to Azure Subscription") +parser.add_argument("propsfile", help="Full Path of properties file. eg createtenancyAzure.properties") +args = parser.parse_args() +config = configparser.RawConfigParser() +config.read(args.propsfile) + +current_time=str(datetime.datetime.now()) +cloud="Azure" + +# Initialize Toolkit Variables +user_dir = "/cd3user/"+cloud.lower()+"/" +toolkit_dir = os.path.dirname(os.path.abspath(__file__))+"/.." +tf_modules_dir = toolkit_dir + "/azurecloud/terraform" + +setupcloud_props_toolkit_file_path = toolkit_dir + "/setUpAzure.properties" + + + +prefix = config.get('Default', 'prefix').strip() +if prefix == "" or prefix == "\n": + print("Invalid Prefix. Please try again......Exiting !!") + exit(1) + +prefixes=[] +f = os.path.basename(__file__).rstrip("py")+"safe" +safe_file = user_dir + f +if os.path.exists(safe_file): + f=open(safe_file,"r") + safe_file_lines = f.readlines() + for l in safe_file_lines: + if "SUCCESS" in l: + prefixes.append(l.split("\t")[0]) + +if prefixes !=[]: + if prefix in prefixes: + print("WARNING!!! Container has already been successfuly connected to the Azure with same prefix. Please proceed only if you re-running the script for new project subscription") + inp = input("\nDo you want to proceed (y/n):") + if inp.lower()=="n": + exit(1) + + +# Initialize Tenancy Variables + +prefix_dir = user_dir +"/" + prefix +config_files= prefix_dir +"/.config_files" + +terraform_files = prefix_dir + "/terraform_files" +az_provider_file = terraform_files + "/provider.tf" +setupcloud_props_file_path = prefix_dir + "/"+prefix+"_setUp"+cloud+".properties" + +connected = 1 + +# Read Config file Variables +try: + subscription_id='' + tenant_id='' + client_id='' + client_secret='' + + subscription_id = config.get('Default', 'subscription_id').strip() + + tenant_id = config.get('Default', 'tenant_id').strip() + + client_id = config.get('Default', 'client_id').strip() + + client_secret = config.get('Default', 'client_secret').strip() + + if (subscription_id == '' or tenant_id == '' or client_id == '' or client_secret == ''): + print("\n\nCreating "+prefix_dir + " without setting up authentication\n") + connected=0 +except Exception as e: + print(e) + print('Check if input properties exist and try again..exiting...') + exit(1) + + +if not os.path.exists(prefix_dir): + os.makedirs(prefix_dir) +if not os.path.exists(config_files): + os.makedirs(config_files) + + +# Copy input properties file to customer_tenancy_dir +shutil.copy(args.propsfile,config_files+"/"+prefix+"_connectAzure.properties") + +if connected == 1: + try: + credential = ClientSecretCredential( + tenant_id=tenant_id, + client_id=client_id, + client_secret=client_secret) + token = credential.get_token("https://management.azure.com/.default") + print("\nAzure credentials are valid. Proceeding...") + except ClientAuthenticationError as e: + print(f"\nAzure credentials are invalid. Exiting!!!") + exit(1) + except Exception as e: + print(f"An unexpected error occurred: {e}") + + +# 3. Generate setUpCloud.properties file +print("Creating Azure specific setUpAzure.properties.................") +with open(setupcloud_props_toolkit_file_path, 'r+') as setUpCloud_file: + setupcloud_props_toolkit_file_data = setUpCloud_file.read().rstrip() + +setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("outdir=", "outdir="+terraform_files) +setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("prefix=", "prefix="+prefix) +if connected == 1: + setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("subscription_id=", "subscription_id=" + subscription_id) + setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("tenant_id=", "tenant_id="+tenant_id) + setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("client_id=", "client_id="+client_id) + setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("client_secret=", "client_secret="+client_secret) + +f = open(setupcloud_props_file_path, "w+") +f.write(setupcloud_props_toolkit_file_data) +f.close() + + +# Copy modules dir to terraform_files folder +try: + shutil.copytree(tf_modules_dir, terraform_files ) +except FileExistsError as fe: + print(fe) + + +# 6. Read variables.tf from examples folder and copy the variables as string + +if connected == 1: + print("Creating subscription specific terraform provider file................") + with open(az_provider_file, 'r+') as provider_file: + az_provider_file_data = provider_file.read().rstrip() + + az_provider_file_data = az_provider_file_data.replace("", subscription_id) + az_provider_file_data = az_provider_file_data.replace("", tenant_id) + az_provider_file_data = az_provider_file_data.replace("", client_id) + az_provider_file_data = az_provider_file_data.replace("", client_secret) + + f = open(az_provider_file, "w+") + f.write(az_provider_file_data) + f.close() + +outfile = prefix_dir+'/'+os.path.basename(__file__).rstrip("py")+"out" +logging.basicConfig(filename=outfile, format='%(message)s', filemode='w', level=logging.INFO) + +print("==================================================================================================================================") +print("\nThe toolkit has been setup successfully. !!!\n") + +f = open(safe_file, "a") +data=prefix + "\t" + "SUCCESS\t"+current_time+"\n" +f.write(data) +f.close() + +logging.info("Prefix Specific Working Directory Path: "+prefix_dir+"\n") + +logging.info("\n######################################") +logging.info("Next Steps for using toolkit via CLI") +logging.info("######################################") +logging.info("Modify "+prefix_dir + "/" +prefix+"_setUpAzure.properties with input values for cd3file and workflow_type") +logging.info("cd /cd3user/oci_tools/cd3_automation_toolkit") +logging.info("python setUpCloud.py azure "+prefix_dir + "/" +prefix+"_setUpAzure.properties") + +with open(outfile, 'r') as log_file: + data = log_file.read().rstrip() +print(data) + +print("==================================================================================================================================") + diff --git a/cd3_automation_toolkit/user-scripts/connectGCP.py b/cd3_automation_toolkit/user-scripts/connectGCP.py new file mode 100644 index 000000000..5c7329b86 --- /dev/null +++ b/cd3_automation_toolkit/user-scripts/connectGCP.py @@ -0,0 +1,284 @@ +import argparse +import configparser +import datetime +import shutil +import distutils +import os,sys,logging +sys.path.append(os.getcwd()+"/../gcp/python/") +from gcpCommonTools import * + + + +# Execution of code begins here +parser = argparse.ArgumentParser(description="Connects the Container to GCP Tenant") +parser.add_argument("propsfile", help="Full Path of properties file. eg connectGCP.properties") +args = parser.parse_args() +config = configparser.RawConfigParser() +config.read(args.propsfile) + +current_time=str(datetime.datetime.now()) + +# Initialize Toolkit Variables +user_dir = "/cd3user/cloud_accounts/gcp" +user_dir = "/Users/susingla/PyCharmProjects/orahub-develop/cd3user/cloud_accounts/gcp" +auto_keys_dir = user_dir + "/keys" +toolkit_dir = os.path.dirname(os.path.abspath(__file__))+"/.." +tf_modules_dir = toolkit_dir + "/gcp/terraform" +variables_example_file = tf_modules_dir + "/variables_example.tf" +setupcloud_props_toolkit_file_path = toolkit_dir + "/user-scripts/setUpCloud.properties" + +cloud="GCP" + +prefix = config.get('Default', 'prefix').strip() +if prefix == "" or prefix == "\n": + print("Invalid Prefix. Please try again......Exiting !!") + exit(1) + +prefixes=[] +f = os.path.basename(__file__).rstrip("py")+".safe" +safe_file = user_dir + f +if os.path.exists(safe_file): + f=open(safe_file,"r") + safe_file_lines = f.readlines() + for l in safe_file_lines: + if "SUCCESS" in l: + prefixes.append(l.split("\t")[0]) + +if prefixes !=[]: + if prefix in prefixes: + print("WARNING!!! Container has already been successfuly connected to the GCP with same prefix. Please proceed only if you re-running the script for new project subscription") + inp = input("\nDo you want to proceed (y/n):") + if inp.lower()=="n": + exit(1) + +# Initialize Tenancy Variables +prefix_dir = user_dir +"/" + prefix +config_files= prefix_dir +"/.config_files" + +terraform_files = prefix_dir + "/terraform_files/" +outdir_safe=terraform_files+"/.safe" +setupcloud_props_file_path = prefix_dir + "/"+prefix+"_setUpCloud.properties" + +# Read Config file Variables +try: + organization_id='' + config_file='' + + organization_id = config.get('Default', 'organization_id').strip() + if organization_id == "" or organization_id == "\n": + print("organization_id cannot be left empty...Exiting !!") + exit(1) + + config_file = config.get('Default', 'config_file').strip() + if config_file == "" or config_file == "\n": + config_file = auto_keys_dir +"/gcp_api_private.json" + + if not os.path.isfile(config_file): + print("Invalid JSON Key File at " + config_file + ". Please try again......Exiting !!") + exit(1) + + + outdir_structure_file = config.get('Default', 'outdir_structure_file').strip() + ssh_public_key = config.get('Default', 'ssh_public_key').strip() + +except Exception as e: + print(e) + print('Check if input properties exist and try again..exiting...') + exit(1) + + +if not os.path.exists(prefix_dir): + os.makedirs(prefix_dir) +if not os.path.exists(config_files): + os.makedirs(config_files) +if not os.path.exists(outdir_safe): + os.makedirs(outdir_safe) + +# Copy input properties file to customer_tenancy_dir +shutil.copy(args.propsfile,config_files+"/"+prefix+"_"+os.path.basename(args.propsfile)) + +# 1. Copy outdir_structure_file and config file +# Copy default outdir_structure_file +shutil.copy(toolkit_dir+'/user-scripts/outdir_structure_file.properties', toolkit_dir+'/user-scripts/.outdir_structure_file.properties') + +_outdir_structure_file = '' +dir_values = [] +if (outdir_structure_file != '' and outdir_structure_file != "\n"): + if not os.path.isfile(outdir_structure_file): + print("Invalid outdir_structure_file. Please provide correct file path......Exiting !!") + exit(1) + else: + outdir_config = configparser.RawConfigParser() + outdir_config.read(outdir_structure_file) + for key, value in outdir_config.items("Default"): + if value == '': + print("Out Directory is missing for one or more parameters, for eg. " + key) + print("Please check " + outdir_structure_file) + exit(1) + if value not in dir_values: + dir_values.append(str(value)) + + _outdir_structure_file = prefix_dir+ "/"+prefix+"_outdir_structure_file.properties" + #if not os.path.exists(_outdir_structure_file): + shutil.copyfile(outdir_structure_file, _outdir_structure_file) + print("\nUsing different directories for GCP services as per the input outdir_structure_file..........") +else: + print("\nUsing single out directory for resources..........") + +filename = os.path.basename(config_file) +_config_file=config_files + "/" + filename +shutil.copy(config_file, _config_file) +os.chmod(_config_file,0o600) + + +# 2. Authenticate and Get Projects +gct = gcpCommonTools() +credentials = gct.authenticate(config_file) +gct.get_organization_projects(organization_id,credentials) + +# 3. Generate setUpCloud.properties file +print("Creating GCP specific setUpCloud.properties.................") +with open(setupcloud_props_toolkit_file_path, 'r+') as setUpCloud_file: + setupcloud_props_toolkit_file_data = setUpCloud_file.read().rstrip() + +setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("outdir=", "outdir="+terraform_files) +setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("prefix=", "prefix="+prefix) +setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("organization_id=", "organization_id=" + organization_id) +setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("config_file=", "config_file="+_config_file) +setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("outdir_structure_file=", "outdir_structure_file="+_outdir_structure_file) + +f = open(setupcloud_props_file_path, "w+") +f.write(setupcloud_props_toolkit_file_data) +f.close() + +# 4.Create the TF related files for each project +if not os.path.exists(terraform_files): + os.makedirs(terraform_files) + + +print("Creating GCP specific project directories, terraform provider , variables files.................") +#regions_file_data = "" + +# 5. Read variables.tf from examples folder and copy the variables as string +for id,name in gct.projects.items(): + project=id + + with open(variables_example_file, 'r+') as var_eg_file: + variables_example_file_data = var_eg_file.read().rstrip() + + variables_example_file_data = variables_example_file_data.replace("", _config_file) + variables_example_file_data = variables_example_file_data.replace("", project) + + # Rerunning script for any new project subscription. Process only new prohect directories else continue + if os.path.exists(terraform_files + project): + continue + + os.mkdir(terraform_files + project) + f = open(terraform_files + "/" + project + "/variables_" + project + ".tf", "w+") + f.write(variables_example_file_data) + f.close() + + # 6. Copy terraform modules and variables file to outdir + distutils.dir_util.copy_tree(tf_modules_dir, terraform_files + "/" + project) + + # Manage single and multiple outdir + if (outdir_structure_file == '' or outdir_structure_file == "\n"): + # remove depends_on for single outdir + project_dir = terraform_files + "/" + project + "/" + single_outdir_config = configparser.RawConfigParser() + outdir_config_file = os.path.dirname(os.path.abspath(__file__)) + "/.outdir_structure_file.properties" + + single_outdir_config.read(outdir_config_file) + keys = [] + for key, val in single_outdir_config.items("Default"): + keys.append(key) + for file in os.listdir(project_dir): + # name=file.removesuffix(".tf") + name = file[:-len(".tf")] + if name in keys: + file = project_dir + "/" + file + with open(file, 'r+') as tf_file: + module_data = tf_file.read().rstrip() + module_data = module_data.replace("# depends_on", "depends_on") + tf_file.close() + f = open(file, "w+") + f.write(module_data) + f.close() + else: + project_dir = terraform_files + "/" + project + "/" + for service, service_dir in outdir_config.items("Default"): + service = service.strip().lower() + service_dir = service_dir.strip() + + # Keep the .tf file in default region directory if directory name is empty + if service_dir == "" or service_dir == "\n" or service!='instance': + continue + + project_service_dir=project_dir+service_dir + if not os.path.exists(project_service_dir): + os.mkdir(project_service_dir) + + if (service == 'instance'): + if (os.path.isdir(project_service_dir + '/scripts')): + shutil.rmtree(project_service_dir + '/scripts') + if (os.path.exists(project_dir + 'scripts')): + shutil.move(project_dir + 'scripts', project_service_dir + '/') + with open(project_dir + service + ".tf", 'r+') as tf_file: + module_data = tf_file.read().rstrip() + module_data = module_data.replace("\"./modules", "\"../modules") + + f = open(project_service_dir + "/" + service + ".tf", "w+") + f.write(module_data) + f.close() + os.remove(project_dir + service + ".tf") + + shutil.copyfile(project_dir + "variables_" + project + ".tf", + project_service_dir + "/" + "variables_" + project + ".tf") + shutil.copyfile(project_dir + "provider.tf", project_service_dir + "/" + "provider.tf") + + + os.remove(terraform_files + "/" + project + "/" + "variables_" + project + ".tf") + os.remove(terraform_files + "/" + project + "/" + "provider.tf") + + # 8. Remove terraform example variable file from outdir + os.remove(terraform_files + "/" + project + "/variables_example.tf") + + +# Logging information +f = os.path.basename(__file__).rstrip("py")+".out" +outfile = prefix_dir + "/"+ f +logging.basicConfig(filename=outfile, format='%(message)s', filemode='w', level=logging.INFO) + +print("==================================================================================================================================") +print("\nThe toolkit has been setup successfully. !!!\n") +f = open(safe_file, "a") +data="GCP\t"+prefix + "\t" + "SUCCESS\t"+current_time+"\n" +f.write(data) +f.close() + +logging.info("Tenant Specific Working Directory Path: "+prefix_dir+"\n") + +logging.info("\n######################################") +logging.info("Next Steps for using toolkit via CLI") +logging.info("######################################") +logging.info("Modify "+prefix_dir + "/" +prefix+"_setUpGCP.properties with input values for cd3file and workflow_type") +logging.info("cd "+os.path.dirname(os.path.abspath(__file__))) +logging.info("python setUpGCP.py "+setupcloud_props_file_path) + +with open(outfile, 'r') as log_file: + data = log_file.read().rstrip() +print(data) + +print("==================================================================================================================================") + + + + + + + + + + + + diff --git a/cd3_automation_toolkit/user-scripts/createTenancyConfig.py b/cd3_automation_toolkit/user-scripts/createTenancyConfig.py index b26b83590..7a9eedbb8 100644 --- a/cd3_automation_toolkit/user-scripts/createTenancyConfig.py +++ b/cd3_automation_toolkit/user-scripts/createTenancyConfig.py @@ -17,9 +17,11 @@ from oci.object_storage import ObjectStorageClient import glob import subprocess -sys.path.append(os.getcwd()+"/..") from os import environ -from commonTools import * +sys.path.append(os.getcwd()) +sys.path.append(os.getcwd()+"/..") +from ocicloud.python.ociCommonTools import * +from ocicloud.python.cd3Services import * from copy import deepcopy from subprocess import DEVNULL global topic_name @@ -203,9 +205,9 @@ def update_devops_config(prefix, repo_ssh_url,files_in_repo,dir_values,devops_us # create symlink for Git Config file for SSH operations. src = git_config_file - if not os.path.exists("/cd3user/.ssh"): - os.makedirs("/cd3user/.ssh") - dst = "/cd3user/.ssh/config" + if not os.path.exists("/home/cd3user/.ssh"): + os.makedirs("/home/cd3user/.ssh") + dst = "/home/cd3user/.ssh/config" try: os.symlink(src,dst) except FileExistsError as e: @@ -364,7 +366,7 @@ def create_bucket(config, signer): auto_keys_dir = user_dir + "/tenancies/keys" toolkit_dir = os.path.dirname(os.path.abspath(__file__))+"/.." #toolkit_dir = user_dir +"/oci_tools/cd3_automation_toolkit" -terraform_dir = toolkit_dir + "/user-scripts/terraform" +terraform_dir = toolkit_dir + "/ocicloud/terraform" variables_example_file = terraform_dir + "/variables_example.tf" setupoci_props_toolkit_file_path = toolkit_dir + "/setUpOCI.properties" @@ -537,7 +539,7 @@ def create_bucket(config, signer): dir_values = [] # Copy input properties file to customer_tenancy_dir -shutil.copy(args.propsfile,config_files+"/"+prefix+"_"+os.path.basename(args.propsfile)) +shutil.copy(args.propsfile,config_files+"/"+prefix+"_tenancyconfig.properties") # 1. Copy outdir_structure_file @@ -626,7 +628,7 @@ def create_bucket(config, signer): tenancy_id=tenancy ## Authenticate -ct = commonTools() +ct = ociCommonTools() config, signer = ct.authenticate(auth_mechanism, config_file_path) _realm = config['tenancy'].split(".")[2] cloud_domain = "."+oci.regions.REALMS[_realm] @@ -636,7 +638,7 @@ def create_bucket(config, signer): cd3service.fetch_regions(config, signer) #This is needed to be initialised again -ct = commonTools() +ct = ociCommonTools() try: ct.get_subscribedregions(config,signer) except Exception as e: @@ -796,6 +798,9 @@ def create_bucket(config, signer): ''' # 4. Generate setUpOCI.properties file +# copying blank template to tenancy prefix for export ready +shutil.copy(f'{toolkit_dir}/example/CD3-Blank-template.xlsx',f'{customer_tenancy_dir}/CD3-Blank-template.xlsx') + #if not os.path.isfile(setupoci_props_file_path): print("Creating Tenancy specific setUpOCI.properties.................") with open(setupoci_props_toolkit_file_path, 'r+') as setUpOci_file: @@ -807,6 +812,7 @@ def create_bucket(config, signer): setupoci_props_toolkit_file_data = setupoci_props_toolkit_file_data.replace("config_file=", "config_file="+config_file_path) setupoci_props_toolkit_file_data = setupoci_props_toolkit_file_data.replace("outdir_structure_file=", "outdir_structure_file="+_outdir_structure_file) setupoci_props_toolkit_file_data = setupoci_props_toolkit_file_data.replace("tf_or_tofu=", "tf_or_tofu="+tf_or_tofu) +setupoci_props_toolkit_file_data = setupoci_props_toolkit_file_data.replace("cd3file=", "cd3file="+customer_tenancy_dir+"/CD3-Blank-template.xlsx") f = open(setupoci_props_file_path, "w+") f.write(setupoci_props_toolkit_file_data) @@ -1134,7 +1140,7 @@ def create_bucket(config, signer): logging.info("Next Steps for using toolkit via CLI") logging.info("######################################") logging.info("Modify "+customer_tenancy_dir + "/" +prefix+"_setUpOCI.properties with input values for cd3file and workflow_type") -logging.info("cd "+user_dir+"/oci_tools/cd3_automation_toolkit/") +logging.info("cd "+toolkit_dir) logging.info("python setUpOCI.py "+customer_tenancy_dir + "/" +prefix+"_setUpOCI.properties") with open(outfile, 'r') as log_file: diff --git a/cd3_automation_toolkit/user-scripts/deleteTenancyConfig.py b/cd3_automation_toolkit/user-scripts/deleteTenancyConfig.py index 6ea6df177..c7085adc9 100644 --- a/cd3_automation_toolkit/user-scripts/deleteTenancyConfig.py +++ b/cd3_automation_toolkit/user-scripts/deleteTenancyConfig.py @@ -18,10 +18,12 @@ import oci from oci.object_storage import ObjectStorageClient from copy import deepcopy -sys.path.append(os.getcwd()+"/..") import subprocess from os import environ -from commonTools import * +sys.path.append(os.getcwd()) +sys.path.append(os.getcwd()+"/..") +from ocicloud.python.ociCommonTools import * +from ocicloud.python.cd3Services import * @@ -165,7 +167,7 @@ repo_name = devops_repo.split("/")[1] devops_exists = True - ct = commonTools() + ct = ociCommonTools() config, signer = ct.authenticate(auth_mechanism, config_file_path) try: ct.get_subscribedregions(config,signer) diff --git a/cd3_automation_toolkit/user-scripts/setUpAzure.py b/cd3_automation_toolkit/user-scripts/setUpAzure.py new file mode 100644 index 000000000..1fb046cc0 --- /dev/null +++ b/cd3_automation_toolkit/user-scripts/setUpAzure.py @@ -0,0 +1,243 @@ +import argparse +import configparser +from collections import namedtuple +import datetime,os +import sys +sys.path.append(os.getcwd()) +sys.path.append(os.getcwd()+"/..") +from azurecloud.python import * +from common.python import * + + +def match_options(options, prim_options): + print("match_options") + user_input = "" + # Iterate over options. Print number and option + for i, option in enumerate(options, 1): + if option.name in prim_options: + user_input += "," + str(i) + user_input = user_input.split(',')[1:] + try: + return [options[int(choice) - 1] for choice in user_input] + except IndexError as ie: + print("\nInvalid Option.....Exiting!!") + exit(1) + + +def show_options(options, quit=False, menu=False, extra=None, index=0): + # Just add whitespace between number and option. It just makes it look better + number_offset = len(str(len(options))) + 1 + # Iterate over options. Print number and option + for i, option in enumerate(options, index): + print(f'{str(i)+".":<{number_offset}} {option.name}') + if quit: + print(f'{"q"+".":<{number_offset}} Press q to quit') + if menu: + print(f'{"m"+".":<{number_offset}} Press m to go back to Main Menu') + if extra: + print(extra) + user_input = input('Enter your choice (specify comma separated to choose multiple choices): ') + user_input = user_input.split(',') + if 'q' in user_input or 'm' in user_input: + return user_input + # Subtract one to account for zero-indexing. The options start at 1 + # #return [options[int(choice)-1] for choice in user_input] + + try: + for choice in user_input: + if int(choice) - index < 0: + print("\nInvalid Option.....Exiting!!") + exit(1) + elif options[int(choice) - index].name == "Execute All": + options.pop(0) + return options + except ValueError as ie: + print("\nInvalid Input.....Try again!!\n") + options = show_options(inputs, quit=True, index=index) + return options + + try: + return [options[int(choice)-index] for choice in user_input] + except IndexError as ie: + print("\nInvalid Option.....Exiting!!") + exit(1) + except ValueError as ie: + print("\nInvalid Input.....Try again!!\n") + options = show_options(inputs, quit=True, index=index) + return options + +def execute_options(options, *args, **kwargs): + global menu, quit + if 'm' in options or 'q' in options: + menu = 'm' in options + quit = 'q' in options + else: + for option in options: + if option.name == "Execute All": + continue + else: + with section(option.text): + option.callback(*args, **kwargs) + + +def create_adb_azure(): + create_terraform_adb_azure(inputfile, outdir, prefix) + +def create_exa_azure(): + create_terraform_exa_infra_azure(inputfile, outdir, prefix) + create_terraform_exa_vmclusters_azure(inputfile, outdir, prefix) + + +def create_db_at_azure(execute_all=False): + options = [ + Option('Add/Modify/Delete ADB @Azure', create_adb_azure, 'Processing ADB-Azure Tab') + #Option('Add/Modify/Delete Exa @Azure', create_exa_azure, 'Processing Exa-Azure Tabs') + # Option('Enable LBaaS Logs', enable_lb_logs, 'LBaaS Logs') + ] + options = show_options(options, quit=True, menu=True, index=1) + if not execute_all: + execute_options(options) + +''' +def export_az_adb(): + export_az_oci_adb(inputfile, outdir, credentials) +''' +def export_az_oci_exa(): + export_az_oci_adb(inputfile, outdir, credentials) + +def export_db_at_azure(execute_all=False): + options = [ + Option('Export ADB @Azure', export_adb_azure, 'Exporting ADB-Azure'), + # Option('Export Exa @Azure', export_az_oci_exa, 'Processing Exa-Azure') + ] + options = show_options(options, quit=True, menu=True, index=1) + if not execute_all: + execute_options(options,inputfile, outdir, credentials) + + create_terraform_adb_azure(inputfile, outdir, prefix) + + +#Execution starts here +global devops +global updated_paths +global import_scripts +updated_paths = [] +import_scripts = [] +# Opt-in to IMDS lookup +exec_start_time = datetime.datetime.now() +parser = argparse.ArgumentParser(description='Sets Up OCI via TF') +parser.add_argument('propsfile', help="Full Path of properties file containing input variables. eg setUpAzure.properties") +#parser.add_argument('--main_options', default="") +#parser.add_argument('--sub_options', default="") +#parser.add_argument('--sub_child_options', default="") +#parser.add_argument('--add_filter', default=None) +#parser.add_argument('--devops', default=False) +args = parser.parse_args() +setUpAz_props = configparser.RawConfigParser() +setUpAz_props.read(args.propsfile) +#devops = args.devops +#main_options = args.main_options.split(",") +#sub_options = args.sub_options.split(",") +#sub_child_options = args.sub_child_options.split(",") + +#Read Config file Variables +try: + workflow_type = setUpAz_props.get('Default', 'workflow_type').strip().lower() + + if (workflow_type == 'export_resources'): + non_gf_tenancy = True + else: + non_gf_tenancy = False + + inputfile = setUpAz_props.get('Default','cd3file').strip() + outdir = setUpAz_props.get('Default', 'outdir').strip() + prefix = setUpAz_props.get('Default', 'prefix').strip() + tf_or_tofu = "terraform" + + if not outdir: + exit_menu('input outdir location cannot be left blank. Exiting... ') + elif not prefix: + exit_menu('input prefix value cannot be left blank. Exiting... ') + elif not inputfile: + exit_menu('input cd3file location cannot be left blank. Exiting... ') + elif '.xls' not in inputfile: + exit_menu('valid formats for input cd3file are either .xls or .xlsx') +except Exception as e: + exit_menu(str(e) + ". Check input properties file and try again. Exiting... ") + + + +Option = namedtuple('Option', ['name', 'callback', 'text']) +extra = '' + +# Pre-work +if not os.path.exists(outdir): + os.makedirs(outdir) + + +#if devops: + # Set Export filters from devops + #export_filters = args.add_filter.split("@") if args.add_filter else [] + #ct.get_export_filters(export_filters) + +## Menu Options +if non_gf_tenancy: + + ct = azrCommonTools() + credentials = ct.authenticate(args.propsfile) + + # verify_outdir_is_empty() + print("\nworkflow_type set to export_resources. Export existing Azure objects and Synch with TF state") + print("We recommend to not have any existing tfvars/tfstate files for export out directory") + #export_regions = get_region_list(rm=False,vizoci=False) + #compartments = ct.get_compartment_map(var_file, "OCI Resources") + #export_tags_list = get_tags_list() + + inputs = [ + Option("Export DB @Azure", export_db_at_azure, "Export DB @Azure"), + + ] + +else: + inputs = [ + Option('Create DB @Azure', create_db_at_azure, 'Create DB @Azure'), + + ] +''' +if main_options and args.main_options != "": + options = match_options(inputs, main_options) + for option in options: + with section(option.text, header=True): + option.callback(prim_options=sub_options) +else: +''' +if True: + print("\nChoose appropriate option from below :\n") + # Run menu + menu = True + while menu: + if non_gf_tenancy: + options = show_options(inputs, quit=True, index=1) + else: + options = show_options(inputs, quit=True, extra='\nSee example folder for sample input files\n', index=1) + if 'q' in options: + exit_menu('Exiting...') + for option in options: + menu = False + with section(option.text, header=True): + option.callback() + if menu: + break +# write updated paths to a file +''' +updated_paths_file = f'{outdir}/.safe/updated_paths.safe' +with open(updated_paths_file, 'w+') as f: + for item in updated_paths: + f.write(str(item).replace('//', '/') + "\n") +f.close() +import_scripts_file = f'{outdir}/.safe/import_scripts.safe' +with open(import_scripts_file, 'w+') as f: + for item in import_scripts: + f.write(str(item).replace('//', '/') + "\n") +f.close() +''' \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/adb.tf b/cd3_automation_toolkit/user-scripts/terraform/adb.tf deleted file mode 100755 index 7f4f2b448..000000000 --- a/cd3_automation_toolkit/user-scripts/terraform/adb.tf +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. -# -############################# -## Module Block - Autonomous database -## Create autonomous database -############################# -data "oci_core_subnets" "oci_subnets_adb" { - # depends_on = [module.subnets] # Uncomment to create Network and FSS together - #for_each = var.adb != null ? var.adb : {} - for_each = { for k, v in var.adb : k => v if v.vcn_name != null } - compartment_id = each.value.network_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0 ? each.value.network_compartment_id : var.compartment_ocids[each.value.network_compartment_id]) : var.compartment_ocids[each.value.network_compartment_id] - display_name = each.value.subnet_id - vcn_id = data.oci_core_vcns.oci_vcns_adb[each.key].virtual_networks.*.id[0] -} - -data "oci_core_vcns" "oci_vcns_adb" { - # depends_on = [module.vcns] # Uncomment to create Network and FSS together - #for_each = var.adb != null ? var.adb : {} - for_each = { for k, v in var.adb : k => v if v.vcn_name != null } - compartment_id = each.value.network_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0 ? each.value.network_compartment_id : var.compartment_ocids[each.value.network_compartment_id]) : var.compartment_ocids[each.value.network_compartment_id] - display_name = each.value.vcn_name -} - -module "adb" { - source = "./modules/database/adb" - for_each = var.adb != null ? var.adb : {} - # depends_on = [module.nsgs] - admin_password = each.value.admin_password - character_set = each.value.character_set - compartment_id = each.value.compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id]) : null - cpu_core_count = each.value.cpu_core_count - database_edition = each.value.database_edition - data_storage_size_in_tbs = each.value.data_storage_size_in_tbs - db_name = each.value.db_name - db_version = each.value.db_version - db_workload = each.value.db_workload - defined_tags = each.value.defined_tags - display_name = each.value.display_name - license_model = each.value.license_model - ncharacter_set = each.value.ncharacter_set - customer_contacts = each.value.customer_contacts - network_compartment_id = each.value.network_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0 ? each.value.network_compartment_id : var.compartment_ocids[each.value.network_compartment_id]) : null - network_security_group_ids = each.value.nsg_ids - freeform_tags = each.value.freeform_tags - subnet_id = each.value.subnet_id != null ? (length(regexall("ocid1.subnet.oc*", each.value.subnet_id)) > 0 ? each.value.subnet_id : data.oci_core_subnets.oci_subnets_adb[each.key].subnets.*.id[0]) : null - vcn_name = each.value.vcn_name != null ? each.value.vcn_name : null - whitelisted_ips = each.value.whitelisted_ips - - #Optional parameters for ADB - # are_primary_whitelisted_ips_used = var.autonomous_database_are_primary_whitelisted_ips_used - # autonomous_container_database_id = oci_database_autonomous_container_database.test_autonomous_container_database.id - # autonomous_database_backup_id = oci_database_autonomous_database_backup.test_autonomous_database_backup.id - # autonomous_database_id = oci_database_autonomous_database.test_autonomous_database.id - # autonomous_maintenance_schedule_type = var.autonomous_database_autonomous_maintenance_schedule_type - # clone_type = var.autonomous_database_clone_type - # customer_contacts { - - #Optional - # email = var.autonomous_database_customer_contacts_email - # } - # data_safe_status = var.autonomous_database_data_safe_status - # data_storage_size_in_gb = var.autonomous_database_data_storage_size_in_gb - # is_access_control_enabled = var.autonomous_database_is_access_control_enabled - # is_auto_scaling_enabled = var.autonomous_database_is_auto_scaling_enabled - # is_auto_scaling_for_storage_enabled = var.autonomous_database_is_auto_scaling_for_storage_enabled - # is_data_guard_enabled = var.autonomous_database_is_data_guard_enabled - # is_dedicated = var.autonomous_database_is_dedicated - # is_free_tier = var.autonomous_database_is_free_tier - # is_local_data_guard_enabled = var.autonomous_database_is_local_data_guard_enabled - # is_mtls_connection_required = var.autonomous_database_is_mtls_connection_required - # is_preview_version_with_service_terms_accepted = var.autonomous_database_is_preview_version_with_service_terms_accepted - # kms_key_id = oci_kms_key.test_key.id - # max_cpu_core_count = var.autonomous_database_max_cpu_core_count - # ocpu_count = var.autonomous_database_ocpu_count - # private_endpoint_label = var.autonomous_database_private_endpoint_label - # refreshable_mode = var.autonomous_database_refreshable_mode - # scheduled_operations { - # #Required - # day_of_week { - # #Required - # name = var.autonomous_database_scheduled_operations_day_of_week_name - # } - - # #Optional - # scheduled_start_time = var.autonomous_database_scheduled_operations_scheduled_start_time - # scheduled_stop_time = var.autonomous_database_scheduled_operations_scheduled_stop_time - # } - # source = var.autonomous_database_source - # source_id = oci_database_source.test_source.id - # standby_whitelisted_ips = var.autonomous_database_standby_whitelisted_ips - # timestamp = var.autonomous_database_timestamp - # vault_id = oci_kms_vault.test_vault.id - # whitelisted_ips = var.autonomous_database_whitelisted_ips - -} \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/database/adb/main.tf b/cd3_automation_toolkit/user-scripts/terraform/modules/database/adb/main.tf deleted file mode 100644 index 725197132..000000000 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/database/adb/main.tf +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. -# -################################ -## Resource Block - Autonomous database -## Create autonomous database -################################ - -resource "oci_database_autonomous_database" "autonomous_database" { - #Required - compartment_id = var.compartment_id - db_name = var.db_name - - #Optional - admin_password = var.admin_password - character_set = var.character_set - cpu_core_count = var.cpu_core_count - database_edition = var.database_edition - data_storage_size_in_tbs = var.data_storage_size_in_tbs - db_version = var.db_version - db_workload = var.db_workload - defined_tags = var.defined_tags - display_name = var.display_name - license_model = var.license_model - ncharacter_set = var.ncharacter_set - dynamic "customer_contacts" { - for_each = var.customer_contacts!=null ? (var.customer_contacts[0] != "" ? var.customer_contacts : []) : [] - content { - email = customer_contacts.value - } - } - nsg_ids = length(var.network_security_group_ids) != 0 ? (local.nsg_ids == [] ? ["INVALID NSG Name"] : local.nsg_ids) : null - freeform_tags = var.freeform_tags - subnet_id = var.subnet_id - whitelisted_ips = var.whitelisted_ips -} \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-subscription/main.tf b/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-subscription/main.tf deleted file mode 100644 index 253535843..000000000 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-subscription/main.tf +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. -# -############################ -# Resource Block - ManagementServices -# Create Notifications_Subscriptions -############################ - -resource "oci_ons_subscription" "subscription" { - - #Required - compartment_id = var.compartment_id - endpoint = var.endpoint - protocol = var.protocol - topic_id = var.topic_id - - #Optional - defined_tags = var.defined_tags - freeform_tags = var.freeform_tags - -} diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-subscription/outputs.tf b/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-subscription/outputs.tf deleted file mode 100644 index fed34382a..000000000 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-subscription/outputs.tf +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. -# -############################ -# Output Block - ManagementServices -# Create Notifications_Subscriptions -############################ - -output "topic_subscription_tf_id" { - description = "Topic Subscription OCID" - value = oci_ons_subscription.subscription.id -} \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-subscription/variables.tf b/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-subscription/variables.tf deleted file mode 100644 index f53a165a3..000000000 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/notification-subscription/variables.tf +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. -# -############################ -# Variables Block - ManagementServices -# Create Notifications_Subscriptions -############################ - -variable "compartment_id" { - type = string -} - -variable "endpoint" { - type = string -} - -variable "protocol" { - type = string -} - -variable "topic_id" { - type = string -} - -variable "defined_tags" { - type = map(any) - default = { "Oracle-Tags.CreatedOn" = "$${oci.datetime}", - "Oracle-Tags.CreatedBy" = "$${iam.principal.name}" - } -} - -variable "freeform_tags" { - type = map(any) - default = {} -} \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/service-connector/data.tf b/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/service-connector/data.tf deleted file mode 100755 index f876b3934..000000000 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/managementservices/service-connector/data.tf +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. -# -#################################### -# Data Block - Service Connector -# Create Service Connector Hub -##################################### - -locals { - log_group_names = var.log_group_names - source_kind = var.source_kind - filtered_logs = [for item in var.log_group_names : item if split("@", item)[2] != "all"] -} - -data "oci_objectstorage_namespace" "os_namespace" { - compartment_id = var.logs_compartment_id -} -data "oci_identity_compartments" "all_compartments" { - #Required - compartment_id = var.logs_compartment_id - compartment_id_in_subtree = true -} -data "oci_streaming_streams" "source_streams" { - for_each = var.source_stream_id - name = each.value - compartment_id = each.key -} -data "oci_streaming_streams" "target_streams" { - for_each = var.stream_id - name = each.value - compartment_id = each.key -} -data "oci_ons_notification_topics" "target_topics" { - for_each = var.topic_id - name = each.value - compartment_id = each.key -} -data "oci_logging_log_groups" "source_log_groups" { - for_each = toset(var.log_group_names) - compartment_id = split("@", each.key)[0] - display_name = split("@", each.key)[1] -} -data "oci_logging_logs" "source_logs" { - for_each = toset(local.filtered_logs) - log_group_id = data.oci_logging_log_groups.source_log_groups[each.key].log_groups[0].id - display_name = split("@", each.key)[2] -} -data "oci_log_analytics_log_analytics_log_groups" "target_log_analytics_log_groups" { - for_each = var.destination_log_group_id - #Required - compartment_id = each.key - namespace = data.oci_objectstorage_namespace.os_namespace.namespace - - #Optional - display_name = each.value -} - -data "oci_functions_applications" "applications" { - for_each = toset(var.function_details) - #Required - compartment_id = split("@", each.key)[0] - - #Optional - display_name = split("@", each.key)[1] -} - -data "oci_functions_functions" "functions" { - for_each = toset(var.function_details) - #Required - application_id = data.oci_functions_applications.applications[each.key].applications[0].id - - #Optional - display_name = split("@", each.key)[2] -} - -data "oci_identity_compartments" "compartments" { - for_each = toset(keys(var.source_monitoring_details)) - #Required - compartment_id = var.logs_compartment_id - - #Optional - access_level = "ANY" - compartment_id_in_subtree = true - state = "ACTIVE" - name = each.value -} \ No newline at end of file diff --git a/jenkins_install/apply.groovy b/jenkins_install/apply.groovy index 103bcbbc3..45566a1ca 100644 --- a/jenkins_install/apply.groovy +++ b/jenkins_install/apply.groovy @@ -104,7 +104,7 @@ pipeline { def toolCmd = env.tf_or_tofu == 'terraform' ? 'terraform' : 'tofu' // Run Terraform/Tofu show and capture the output // Run OPA eval - opaOutput = labelledShell( label: 'Evaluating plan against OPA', script: "cd \"${WORKSPACE}/${env.Region}/${env.Service}\" && ${toolCmd} show -json tfplan.out > tfplan.json && opa eval -f pretty -b /cd3user/oci_tools/cd3_automation_toolkit/user-scripts/OPA/ -i \"${WORKSPACE}/${env.Region}/${env.Service}/tfplan.json\" data.terraform.deny", returnStdout: true).trim() + opaOutput = labelledShell( label: 'Evaluating plan against OPA', script: "cd \"${WORKSPACE}/${env.Region}/${env.Service}\" && ${toolCmd} show -json tfplan.out > tfplan.json && opa eval -f pretty -b /cd3user/oci_tools/cd3_automation_toolkit/common/opa -i \"${WORKSPACE}/${env.Region}/${env.Service}/tfplan.json\" data.terraform.deny", returnStdout: true).trim() if (opaOutput == '[]') { echo "No OPA rules are violated. Proceeding with the next stage." } else { diff --git a/jenkins_install/jenkins.sh b/jenkins_install/jenkins.sh index 0a2ab258e..e17a11ae7 100644 --- a/jenkins_install/jenkins.sh +++ b/jenkins_install/jenkins.sh @@ -20,8 +20,8 @@ fi #cp -r ${JENKINS_INSTALL}/scriptler $JENKINS_HOME cp ${JENKINS_INSTALL}/jcasc.yaml "$JENKINS_HOME/" -if [ ! -e "/cd3user/.ssh/config" ]; then - ln -s /cd3user/tenancies/jenkins_home/git_config /cd3user/.ssh/config +if [ ! -e "/home/cd3user/.ssh/config" ]; then + ln -s /cd3user/tenancies/jenkins_home/git_config /home/cd3user/.ssh/config fi # Copy scriptler directory diff --git a/othertools/cis_reports.py b/othertools/cis_reports.py index 632c9c0a9..11f8fa3e2 100644 --- a/othertools/cis_reports.py +++ b/othertools/cis_reports.py @@ -1,5 +1,5 @@ ########################################################################## -# Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved. # This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. # # cis_reports.py @@ -35,9 +35,16 @@ except Exception: OUTPUT_TO_XLSX = False -RELEASE_VERSION = "2.8.4" -PYTHON_SDK_VERSION = "2.129.4" -UPDATED_DATE = "July 26, 2024" +try: + import matplotlib.pyplot as plt + import numpy as np + OUTPUT_DIAGRAMS = True +except Exception: + OUTPUT_DIAGRAMS = False + +RELEASE_VERSION = "3.0.1" +PYTHON_SDK_VERSION = "2.152.1" +UPDATED_DATE = "July 16, 2025" ########################################################################## @@ -86,6 +93,7 @@ class CIS_Report: __KMS_DAYS_OLD = 365 __home_region = [] __days_to_expiry = 30 + __days_used = 45 # Time Format __iso_time_format = "%Y-%m-%dT%H:%M:%S" @@ -113,12 +121,20 @@ class CIS_Report: datetime.timedelta(days=__days_to_expiry) str_cert_key_time_max_datetime = cert_key_time_max_datetime.strftime(__iso_time_format) cert_key_time_max_datetime = datetime.datetime.strptime(str_cert_key_time_max_datetime, __iso_time_format) + # For Unused Credentials Check + local_user_time_max_datetime = start_datetime - \ + datetime.timedelta(days=__days_used) + str_local_user_time_max_datetime = local_user_time_max_datetime.strftime(__iso_time_format) + local_user_time_max_datetime = datetime.datetime.strptime(str_local_user_time_max_datetime, __iso_time_format) - def __init__(self, config, signer, proxy, output_bucket, report_directory, report_prefix, report_summary_json, print_to_screen, regions_to_run_in, raw_data, obp, redact_output, oci_url=None, debug=False, all_resources=True): + def __init__(self, config, signer, proxy, output_bucket, report_directory, report_prefix,\ + report_summary_json, print_to_screen, regions_to_run_in, raw_data, obp, \ + redact_output, oci_url=None, debug=False, all_resources=True, \ + disable_api_keys=False): - # CIS Foundation benchmark 2.0.0 - self.cis_foundations_benchmark_2_0 = { + # CIS Foundation benchmark 3.0.0 + self.cis_foundations_benchmark_3_0 = { '1.1': {'section': 'Identity and Access Management', 'recommendation_#': '1.1', 'Title': 'Ensure service level admins are created to manage resources of particular service', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['5.4', '6.7'], 'CCCS Guard Rail': '2,3', 'Remediation': []}, '1.2': {'section': 'Identity and Access Management', 'recommendation_#': '1.2', 'Title': 'Ensure permissions on all resources are given only to the tenancy administrator group', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['3.3'], 'CCCS Guard Rail': '1,2,3', 'Remediation': []}, '1.3': {'section': 'Identity and Access Management', 'recommendation_#': '1.3', 'Title': 'Ensure IAM administrators cannot update tenancy Administrators group', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['3.3', '5.4'], 'CCCS Guard Rail': '2,3', 'Remediation': []}, @@ -134,17 +150,19 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor '1.13': {'section': 'Identity and Access Management', 'recommendation_#': '1.13', 'Title': 'Ensure all OCI IAM user accounts have a valid and current email address', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['5.1'], 'CCCS Guard Rail': '1,2,3', 'Remediation': []}, '1.14': {'section': 'Identity and Access Management', 'recommendation_#': '1.14', 'Title': 'Ensure Instance Principal authentication is used for OCI instances, OCI Cloud Databases and OCI Functions to access OCI resources.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['6.8'], 'CCCS Guard Rail': '6,7', 'Remediation': []}, '1.15': {'section': 'Identity and Access Management', 'recommendation_#': '1.15', 'Title': 'Ensure storage service-level admins cannot delete resources they manage', 'Status': None, 'Level': 2, 'Total': [], 'Findings': [], 'CISv8': ['5.4', '6.8'], 'CCCS Guard Rail': '2,3', 'Remediation': []}, + '1.16': {'section': 'Identity and Access Management', 'recommendation_#': '1.16', 'Title': 'Ensure OCI IAM credentials unused for 45 days or more are disabled', 'Status': None, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['5.3'], 'CCCS Guard Rail': '2', 'Remediation': []}, + '1.17': {'section': 'Identity and Access Management', 'recommendation_#': '1.17', 'Title': 'Ensure there is only one active API Key for any single OCI IAM user.', 'Status': None, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['5'], 'CCCS Guard Rail': '2', 'Remediation': []}, '2.1': {'section': 'Networking', 'recommendation_#': '2.1', 'Title': 'Ensure no security lists allow ingress from 0.0.0.0/0 to port 22.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['4.4', '12.3'], 'CCCS Guard Rail': '2,3,5,7,9', 'Remediation': []}, '2.2': {'section': 'Networking', 'recommendation_#': '2.2', 'Title': 'Ensure no security lists allow ingress from 0.0.0.0/0 to port 3389.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['4.4', '12.3'], 'CCCS Guard Rail': '2,3,5,7,9', 'Remediation': []}, '2.3': {'section': 'Networking', 'recommendation_#': '2.3', 'Title': 'Ensure no network security groups allow ingress from 0.0.0.0/0 to port 22.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['4.4', '12.3'], 'CCCS Guard Rail': '2,3,5,7,9', 'Remediation': []}, '2.4': {'section': 'Networking', 'recommendation_#': '2.4', 'Title': 'Ensure no network security groups allow ingress from 0.0.0.0/0 to port 3389.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['4.4', '12.3'], 'CCCS Guard Rail': '2,3,5,7,9', 'Remediation': []}, - '2.5': {'section': 'Networking', 'recommendation_#': '2.5', 'Title': 'Ensure the default security list of every VCN restricts all traffic except ICMP.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['12.3'], 'CCCS Guard Rail': '2,3,5,7,9', 'Remediation': []}, + '2.5': {'section': 'Networking', 'recommendation_#': '2.5', 'Title': 'Ensure the default security list of every VCN restricts all traffic except ICMP within VCN.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['12.3'], 'CCCS Guard Rail': '2,3,5,7,9', 'Remediation': []}, '2.6': {'section': 'Networking', 'recommendation_#': '2.6', 'Title': 'Ensure Oracle Integration Cloud (OIC) access is restricted to allowed sources.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['4.4', '12.3'], 'CCCS Guard Rail': '2,3,5,7,9', 'Remediation': []}, '2.7': {'section': 'Networking', 'recommendation_#': '2.7', 'Title': 'Ensure Oracle Analytics Cloud (OAC) access is restricted to allowed sources or deployed within a Virtual Cloud Network.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['4.4', '12.3'], 'CCCS Guard Rail': '2,3,5,7,9', 'Remediation': []}, '2.8': {'section': 'Networking', 'recommendation_#': '2.8', 'Title': 'Ensure Oracle Autonomous Shared Database (ADB) access is restricted or deployed within a VCN.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['4.4', '12.3'], 'CCCS Guard Rail': '2,3,5,7,9', 'Remediation': []}, - '3.1': {'section': 'Compute', 'recommendation_#': '3.1', 'Title': 'Ensure Compute Instance Legacy Metadata service endpoint is disabled.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['4.6'], 'CCCS Guard Rail': '', 'Remediation': []}, + '3.1': {'section': 'Compute', 'recommendation_#': '3.1', 'Title': 'Ensure Compute Instance Legacy Metadata service endpoint is disabled.', 'Status': True, 'Level': 2, 'Total': [], 'Findings': [], 'CISv8': ['4.6'], 'CCCS Guard Rail': '', 'Remediation': []}, '3.2': {'section': 'Compute', 'recommendation_#': '3.2', 'Title': 'Ensure Secure Boot is enabled on Compute Instance.', 'Status': True, 'Level': 2, 'Total': [], 'Findings': [], 'CISv8': ['4.1'], 'CCCS Guard Rail': '', 'Remediation': []}, '3.3': {'section': 'Compute', 'recommendation_#': '3.3', 'Title': 'Ensure In-transit Encryption is enabled on Compute Instance.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': [''], 'CCCS Guard Rail': '', 'Remediation': []}, @@ -160,11 +178,12 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor '4.10': {'section': 'Logging and Monitoring', 'recommendation_#': '4.10', 'Title': 'Ensure a notification is configured for security list changes.', 'Status': False, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['4.2'], 'CCCS Guard Rail': '11', 'Remediation': []}, '4.11': {'section': 'Logging and Monitoring', 'recommendation_#': '4.11', 'Title': 'Ensure a notification is configured for network security group changes.', 'Status': False, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['4.2'], 'CCCS Guard Rail': '11', 'Remediation': []}, '4.12': {'section': 'Logging and Monitoring', 'recommendation_#': '4.12', 'Title': 'Ensure a notification is configured for changes to network gateways.', 'Status': False, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['4.2'], 'CCCS Guard Rail': '11', 'Remediation': []}, - '4.13': {'section': 'Logging and Monitoring', 'recommendation_#': '4.13', 'Title': 'Ensure VCN flow logging is enabled for all subnets.', 'Status': True, 'Level': 2, 'Total': [], 'Findings': [], 'CISv8': ['8.2', '8.5', '13.6'], 'CCCS Guard Rail': '', 'Remediation': []}, + '4.13': {'section': 'Logging and Monitoring', 'recommendation_#': '4.13', 'Title': 'Ensure VCN flow logging is enabled for all subnets.', 'Status': True, 'Level': 2, 'Total': [], 'Findings': [], 'CISv8': ['8.2', '8.5', '13.6'], 'CCCS Guard Rail': '', 'Remediation': []}, '4.14': {'section': 'Logging and Monitoring', 'recommendation_#': '4.14', 'Title': 'Ensure Cloud Guard is enabled in the root compartment of the tenancy.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['8.2', '8.5', '8.11'], 'CCCS Guard Rail': '1,2,3', 'Remediation': []}, '4.15': {'section': 'Logging and Monitoring', 'recommendation_#': '4.15', 'Title': 'Ensure a notification is configured for Oracle Cloud Guard problems detected.', 'Status': False, 'Level': 2, 'Total': [], 'Findings': [], 'CISv8': ['8.2', '8.11'], 'CCCS Guard Rail': '', 'Remediation': []}, '4.16': {'section': 'Logging and Monitoring', 'recommendation_#': '4.16', 'Title': 'Ensure customer created Customer Managed Key (CMK) is rotated at least annually.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': [], 'CCCS Guard Rail': '6,7', 'Remediation': []}, - '4.17': {'section': 'Logging and Monitoring', 'recommendation_#': '4.17', 'Title': 'Ensure write level Object Storage logging is enabled for all buckets.', 'Status': True, 'Level': 2, 'Total': [], 'Findings': [], 'CISv8': ['8.2'], 'CCCS Guard Rail': '', 'Remediation': []}, + '4.17': {'section': 'Logging and Monitoring', 'recommendation_#': '4.17', 'Title': 'Ensure write level Object Storage logging is enabled for all buckets.', 'Status': True, 'Level': 2, 'Total': [], 'Findings': [], 'CISv8': ['8.2'], 'CCCS Guard Rail': '11', 'Remediation': []}, + '4.18': {'section': 'Logging and Monitoring', 'recommendation_#': '4.18', 'Title': 'Ensure a notification is configured for Local OCI User Authentication.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['8.2'], 'CCCS Guard Rail': '11', 'Remediation': []}, '5.1.1': {'section': 'Storage - Object Storage', 'recommendation_#': '5.1.1', 'Title': 'Ensure no Object Storage buckets are publicly visible.', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'CISv8': ['3.3'], 'CCCS Guard Rail': '', 'Remediation': []}, '5.1.2': {'section': 'Storage - Object Storage', 'recommendation_#': '5.1.2', 'Title': 'Ensure Object Storage Buckets are encrypted with a Customer Managed Key (CMK).', 'Status': True, 'Level': 2, 'Total': [], 'Findings': [], 'CISv8': ['3.11'], 'CCCS Guard Rail': '', 'Remediation': []}, @@ -287,17 +306,32 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor "Description": "OCI instances, OCI database and OCI functions can access other OCI resources either via an OCI API key associated to a user or by being including in a Dynamic Group that has an IAM policy granting it the required access. Access to OCI Resources refers to making API calls to another OCI resource like Object Storage, OCI Vaults, etc.", "Rationale": "Dynamic Groups reduces the risks related to hard coded credentials. Hard coded API keys can be shared and require rotation which can open them up to being compromised. Compromised credentials could allow access to OCI services outside of the expected radius.", "Impact": "For an OCI instance that contains embedded credential audit the scripts and environment variables to ensure that none of them contain OCI API Keys or credentials.", - "Remediation": "Create Dynamic group and Enter Matching Rules to that includes the instances accessing your OCI resources. Refer:\"https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/managingdynamicgroups.htm\".", + "Remediation": "Create Dynamic group and Enter Matching Rules to that includes the instances accessing your OCI resources. Refer: \"https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/managingdynamicgroups.htm\".", "Recommendation": "Evaluate how your instances, functions, and autonomous database interact with other OCI services.", "Observation": "Dynamic Groups reduces the risks related to hard coded credentials. Hard coded API keys can be shared and require rotation which can open them up to being compromised. Compromised credentials could allow access to OCI services outside of the expected radius." }, "1.15": { "Description": "To apply the separation of duties security principle, one can restrict service-level administrators from being able to delete resources they are managing. It means service-level administrators can only manage resources of a specific service but not delete resources for that specific service.

Example policies for global/tenant level for block volume service-administrators:\n
\nAllow group VolumeUsers to manage volumes in tenancy where request.permission!='VOLUME_DELETE'\nAllow group VolumeUsers to manage volume-backups in tenancy where request.permission!='VOLUME_BACKUP_DELETE'\n

Example policies for global/tenant level for file storage system service-administrators:
\nAllow group FileUsers to manage file-systems in tenancy where request.permission!='FILE_SYSTEM_DELETE'\nAllow group FileUsers to manage mount-targets in tenancy where request.permission!='MOUNT_TARGET_DELETE'\nAllow group FileUsers to manage export-sets in tenancy where request.permission!='EXPORT_SET_DELETE'\n


Example policies for global/tenant level for object storage system service-administrators:
\nAllow group BucketUsers to manage objects in tenancy where request.permission!='OBJECT_DELETE'\nAllow group BucketUsers to manage buckets in tenancy where request.permission!='BUCKET_DELETE'\n
", - "Rationale": "Creating service-level administrators without the ability to delete the resource they are managing helps in tightly controlling access to Oracle Cloud Infrastructure (OCI) services by implementing the separation of duties security principle.", "Impact": "", + "Rationale": "Creating service-level administrators without the ability to delete the resource they are managing helps in tightly controlling access to Oracle Cloud Infrastructure (OCI) services by implementing the separation of duties security principle.", + "Impact": "", "Remediation": "Add the appropriate where condition to any policy statement that allows the storage service-level to manage the storage service.", "Recommendation": "To apply a separation of duties security principle, it is recommended to restrict service-level administrators from being able to delete resources they are managing.", "Observation": "IAM Policies that give service administrator the ability to delete service resources." }, + "1.16": { + "Description": "OCI IAM Local users can access OCI resources using different credentials, such as passwords or API keys. It is recommended that credentials that have been unused for 45 days or more be deactivated or removed.", + "Rationale": "Disabling or removing unnecessary OCI IAM local users will reduce the window of opportunity for credentials associated with a compromised or abandoned account to be used.", + "Impact": "Deactivating OCI IAM Local users and deleting of an OCI API Key will remove access to OCI", + "Remediation": "Deactivate OCI IAM Local users or delete of an OCI API Key will remove access to OCI", + "Observation": "User(s) with credentials unused in 45 days" + }, + "1.17":{ + "Description": "API Keys are long-term credentials for an OCI IAM user. They can be used to make programmatic requests to the OCI APIs directly or via, OCI SDKs or the OCI CLI.", + "Rationale": "Having a single API Key for an OCI IAM reduces attack surface area and makes it easier to manage.", + "Impact": "Deletion of an OCI API Key will remove programmatic access to OCI APIs.", + "Remediation": "Remove one of the API Keys.", + "Observation": "OCI API Key will grant programmatic access to OCI APIs with the same access as the user it is attached to." + }, "2.1": { "Description": "Security lists provide stateful or stateless filtering of ingress/egress network traffic to OCI resources on a subnet level. It is recommended that no security group allows unrestricted ingress access to port 22.", "Rationale": "Removing unfettered connectivity to remote console services, such as Secure Shell (SSH), reduces a server's exposure to risk.", @@ -331,10 +365,10 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor "Observation": "Network security groups that allow internet access to port 3389. (Note this does not necessarily mean external traffic can reach a compute instance)." }, "2.5": { - "Description": "A default security list is created when a Virtual Cloud Network (VCN) is created. Security lists provide stateful filtering of ingress and egress network traffic to OCI resources. It is recommended no security list allows unrestricted ingress access to Secure Shell (SSH) via port 22.", - "Rationale": "Removing unfettered connectivity to remote console services, such as SSH on port 22, reduces a server's exposure to unauthorized access.", + "Description": "A default security list is created when a Virtual Cloud Network (VCN) is created and attached to the public subnets in the VCN. Security lists provide stateful or stateless filtering of ingress and egress network traffic to OCI resources in the VCN. It is recommended that the default security list does not allow unrestricted ingress and egress access to resources in the VCN.", + "Rationale": "Removing unfettered connectivity to OCI resource, reduces a server's exposure to unauthorized access or data exfiltration.", "Impact": "For updating an existing environment, care should be taken to ensure that administrators currently relying on an existing ingress from 0.0.0.0/0 have access to ports 22 and/or 3389 through another security group.", - "Remediation": "Select Default Security List for and Remove the Ingress Rule with Source 0.0.0.0/0, IP Protocol 22 and Destination Port Range 22.", + "Remediation": "For updating an existing environment, care should be taken to ensure that administrators currently relying on an existing ingress from 0.0.0.0/0 have access to port 22 through another network security group and servers have egress to specified ports and protocols through another network security group.", "Recommendation": "Create specific custom security lists with workload specific rules and attach to subnets.", "Observation": "Default Security lists that allow more traffic then ICMP." }, @@ -368,7 +402,7 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor "Impact": "If you disable IMDSv1 on an instance that does not support IMDSv2, you might not be able to connect to the instance when you launch it.", "Remediation": "For each instance select the instance name, under the Instance Details section, next to Instance Metadata Service, click Edit and for the Instance metadata service, select the Version 2 only option.", "Recommendation": "It is recommended that all OCI instances use Instance Metadata Service version 2 (IMDSv2).", - "Observation": "Instances that allow Instance Metadata Service v1." + "Observation": "Instances that allow Instance Metadata Service v1." }, "3.2": { "Description": "Shielded Instances with Secure Boot enabled prevents unauthorized boot loaders and operating systems from booting. This prevent rootkits, bootkits, and unauthorized software from running before the operating system loads. Secure Boot verifies the digital signature of the system's boot software to check its authenticity. The digital signature ensures the operating system has not been tampered with and is from a trusted source. When the system boots and attempts to execute the software, it will first check the digital signature to ensure validity. If the digital signature is not valid, the system will not allow the software to run. Secure Boot is a feature of UEFI(Unified Extensible Firmware Interface) that only allows approved operating systems to boot up.", @@ -376,15 +410,15 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor "Impact": " To enable you have to terminate the instance and create a new one. Also, Shielded instances do not support live migration. During an infrastructure maintenance event, Oracle Cloud Infrastructure live migrates supported VM instances from the physical VM host that needs maintenance to a healthy VM host with minimal disruption to running instances. If you enable Secure Boot on an instance, the instance cannot be migrated, because the hardware TPM is not migratable. This may result in an outage because the TPM can't be migrate from a unhealthy host to healthy host.", "Remediation": "Terminate the old instance. Create a new instance and ensure on Secure Boot is toggled on under the Security section.", "Recommendation": "", - "Observation": "Instances that don't enable Secure Boot." + "Observation": "Instances that don't enable Secure Boot." }, "3.3": { "Description": "The Block Volume service provides the option to enable in-transit encryption for paravirtualized volume attachments on virtual machine (VM) instances.", "Rationale": "All the data moving between the instance and the block volume is transferred over an internal and highly secure network. If you have specific compliance requirements related to the encryption of the data while it is moving between the instance and the block volume, you should enable the in-transit encryption option.", "Impact": "In-transit encryption for boot and block volumes is only available for virtual machine (VM) instances launched from platform images, along with bare metal instances that use the following shapes: BM.Standard.E3.128, BM.Standard.E4.128, BM.DenseIO.E4.128. It is not supported on other bare metal instances.", - "Remediation": "Terminate the old instance. Create a new instance and ensure Use in-transit encryption is toggled on under the Boot volume section.", + "Remediation": "If available, edit the instance and enable in-transit encryption. If not available terminate the old instance. Create a new instance and ensure Use in-transit encryption is toggled on under the Boot volume section.", "Recommendation": "", - "Observation": "Instances that don't enable in-transit encryption." + "Observation": "Instances that don't enable in-transit encryption." }, "4.1": { "Description": "Using default tags is a way to ensure all resources that support tags are tagged during creation. Tags can be based on static values or based on computed values. It is recommended to setup default tags early on to ensure all created resources will get tagged.\nTags are scoped to Compartments and are inherited by Child Compartments. The recommendation is to create default tags like “CreatedBy” at the Root Compartment level to ensure all resources get tagged.\nWhen using Tags it is important to ensure that Tag Namespaces are protected by IAM Policies otherwise this will allow users to change tags or tag values.\nDepending on the age of the OCI Tenancy there may already be Tag defaults setup at the Root Level and no need for further action to implement this action.", @@ -498,13 +532,13 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor "Recommendation": "", "Observation": "Cloud Guard has not been configured in the root compartment of the tenancy." }, - "4.15" : { + "4.15": { "Description": "Cloud Guard detects misconfigured resources and insecure activity within a tenancy and provides security administrators with the visibility to resolve these issues. Upon detection, Cloud Guard generates a Problem. It is recommended to setup an Event Rule and Notification that gets triggered when Oracle Cloud Guard Problems are created, dismissed or remediated. Event Rules are compartment scoped and will detect events in child compartments. It is recommended to create the Event rule at the root compartment level.", "Rationale": "Cloud Guard provides an automated means to monitor a tenancy for resources that are configured in an insecure manner as well as risky network activity from these resources. Monitoring and alerting on Problems detected by Cloud Guard will help in identifying changes to the security posture.", "Impact": "There is no performance impact when enabling the above described features but depending on the amount of notifications sent per month there may be a cost associated.", - "Remediation": "Create a Rule Condition by selecting Cloud Guard in the Service Name Drop-down and selecting Detected – Problem, Remediated – Problem and Dismissed - Problem. In the Actions section select Notifications as Action Type and selct the compartment and topic to be used.", + "Remediation": "Create a Rule Condition by selecting Cloud Guard in the Service Name Drop-down and selecting Detected – Problem, Remediated – Problem and Dismissed - Problem. In the Actions section select Notifications as Action Type and select the compartment and topic to be used.", "Recommendation": "", - "Observation": "notifications have been configured for Cloud Guard Problems." + "Observation": "notifications have been configured for Cloud Guard Problems." }, "4.16": { "Description": "Oracle Cloud Infrastructure Vault securely stores master encryption keys that protect your encrypted data. You can use the Vault service to rotate keys to generate new cryptographic material. Periodically rotating keys limits the amount of data encrypted by one key version.", @@ -522,6 +556,14 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor "Recommendation": "", "Observation": "object stores have no write level logging enabled." }, + "4.18": { + "Description": "It is recommended that an Event Rule and Notification be set up when a user in the via OCI local authentication. Event Rules are compartment-scoped and will detect events in child compartments. This Event rule is required to be created at the root compartment level.", + "Rationale": "Users should rarely use OCI local authenticated and be authenticated via organizational standard Identity providers, not local credentials. Access in this matter would represent a break glass activity and should be monitored to see if changes made impact the security posture.", + "Impact": "There is no performance impact when enabling the above-described features but depending on the amount of notifications sent per month there may be a cost associated.", + "Remediation": "Create a Rule Condition by selecting Identity SignOn in the Service Name Drop-down and selecting and Event Types: Interactive Login. In the Actions section select Notifications as Action Type and select the compartment and topic to be used.", + "Recommendation": "", + "Observation": "notifications have been configured Local User Authentication." + }, "5.1.1": { "Description": "A bucket is a logical container for storing objects. It is associated with a single compartment that has policies that determine what action a user can perform on a bucket and on all the objects in the bucket. It is recommended that no bucket be publicly accessible.", "Rationale": "Removing unfettered reading of objects in a bucket reduces an organization's exposure to data loss.", @@ -602,7 +644,7 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor } # CIS and OBP Regional Data # 4.6 is not regional because OCI IAM Policies only exist in the home region - self.__cis_regional_checks = {"4.3","4.4","4.5","4.7", "4.8", "4.9", "4.10", "4.11", "4.12"} + self.__cis_regional_checks = {"4.3","4.4","4.5","4.7","4.8", "4.9", "4.10", "4.11", "4.12", "4.18"} self.__obp_regional_checks = {} # CIS monitoring notifications check @@ -684,10 +726,13 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor 'com.oraclecloud.servicegateway.updateservicegateway' ], - "4.15" : [ + "4.15": [ 'com.oraclecloud.cloudguard.problemdetected', 'com.oraclecloud.cloudguard.problemdismissed', 'com.oraclecloud.cloudguard.problemremediated' + ], + "4.18": [ + 'com.oraclecloud.identitysignon.interactivelogin' ] } @@ -734,6 +779,12 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor "volume-backups": ["request.permission=VOLUME_BACKUP_DELETE"], "boot-volume-backups": ["request.permission=BOOT_VOLUME_BACKUP_DELETE"]}} + # CIS Network Filter Check + self.all_traffic_rules = [{'ruleAction': 'INCLUDE', 'protocol': 'all', 'udpOptions': None, 'isEnabled': True, 'sourceCidr': None, + 'samplingRate': 1, 'flowLogType': 'ALL', 'destinationCidr': None, 'icmpOptions': None, 'priority': 0, 'tcpOptions': None}, + {'ruleAction': 'INCLUDE','protocol': 'all','udpOptions': None,'sourceCidr': '0.0.0.0/0','isEnabled': True,'samplingRate': 1, + 'flowLogType': 'ALL','icmpOptions': None,'destinationCidr': '0.0.0.0/0','priority': 0,'tcpOptions': None}] + # Tenancy Data self.__tenancy = None self.__cloud_guard_config = None @@ -755,6 +806,9 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor self.__network_security_groups = [] self.__network_security_lists = [] self.__network_subnets = [] + self.__network_vcns = {} + self.__network_capturefilters = {} + self.__network_fastconnects = {} # Indexed by DRG ID self.__network_drgs = {} # Indexed by DRG ID self.__raw_network_drgs = [] @@ -777,7 +831,8 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor # For Logging & Monitoring checks self.__event_rules = [] self.__logging_list = [] - self.__subnet_logs = {} + self.__subnet_logs = {} # to be deleted + self.__all_logs = {} self.__write_bucket_logs = {} self.__read_bucket_logs = {} self.__load_balancer_access_logs = [] @@ -923,23 +978,28 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor self.__raw_regions.append(record) # By Default it is today's date - self.__report_directory = f'{report_directory}/' if report_directory else f'{self.__tenancy.name}-{self.report_datetime}' + self.__report_directory = f'{report_directory}' if report_directory else f'{self.__tenancy.name}-{self.report_datetime}' self.__report_prefix = f'{report_prefix}_' if report_prefix else '' self.__report_summary_json = report_summary_json # Checking if a Tenancy has Identity Domains enabled try: - domains_checking_url = "https://login.oci.oraclecloud.com/v1/tenantMetadata/" + self.__tenancy.name - domains_check_raw = requests.get(url=domains_checking_url) - domains_check_dict = json.loads(domains_check_raw.content) - self.__identity_domains_enabled = domains_check_dict['flights']['isHenosisEnabled'] + oci.pagination.list_call_get_all_results( + self.__regions[self.__home_region]['identity_client'].list_domains, + compartment_id = self.__tenancy.id, + lifecycle_state = "ACTIVE", + name="Default" + ).data + self.__identity_domains_enabled=True + print_header("Identity Domains Enabled in Tenancy") except Exception as e: - # To be safe if it fails I'll check - self.__identity_domains_enabled = True - debug("__init__: Exception checking identity domains status\n" + str(e)) - self.__errors.append({"id" : "__init__", "error" : str(e)}) - + if e.status == 404: + print_header("Identity Domains Disabled in Tenancy") + self.__identity_domains_enabled = False + else: + raise RuntimeError( + "Failed to list identity domains." + str(e.args)) # Creating signers and config for all regions self.__create_regional_signers(proxy) @@ -973,6 +1033,9 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor self.__obp_checks = True self.__output_raw_data = True + # Determining if OCI API unused for 45 days check is disable or not + self.__disable_api_keys = disable_api_keys + # Determine if __oci_cloud_url will be override with a different realm ex. OC2 or sovreign region self.__oci_cloud_url = "https://cloud.oracle.com" if oci_url: @@ -989,6 +1052,7 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor self.__oci_block_volumes_uri = self.__oci_cloud_url + "/block-storage/volumes/" self.__oci_fss_uri = self.__oci_cloud_url + "/fss/file-systems/" self.__oci_networking_uri = self.__oci_cloud_url + "/networking/vcns/" + self.__oci_network_capturefilter_uri = self.__oci_cloud_url + "/networking/network-command-center/capture-filters/" self.__oci_adb_uri = self.__oci_cloud_url + "/db/adb/" self.__oci_oicinstance_uri = self.__oci_cloud_url + "/oic/integration-instances/" self.__oci_oacinstance_uri = self.__oci_cloud_url + "/analytics/instances/" @@ -1117,7 +1181,12 @@ def __create_regional_signers(self, proxy): if proxy: search.base_client.session.proxies = {'https': proxy} region_values['certificate_client'] = certificate_client - + + logging_search_client = oci.loggingsearch.LogSearchClient(region_config, signer=region_signer) + if proxy: + search.base_client.session.proxies = {'https': proxy} + region_values['logging_search_client'] = logging_search_client + except Exception as e: debug("__create_regional_signers: error reading" + str(self.__config)) self.__errors.append({"id" : "__create_regional_signers", "error" : str(e)}) @@ -1144,16 +1213,17 @@ def __set_managed_paas_compartment(self): def get_date_iso_format(self, val): if not val: return "" - return str(val)[0:19] + return str(val)[0:19].replace(" ","T") ########################################################################## # Load compartments ########################################################################## def __identity_read_compartments(self): print("\nProcessing Compartments...") + self.__compartments = [] try: debug("__identity_read_compartments: Processing Compartments:") - self.__compartments = oci.pagination.list_call_get_all_results( + self.__compartments += oci.pagination.list_call_get_all_results( self.__regions[self.__home_region]['identity_client'].list_compartments, compartment_id=self.__tenancy.id, compartment_id_in_subtree=True, @@ -1179,11 +1249,11 @@ def __identity_read_compartments(self): "region": "" } self.__raw_compartment.append(record) - self.cis_foundations_benchmark_2_0['6.1']['Total'].append(compartment) + self.cis_foundations_benchmark_3_0['6.1']['Total'].append(compartment) # Add root compartment which is not part of the list_compartments self.__compartments.append(self.__tenancy) - deep_link = self.__oci_compartment_uri + compartment.id + deep_link = self.__oci_compartment_uri + self.__tenancy.id root_compartment = { "id": self.__tenancy.id, "name": self.__tenancy.name, @@ -1248,7 +1318,7 @@ def __identity_read_domains(self): # Creating Identity Domains Client and storing it debug("__identity_read_domains: Creating Identity Domain Client for: " + domain.display_name) domain_dict['IdentityDomainClient'] = oci.identity_domains.IdentityDomainsClient(\ - config=self.__config, service_endpoint=domain.url) + config=self.__config, signer=self.__signer, service_endpoint=domain.url) debug("__identity_read_domains: Created Identity Domain Client for: " + domain.display_name) pwd_policy_dict = oci.util.to_dict(domain_dict['IdentityDomainClient'].get_password_policy(\ @@ -1257,6 +1327,8 @@ def __identity_read_domains(self): domain_dict['password_policy'] = pwd_policy_dict domain_dict['errors'] = None self.__identity_domains.append(domain_dict) + debug("-" * 100) + debug(f"__identity_read_domains: Domain Dict is: {domain_dict}") except Exception as e: debug("Identity Domains Error is for domain " + domain.display_name + "\n" + str(e)) @@ -1446,6 +1518,7 @@ def __identity_read_users(self): 'can_use_db_credentials': user.urn_ietf_params_scim_schemas_oracle_idcs_extension_capabilities_user.can_use_db_credentials if user.urn_ietf_params_scim_schemas_oracle_idcs_extension_capabilities_user else None, 'can_use_o_auth2_client_credentials': user.urn_ietf_params_scim_schemas_oracle_idcs_extension_capabilities_user.can_use_o_auth2_client_credentials if user.urn_ietf_params_scim_schemas_oracle_idcs_extension_capabilities_user else None, 'can_use_smtp_credentials': user.urn_ietf_params_scim_schemas_oracle_idcs_extension_capabilities_user.can_use_smtp_credentials if user.urn_ietf_params_scim_schemas_oracle_idcs_extension_capabilities_user else None, + 'last_successful_login_date': self.get_date_iso_format(user.urn_ietf_params_scim_schemas_oracle_idcs_extension_user_state_user.last_successful_login_date) if user.urn_ietf_params_scim_schemas_oracle_idcs_extension_user_state_user else None, 'groups': [] } # Adding Groups to the user @@ -1457,13 +1530,13 @@ def __identity_read_users(self): record['api_keys'] = self.__identity_read_user_api_key(user_ocid=user.ocid, identity_domain=identity_domain) record['auth_tokens'] = self.__identity_read_user_auth_token(user.ocid, identity_domain=identity_domain) record['customer_secret_keys'] = self.__identity_read_user_customer_secret_key(user.ocid, identity_domain=identity_domain) - record['database_passowrds'] = self.__identity_read_user_database_password(user.ocid,identity_domain=identity_domain) + record['database_passwords'] = self.__identity_read_user_database_password(user.ocid,identity_domain=identity_domain) else: debug("__identity_read_users: skipping user API Key collection for user: " + str(user.user_name)) record['api_keys'] = None record['auth_tokens'] = None record['customer_secret_keys'] = None - record['database_passowrds'] = None + record['database_passwords'] = None self.__users.append(record) except Exception as e: @@ -1507,6 +1580,7 @@ def __identity_read_users(self): 'can_use_db_credentials': user.capabilities.can_use_db_credentials, 'can_use_o_auth2_client_credentials': user.capabilities.can_use_o_auth2_client_credentials, 'can_use_smtp_credentials': user.capabilities.can_use_smtp_credentials, + 'last_successful_login_date': self.get_date_iso_format(user.last_successful_login_time), 'groups': [] } # Adding Groups to the user @@ -1519,7 +1593,7 @@ def __identity_read_users(self): user.id) record['customer_secret_keys'] = self.__identity_read_user_customer_secret_key( user.id) - record['database_passowrds'] = self.__identity_read_user_database_password(user.id) + record['database_passwords'] = self.__identity_read_user_database_password(user.id) self.__users.append(record) print("\tProcessed " + str(len(self.__users)) + " Users") return self.__users @@ -1532,6 +1606,7 @@ def __identity_read_users(self): except Exception as e: raise RuntimeError( "Error in __identity_read_users: " + str(e.args)) + ########################################################################## # Load user api keys ########################################################################## @@ -1551,8 +1626,11 @@ def __identity_read_user_api_key(self, user_ocid, identity_domain=None): record = oci.util.to_dict(api_key) record['deep_link'] = self.__generate_csv_hyperlink(deep_link, api_key.fingerprint) record['time_created'] = self.get_date_iso_format(record['meta']['created']) + apikey_used_in_45_days = self.__identity_check_logging_for_api_activity(user_ocid=user_ocid, api_key=api_key.fingerprint) + record['apikey_used_in_45_days'] = apikey_used_in_45_days api_keys.append(record) + else: user_api_keys_data = oci.pagination.list_call_get_all_results( self.__regions[self.__home_region]['identity_client'].list_api_keys, @@ -1565,6 +1643,8 @@ def __identity_read_user_api_key(self, user_ocid, identity_domain=None): record['deep_link'] = self.__generate_csv_hyperlink(deep_link, api_key.fingerprint) record['id'] = record['key_id'] record['time_created'] = self.get_date_iso_format(record['time_created']) + apikey_used_in_45_days = self.__identity_check_logging_for_api_activity(user_ocid=user_ocid, api_key=api_key.fingerprint) + record['apikey_used_in_45_days'] = apikey_used_in_45_days api_keys.append(record) return api_keys @@ -1577,6 +1657,116 @@ def __identity_read_user_api_key(self, user_ocid, identity_domain=None): raise RuntimeError( "Error in identity_read_user_api_key: " + str(e.args)) + + ########################################################################## + # Search API Key Last Usage Over 45 days https://github.com/tstahl/oci-remove-unused-apikey-cg-responder/blob/main/func.py + ########################################################################## + def __identity_check_logging_for_api_activity(self, user_ocid, api_key): + + apikey_used_in_45_days = [] + + def numOfDays(date1, date2): + #check which date is greater to avoid days output in -ve number + if date2 > date1: + return (date2-date1).days + else: + return (date1-date2).days + + ########################################################################## + # Inputs: start_date(date), end_date(date), data_ranges(list) max_days_between(int) + # Returns: List of dicts with {"start_date" : start_date, "end_date" : end_date}} + ########################################################################## + def get_date_ranges(start_date, end_date, date_ranges, max_days_between=9): + days_between = numOfDays(start_date, end_date) + if days_between > max_days_between: + next_date = start_date + datetime.timedelta(days=max_days_between) + date_ranges.append({"start_date" : start_date, "end_date" : next_date}) + return get_date_ranges(next_date + datetime.timedelta(days=1), end_date, date_ranges, max_days_between=max_days_between) + else: + + date_ranges.append({"start_date" : start_date, "end_date" : end_date}) + return date_ranges + + ########################################################################## + # Inputs: search_query, start_date and end_date in datetime, results + # Returns: Bool if the key was used in + ########################################################################## + def run_logging_search_query_api_usage(search_query, api_key_used, start_date: datetime, end_date: datetime): + if self.__disable_api_keys: + print("***Skipping Processing Audit Logs for API Key Usage...***") + return api_key_used + else: + print("Processing Audit Logs for API Key Usage...") + for region_key, region_values in self.__regions.items(): + try: + + response = region_values['logging_search_client'].search_logs( + search_logs_details=oci.loggingsearch.models.SearchLogsDetails( + search_query=search_query, + time_start=start_date, + time_end=end_date, + is_return_field_info=False), + limit=100) + + audit_logs = response.data + debug(f"run_logging_search_query_api_usage: response is: {response.data}") + if audit_logs.summary.result_count > 0: + for result in audit_logs.results: + userInfo = { + "principalName" : result.data["data.identity.principalName"], + "principalId" : result.data["data.identity.principalId"] + } + debug(f'run_logging_search_query_api_usage: Audit search results: {userInfo}') + api_key_used.append(userInfo) + break + + else: + debug('run_logging_search_query_api_usage: No APIKey usage records found in the past 14 days in') + + return api_key_used + except Exception as e: + self.__errors.append({"id" : "run_logging_search_query_api_usage", "error" : str(e)}) + debug('run_logging_search_query_api_usage: Exception is:') + debug("\tException is : " + str(e)) + return api_key_used + + debug("__identity_check_logging_for_api_activity: Checking API Key") + principle_id = f'{self.__tenancy.id}/{user_ocid}/{api_key}' + debug(f'__identity_check_logging_for_api_activity: API key is: {api_key}') + + tenancy_search_str = f'\"{self.__tenancy.id}/_Audit_Include_Subcompartment\"' + search_query = "search " + tenancy_search_str + """ | data.identity.credentials = '""" + principle_id + """' and data.identity.tenantId = '""" + self.__tenancy.id + """' | summarize count() by data.identity.principalId, data.identity.principalName""" + debug(f'__identity_check_logging_for_api_activity: Search Query is: {search_query}') + + end_date = self.start_datetime + start_date = end_date - datetime.timedelta(days=self.__days_used) + + search_date_range = get_date_ranges(start_date=start_date, \ + end_date=end_date, \ + date_ranges=[], + max_days_between=13) + + debug(f'__identity_check_logging_for_api_activity: Initiated Threads for dates range : {str(search_date_range)}') + + threads = [] + for dates in search_date_range: + thread = Thread(target=run_logging_search_query_api_usage, \ + args=(search_query, apikey_used_in_45_days, \ + dates['start_date'], dates['end_date'])) + threads.append(thread) + + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + + if apikey_used_in_45_days: + return True + else: + return False + + ########################################################################## # Load user auth tokens ########################################################################## @@ -1681,6 +1871,7 @@ def __identity_read_user_database_password(self, user_ocid, identity_domain=None deep_link = self.__oci_users_uri + "/domains/" + identity_domain['id'] + "/users/" + user_ocid + "/db-passwords" record = oci.util.to_dict(password) record['deep_link'] = deep_link + record['time_created'] = self.get_date_iso_format(record['meta']['created']) database_password.append(record) return database_password @@ -1701,7 +1892,9 @@ def __identity_read_user_database_password(self, user_ocid, identity_domain=None debug("__identity_read_user_database_password: Got Password") deep_link = self.__oci_users_uri + user_ocid + "/db-password" record = oci.util.to_dict(password) + record['ocid'] = record['id'] record['deep_link'] = deep_link + record['time_created'] = self.get_date_iso_format(record['time_created']) database_password.append(record) return database_password @@ -1722,7 +1915,8 @@ def __identity_read_tenancy_policies(self): policies_data = oci.pagination.list_call_get_all_results( self.__regions[self.__home_region]['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query Policy resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query Policy resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data for policy in policies_data: @@ -1818,7 +2012,9 @@ def __os_read_buckets(self): buckets_data = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query Bucket resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query Bucket resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id + ).data # Getting Bucket Info for bucket in buckets_data: @@ -1843,6 +2039,7 @@ def __os_read_buckets(self): "defined_tags": bucket_info.defined_tags, "freeform_tags": bucket_info.freeform_tags, "region": region_key, + "source_resource" : bucket_info.name + "-" + region_key, "notes": "" } self.__buckets.append(record) @@ -1864,6 +2061,7 @@ def __os_read_buckets(self): "defined_tags": bucket.defined_tags, "freeform_tags": "", "region": region_key, + "source_resource" : bucket.display_name + "-" + region_key, "notes": str(e) } self.__buckets.append(record) @@ -1882,7 +2080,9 @@ def __block_volume_read_block_volumes(self): volumes_data = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query Volume resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query Volume resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id + ).data # Getting Block Volume inf @@ -1898,18 +2098,11 @@ def __block_volume_read_block_volumes(self): "compartment_id": volume.compartment_id, "size_in_gbs": volume.additional_details['sizeInGBs'], "size_in_mbs": volume.additional_details['sizeInMBs'], - # "source_details": volume.source_details, - "time_created": volume.time_created.strftime(self.__iso_time_format), - # "volume_group_id": volume.volume_group_id, - # "vpus_per_gb": volume.vpus_per_gb, - # "auto_tuned_vpus_per_gb": volume.auto_tuned_vpus_per_gb, "availability_domain": volume.availability_domain, - # "block_volume_replicas": volume.block_volume_replicas, - # "is_auto_tune_enabled": volume.is_auto_tune_enabled, - # "is_hydrated": volume.is_hydrated, + "time_created": volume.time_created.strftime(self.__iso_time_format), + "system_tags": volume.system_tags, "defined_tags": volume.defined_tags, "freeform_tags": volume.freeform_tags, - "system_tags": volume.system_tags, "region": region_key, "notes": "" } @@ -1923,18 +2116,11 @@ def __block_volume_read_block_volumes(self): "compartment_id": "", "size_in_gbs": "", "size_in_mbs": "", - # "source_details": "", - "time_created": "", - # "volume_group_id": "", - # "vpus_per_gb": "", - # "auto_tuned_vpus_per_gb": "", "availability_domain": "", - # "block_volume_replicas": "", - # "is_auto_tune_enabled": "", - # "is_hydrated": "", + "time_created": "", + "system_tags": "", "defined_tags": "", "freeform_tags": "", - "system_tags": "", "region": region_key, "notes": str(e) } @@ -1953,7 +2139,8 @@ def __boot_volume_read_boot_volumes(self): boot_volumes_data = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query BootVolume resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query BootVolume resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data for boot_volume in boot_volumes_data: @@ -1963,20 +2150,13 @@ def __boot_volume_read_boot_volumes(self): "id": boot_volume.identifier, "display_name": boot_volume.display_name, "deep_link": self.__generate_csv_hyperlink(deep_link, boot_volume.display_name), - # "image_id": boot_volume.image_id, "kms_key_id": boot_volume.additional_details['kmsKeyId'], "lifecycle_state": boot_volume.lifecycle_state, + "compartment_id": boot_volume.compartment_id, "size_in_gbs": boot_volume.additional_details['sizeInGBs'], "size_in_mbs": boot_volume.additional_details['sizeInMBs'], "availability_domain": boot_volume.availability_domain, "time_created": boot_volume.time_created.strftime(self.__iso_time_format), - "compartment_id": boot_volume.compartment_id, - # "auto_tuned_vpus_per_gb": boot_volume.auto_tuned_vpus_per_gb, - # "boot_volume_replicas": boot_volume.boot_volume_replicas, - # "is_auto_tune_enabled": boot_volume.is_auto_tune_enabled, - # "is_hydrated": boot_volume.is_hydrated, - # "source_details": boot_volume.source_details, - # "vpus_per_gb": boot_volume.vpus_per_gb, "system_tags": boot_volume.system_tags, "defined_tags": boot_volume.defined_tags, "freeform_tags": boot_volume.freeform_tags, @@ -1987,21 +2167,14 @@ def __boot_volume_read_boot_volumes(self): record = { "id": boot_volume.identifier, "display_name": boot_volume.display_name, - "deep_link": self.__generate_csv_hyperlink(deep_link, boot_volume.display_name), - # "image_id": "", + "deep_link": "", "kms_key_id": "", "lifecycle_state": "", + "compartment_id": "", "size_in_gbs": "", "size_in_mbs": "", "availability_domain": "", "time_created": "", - "compartment_id": "", - # "auto_tuned_vpus_per_gb": "", - # "boot_volume_replicas": "", - # "is_auto_tune_enabled": "", - # "is_hydrated": "", - # "source_details": "", - # "vpus_per_gb": "", "system_tags": "", "defined_tags": "", "freeform_tags": "", @@ -2023,7 +2196,8 @@ def __fss_read_fsss(self): fss_data = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query FileSystem resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query FileSystem resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data for fss in fss_data: @@ -2085,7 +2259,8 @@ def __network_read_network_security_groups_rules(self): nsgs_data = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query NetworkSecurityGroup resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query NetworkSecurityGroup resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data # Looping through NSGs to to get @@ -2149,7 +2324,8 @@ def __network_read_network_security_lists(self): security_lists_data = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query SecurityList resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query SecurityList resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data # Looping through Security Lists to to get @@ -2216,7 +2392,8 @@ def __network_read_network_subnets(self): subnets_data = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query Subnet resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query Subnet resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data try: @@ -2284,6 +2461,63 @@ def __network_read_network_subnets(self): raise RuntimeError( "Error in __network_read_network_subnets " + str(e.args)) + ########################################################################## + # Network VCNs Lists + ########################################################################## + def __network_read_network_vcns(self): + try: + for region_key, region_values in self.__regions.items(): + vcn_data = oci.pagination.list_call_get_all_results( + region_values['search_client'].search_resources, + search_details=oci.resource_search.models.StructuredSearchDetails( + query="query VCN resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id + ).data + + for vcn in vcn_data: + deep_link = self.__oci_networking_uri + vcn.identifier + '?region=' + region_key + record = oci.util.to_dict(vcn) + record['deep_link'] = deep_link + record['subnets'] = {} + record['network_security_groups'] = {} + record['security_lists'] = {} + # Adding VCN to VCN list + self.__network_vcns[vcn.identifier] = record + + print("\tProcessed " + str(len(self.__network_vcns)) + " Virtual Cloud Networks ") + return self.__network_vcns + except Exception as e: + raise RuntimeError( + "Error in __network_read_network_vcns " + str(e.args)) + + ########################################################################## + # Network Capture Filters Dictionary + ########################################################################## + def __network_read_network_capturefilters(self): + try: + for region_key, region_values in self.__regions.items(): + capturefilter_data = oci.pagination.list_call_get_all_results( + region_values['search_client'].search_resources, + search_details=oci.resource_search.models.StructuredSearchDetails( + query="query capturefilter resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id + ).data + + for filter in capturefilter_data: + deep_link = self.__oci_network_capturefilter_uri + filter.identifier + '?region=' + region_key + record = oci.util.to_dict(filter) + record['deep_link'] = deep_link + + # Adding CaptureFilter to CaptureFilter Dict + self.__network_capturefilters[filter.identifier] = record + + print("\tProcessed " + str(len(self.__network_capturefilters)) + " Network Capture Filters ") + + return self.__network_subnets + except Exception as e: + raise RuntimeError( + "Error in __network_read_network_capturefilters " + str(e.args)) + ########################################################################## # Load DRG Attachments ########################################################################## @@ -2295,7 +2529,8 @@ def __network_read_drg_attachments(self): drg_resources = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query DrgAttachment resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query DrgAttachment resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data compartments = set() @@ -2380,7 +2615,8 @@ def __network_read_drgs(self): drg_resources = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query Drg resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query Drg resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data compartments = set() @@ -2464,7 +2700,8 @@ def __network_read_fastonnects(self): fastconnects = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query VirtualCircuit resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query VirtualCircuit resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data compartments = set() @@ -2578,7 +2815,8 @@ def __network_read_ip_sec_connections(self): ip_sec_connections_data = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query IPSecConnection resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query IPSecConnection resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data for ip_sec in ip_sec_connections_data: @@ -2677,7 +2915,7 @@ def __network_read_ip_sec_connections(self): ############################################ def __network_topology_dump(self): debug("__network_topology_dump: Starting") - if type(self.__signer) is not oci.auth.signers.InstancePrincipalsDelegationTokenSigner: + if type(self.__signer) is oci.auth.signers.InstancePrincipalsDelegationTokenSigner: self.__errors.append({"id": "__network_topology_dump", "error": "Delegated Tokens via Cloud Shell not supported." }) return def api_function(region_key, region_values, tenancy_id): @@ -2720,7 +2958,8 @@ def __adb_read_adbs(self): adb_query_resources = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query AutonomousDatabase resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query AutonomousDatabase resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data compartments = set() @@ -2769,7 +3008,8 @@ def __oic_read_oics(self): oic_resources = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query IntegrationInstance resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query IntegrationInstance resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data compartments = set() @@ -2846,7 +3086,8 @@ def __oac_read_oacs(self): oac_resources = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query AnalyticsInstance resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query AnalyticsInstance resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data compartments = set() @@ -2916,7 +3157,8 @@ def __events_read_event_rules(self): events_rules_data = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query EventRule resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query EventRule resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data for event_rule in events_rules_data: @@ -2951,7 +3193,8 @@ def __logging_read_log_groups_and_logs(self): log_groups = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query LogGroup resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query LogGroup resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data # Looping through log groups to get logs @@ -2978,6 +3221,7 @@ def __logging_read_log_groups_and_logs(self): log_group_id=log_group.identifier ).data for log in logs: + deep_link = self.__oci_loggroup_uri + log_group.identifier + "/logs/" + log.id + '?region=' + region_key log_record = { "compartment_id": log.compartment_id, @@ -2992,41 +3236,100 @@ def __logging_read_log_groups_and_logs(self): "time_created": log.time_created.strftime(self.__iso_time_format), "time_last_modified": str(log.time_last_modified), "defined_tags": log.defined_tags, - "freeform_tags": log.freeform_tags + "freeform_tags": log.freeform_tags, + "region" : region_key } try: - if log.configuration: - log_record["configuration_compartment_id"] = log.configuration.compartment_id, - log_record["source_category"] = log.configuration.source.category, - log_record["source_parameters"] = log.configuration.source.parameters, - log_record["source_resource"] = log.configuration.source.resource, - log_record["source_service"] = log.configuration.source.service, - log_record["source_source_type"] = log.configuration.source.source_type - log_record["archiving_enabled"] = log.configuration.archiving.is_enabled - - if log.configuration.source.service == 'flowlogs': - self.__subnet_logs[log.configuration.source.resource] = {"log_group_id": log.log_group_id, "log_id": log.id} - - elif log.configuration.source.service == 'objectstorage' and 'write' in log.configuration.source.category: - # Only write logs - self.__write_bucket_logs[log.configuration.source.resource] = {"log_group_id": log.log_group_id, "log_id": log.id, "region": region_key} - - elif log.configuration.source.service == 'objectstorage' and 'read' in log.configuration.source.category: - # Only read logs - self.__read_bucket_logs[log.configuration.source.resource] = {"log_group_id": log.log_group_id, "log_id": log.id, "region": region_key} - - elif log.configuration.source.service == 'loadbalancer' and 'error' in log.configuration.source.category: - self.__load_balancer_error_logs.append( - log.configuration.source.resource) - elif log.configuration.source.service == 'loadbalancer' and 'access' in log.configuration.source.category: - self.__load_balancer_access_logs.append( - log.configuration.source.resource) - elif log.configuration.source.service == 'apigateway' and 'access' in log.configuration.source.category: - self.__api_gateway_access_logs.append( - log.configuration.source.resource) - elif log.configuration.source.service == 'apigateway' and 'error' in log.configuration.source.category: - self.__api_gateway_error_logs.append( - log.configuration.source.resource) + try: + if log_record["log_type"] == "SERVICE" and log_record['lifecycle_state'] == "ACTIVE": + log_record["configuration_compartment_id"] = log.configuration.compartment_id + log_record["source_category"] = log.configuration.source.category + log_record["source_parameters"] = log.configuration.source.parameters + log_record["source_source_type"] = log.configuration.source.source_type + log_record["source_service"] = log.configuration.source.service + # Object storage buckets are indexed by BucketName-region + if log_record["source_service"] == "objectstorage": + log_record["source_resource"] = log.configuration.source.resource + "-" + region_key + else: + log_record["source_resource"] = log.configuration.source.resource + log_record["archiving_enabled"] = log.configuration.archiving.is_enabled + if log_record["source_parameters"] and isinstance(log_record["source_parameters"],dict): + log_record["capture_filter"] = log.configuration.source.parameters["capture_filter"] + else: + log_record["capture_filter"] = None + + elif log_record["lifecycle_state"] == "ACTIVE": + log_record["source_category"] = log.log_type + log_record["source_service"] = log.log_type + log_record["source_resource"] = log.id + log_record["capture_filter"] = None + except Exception as e: + print(log) + print(e) + + #### TESTING SOMETHING NEW #### + + try: + ## Active means your logging + if log_record['lifecycle_state'] == 'ACTIVE': + if self.__all_logs: + + if log_record["source_service"] in self.__all_logs: + + if log_record["source_category"] in self.__all_logs[log_record["source_service"]]: + debug("\t__logging_read_log_groups_and_logs: Adding log for existing service and category ") + self.__all_logs[log_record["source_service"]][log_record["source_category"]][log_record["source_resource"]] = log_record + + else: + debug(f'\t__logging_read_log_groups_and_logs: Adding category {log_record["source_category"]}') + self.__all_logs[log_record["source_service"]][log_record["source_category"]] = {} + self.__all_logs[log_record["source_service"]][log_record["source_category"]][log_record["source_resource"]] = log_record + + else: + debug(f'\t__logging_read_log_groups_and_logs: Adding Service {log_record["source_service"]}, and category {log_record["source_category"]}') + self.__all_logs[log_record["source_service"]] = {} + self.__all_logs[log_record["source_service"]][log_record["source_category"]] = {} + self.__all_logs[log_record["source_service"]][log_record["source_category"]][log_record["source_resource"]] = log_record + + else: + debug(f'\t__logging_read_log_groups_and_logs: Starting Dict: Adding Service {log_record["source_service"]}, and category {log_record["source_category"]}' ) + self.__all_logs[log_record["source_service"]] = {} + self.__all_logs[log_record["source_service"]][log_record["source_category"]] = {} + self.__all_logs[log_record["source_service"]][log_record["source_category"]][log_record["source_resource"]] = log_record + + + except Exception as e: + print(f'\tFailed to parse log: {log_record["id"]}') + self.__errors.append({"id" : log_record["id"], "error" : str(e)}) + print("*" * 80) + print(log_record) + print("#" * 80) + print(e) + print("*" * 80) + + # if log.configuration.source.service == 'flowlogs': + # self.__subnet_logs[log.configuration.source.resource] = {"log_group_id": log.log_group_id, "log_id": log.id} + + # elif log.configuration.source.service == 'objectstorage' and 'write' in log.configuration.source.category: + # # Only write logs + # self.__write_bucket_logs[log.configuration.source.resource] = {"log_group_id": log.log_group_id, "log_id": log.id, "region": region_key} + + # elif log.configuration.source.service == 'objectstorage' and 'read' in log.configuration.source.category: + # # Only read logs + # self.__read_bucket_logs[log.configuration.source.resource] = {"log_group_id": log.log_group_id, "log_id": log.id, "region": region_key} + + # elif log.configuration.source.service == 'loadbalancer' and 'error' in log.configuration.source.category: + # self.__load_balancer_error_logs.append( + # log.configuration.source.resource) + # elif log.configuration.source.service == 'loadbalancer' and 'access' in log.configuration.source.category: + # self.__load_balancer_access_logs.append( + # log.configuration.source.resource) + # elif log.configuration.source.service == 'apigateway' and 'access' in log.configuration.source.category: + # self.__api_gateway_access_logs.append( + # log.configuration.source.resource) + # elif log.configuration.source.service == 'apigateway' and 'error' in log.configuration.source.category: + # self.__api_gateway_error_logs.append( + # log.configuration.source.resource) except Exception as e: self.__errors.append({"id" : log.id, "error" : str(e)}) # Append Log to log List @@ -3034,11 +3337,11 @@ def __logging_read_log_groups_and_logs(self): except Exception as e: self.__errors.append({"id" : log_group.identifier, "error" : str(e) }) record['notes'] = str(e) - - self.__logging_list.append(record) print("\tProcessed " + str(len(self.__logging_list)) + " Log Group Logs") + # print(self.__all_logs) + return self.__logging_list except Exception as e: raise RuntimeError( @@ -3055,7 +3358,8 @@ def __kms_read_keys(self): keys_data = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query Key resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query Key resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data vaults_set = set() @@ -3300,7 +3604,8 @@ def __ons_read_subscriptions(self): subs_data = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query OnsSubscription resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query OnsSubscription resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data debug("\t__ons_read_subscriptions: Recieved " + str(len(subs_data)) + " subscriptions in region " + str(region_key)) for sub in subs_data: @@ -3372,7 +3677,8 @@ def __sch_read_service_connectors(self): service_connectors_data = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query ServiceConnector resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'") + query="query ServiceConnector resources return allAdditionalFields where compartmentId != '" + self.__managed_paas_compartment_id + "'"), + tenant_id=self.__tenancy.id ).data # Getting Bucket Info @@ -3449,7 +3755,8 @@ def __search_resources_in_root_compartment(self): structured_search_query = oci.resource_search.models.StructuredSearchDetails(query=query_non_compliant) search_results = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, - search_details=structured_search_query + search_details=structured_search_query, + tenant_id=self.__tenancy.id ).data for item in search_results: @@ -3464,7 +3771,8 @@ def __search_resources_in_root_compartment(self): structured_search_all_query = oci.resource_search.models.StructuredSearchDetails(query=query_all_resources) structured_search_all_resources = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, - search_details=structured_search_all_query + search_details=structured_search_all_query, + tenant_id=self.__tenancy.id ).data for item in structured_search_all_resources: @@ -3476,7 +3784,7 @@ def __search_resources_in_root_compartment(self): "id": item.identifier, "region": region_key } - self.cis_foundations_benchmark_2_0['6.2']['Total'].append(record) + self.cis_foundations_benchmark_3_0['6.2']['Total'].append(record) except Exception: self.__errors.append({"id": "search_resources_in_root_compartment Invalid OCID", "error" : str(item)}) debug(f'__search_resources_in_root_compartment: Invalid OCID: {str(item)}') @@ -3498,7 +3806,8 @@ def __search_query_resource_type(self, resource_type, search_client): results = oci.pagination.list_call_get_all_results( search_client.search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query=query) + query=query), + tenant_id=self.__tenancy.id ).data return oci.util.to_dict(results) @@ -3565,9 +3874,11 @@ def __core_instance_read_compute(self): # Returning Instances print("\tProcessed " + str(len(self.__Instance)) + " Compute Instances") - return self.__service_connectors + return self.__Instance except Exception as e: - raise RuntimeError("Error in __core_instance_read_compute " + str(e.args)) + print("__core_instance_read_compute failed to process: " + str(e)) + self.__errors.append({'id' : '__core_instance_read_compute', 'error' : str(e)}) + return [] ########################################################################## @@ -3590,7 +3901,8 @@ def __certificates_read_certificates(self): certificates_data = oci.pagination.list_call_get_all_results( region_values['search_client'].search_resources, search_details=oci.resource_search.models.StructuredSearchDetails( - query="query certificate resources return allAdditionalFields") + query="query certificate resources return allAdditionalFields"), + tenant_id=self.__tenancy.id ).data cert_compartments = {} debug("\t__certificates_read_certificates: Got Ceritificates from ") @@ -3619,8 +3931,18 @@ def __certificates_read_certificates(self): debug("__certificates_read_certificates failed to process: " + str(e)) print("\tProcessed " + str(len(self.__raw_oci_certificates)) + " Certificates") - - + ########################################################################## + # Unifying Network information into a single object for easier processing + ########################################################################## + def __unify_network_data(self): + for subnet in self.__network_subnets: + self.__network_vcns[subnet['vcn_id']]['subnets'][subnet['id']] = subnet + for nsg in self.__network_security_groups: + self.__network_vcns[nsg['vcn_id']]['network_security_groups'][nsg['id']] = nsg + for sl in self.__network_security_lists: + self.__network_vcns[sl['vcn_id']]['security_lists'][sl['id']] = sl + + ########################################################################## # Analyzes Tenancy Data for CIS Report ########################################################################## @@ -3643,8 +3965,8 @@ def __report_cis_analyze_tenancy_data(self): and ("to manage all-resources".upper() in statement.upper()) \ and policy['name'].upper() != "Tenant Admin Policy".upper(): # If there are more than manage all-resources in you don't meet this rule - self.cis_foundations_benchmark_2_0['1.1']['Status'] = False - self.cis_foundations_benchmark_2_0['1.1']['Findings'].append(policy) + self.cis_foundations_benchmark_3_0['1.1']['Status'] = False + self.cis_foundations_benchmark_3_0['1.1']['Findings'].append(policy) break # 1.2 Check @@ -3654,8 +3976,8 @@ def __report_cis_analyze_tenancy_data(self): and "to manage all-resources in tenancy".upper() in statement.upper() \ and policy['name'].upper() != "Tenant Admin Policy".upper(): - self.cis_foundations_benchmark_2_0['1.2']['Status'] = False - self.cis_foundations_benchmark_2_0['1.2']['Findings'].append( + self.cis_foundations_benchmark_3_0['1.2']['Status'] = False + self.cis_foundations_benchmark_3_0['1.2']['Findings'].append( policy) # 1.3 Check - May want to add a service check @@ -3671,78 +3993,98 @@ def __report_cis_analyze_tenancy_data(self): if all(permission.upper() in clean_where_clause for permission in self.cis_iam_checks['1.3']["targets"]): pass else: - self.cis_foundations_benchmark_2_0['1.3']['Findings'].append(policy) - self.cis_foundations_benchmark_2_0['1.3']['Status'] = False + self.cis_foundations_benchmark_3_0['1.3']['Findings'].append(policy) + self.cis_foundations_benchmark_3_0['1.3']['Status'] = False else: - self.cis_foundations_benchmark_2_0['1.3']['Findings'].append(policy) - self.cis_foundations_benchmark_2_0['1.3']['Status'] = False + self.cis_foundations_benchmark_3_0['1.3']['Findings'].append(policy) + self.cis_foundations_benchmark_3_0['1.3']['Status'] = False # CIS Total 1.1,1,2,1.3 Adding - All IAM Policies for to CIS Total - self.cis_foundations_benchmark_2_0['1.1']['Total'] = self.__policies - self.cis_foundations_benchmark_2_0['1.2']['Total'] = self.__policies - self.cis_foundations_benchmark_2_0['1.3']['Total'] = self.__policies + self.cis_foundations_benchmark_3_0['1.1']['Total'] = self.__policies + self.cis_foundations_benchmark_3_0['1.2']['Total'] = self.__policies + self.cis_foundations_benchmark_3_0['1.3']['Total'] = self.__policies # 1.4 Check - Password Policy - Only in home region - if self.__tenancy_password_policy: - if self.__tenancy_password_policy.password_policy.is_lowercase_characters_required: - self.cis_foundations_benchmark_2_0['1.4']['Status'] = True + if not(self.__identity_domains_enabled) and self.__tenancy_password_policy: + if self.__tenancy_password_policy.password_policy.minimum_password_length >= 14: + self.cis_foundations_benchmark_3_0['1.4']['Status'] = True + else: + self.cis_foundations_benchmark_3_0['1.4']['Status'] = False else: - self.cis_foundations_benchmark_2_0['1.4']['Status'] = None + self.cis_foundations_benchmark_3_0['1.4']['Status'] = None - # 1.5 and 1.6 Checking Identity Domains Password Policy for expiry less than 365 and + # 1.4, 1.5 and 1.6 Checking Identity Domains Password Policy for expiry less than 365 and debug("__report_cis_analyze_tenancy_data: Identity Domains Enabled is: " + str(self.__identity_domains_enabled)) if self.__identity_domains_enabled: for domain in self.__identity_domains: if domain['password_policy']: debug("Policy " + domain['display_name'] + " password expiry is " + str(domain['password_policy']['password_expires_after'])) debug("Policy " + domain['display_name'] + " reuse is " + str(domain['password_policy']['num_passwords_in_history'])) + debug("Policy " + domain['display_name'] + " length is " + str(domain['password_policy']['min_length'])) + + + if domain['password_policy']['min_length']: + if domain['password_policy']['min_length'] > 14: + self.cis_foundations_benchmark_3_0['1.4']['Findings'].append(domain) + else: + self.cis_foundations_benchmark_3_0['1.4']['Findings'].append(domain) + if domain['password_policy']['password_expires_after']: if domain['password_policy']['password_expires_after'] > 365: - self.cis_foundations_benchmark_2_0['1.5']['Findings'].append(domain) - + self.cis_foundations_benchmark_3_0['1.5']['Findings'].append(domain) + else: + self.cis_foundations_benchmark_3_0['1.5']['Findings'].append(domain) if domain['password_policy']['num_passwords_in_history']: if domain['password_policy']['num_passwords_in_history'] < 24: - self.cis_foundations_benchmark_2_0['1.6']['Findings'].append(domain) + self.cis_foundations_benchmark_3_0['1.6']['Findings'].append(domain) + else: + self.cis_foundations_benchmark_3_0['1.6']['Findings'].append(domain) else: - debug("__report_cis_analyze_tenancy_data 1.5 and 1.6 no password policy") - self.cis_foundations_benchmark_2_0['1.5']['Findings'].append(domain) - self.cis_foundations_benchmark_2_0['1.6']['Findings'].append(domain) + debug("__report_cis_analyze_tenancy_data 1.4, 1.5 and 1.6 no password policy") + self.cis_foundations_benchmark_3_0['1.4']['Findings'].append(domain) + self.cis_foundations_benchmark_3_0['1.5']['Findings'].append(domain) + self.cis_foundations_benchmark_3_0['1.6']['Findings'].append(domain) + if self.cis_foundations_benchmark_3_0['1.4']['Findings']: + self.cis_foundations_benchmark_3_0['1.4']['Status'] = False + else: + self.cis_foundations_benchmark_3_0['1.4']['Status'] = True - if self.cis_foundations_benchmark_2_0['1.5']['Findings']: - self.cis_foundations_benchmark_2_0['1.5']['Status'] = False + if self.cis_foundations_benchmark_3_0['1.5']['Findings']: + self.cis_foundations_benchmark_3_0['1.5']['Status'] = False else: - self.cis_foundations_benchmark_2_0['1.5']['Status'] = True + self.cis_foundations_benchmark_3_0['1.5']['Status'] = True - if self.cis_foundations_benchmark_2_0['1.6']['Findings']: - self.cis_foundations_benchmark_2_0['1.6']['Status'] = False + if self.cis_foundations_benchmark_3_0['1.6']['Findings']: + self.cis_foundations_benchmark_3_0['1.6']['Status'] = False else: - self.cis_foundations_benchmark_2_0['1.6']['Status'] = True + self.cis_foundations_benchmark_3_0['1.6']['Status'] = True # Adding all identity domains to Total - self.cis_foundations_benchmark_2_0['1.5']['Total'] = self.__identity_domains - self.cis_foundations_benchmark_2_0['1.6']['Total'] = self.__identity_domains + self.cis_foundations_benchmark_3_0['1.4']['Total'] = self.__identity_domains + self.cis_foundations_benchmark_3_0['1.5']['Total'] = self.__identity_domains + self.cis_foundations_benchmark_3_0['1.6']['Total'] = self.__identity_domains # 1.7 Check - Local Users w/o MFA for user in self.__users: if not(user['is_federated']) and user['can_use_console_password'] and not (user['is_mfa_activated']) and user['lifecycle_state']: - self.cis_foundations_benchmark_2_0['1.7']['Status'] = False - self.cis_foundations_benchmark_2_0['1.7']['Findings'].append( + self.cis_foundations_benchmark_3_0['1.7']['Status'] = False + self.cis_foundations_benchmark_3_0['1.7']['Findings'].append( user) # CIS Total 1.7 Adding - All Users to CIS Total - self.cis_foundations_benchmark_2_0['1.7']['Total'] = self.__users + self.cis_foundations_benchmark_3_0['1.7']['Total'] = self.__users # 1.8 Check - API Keys over 90 for user in self.__users: if user['api_keys']: for key in user['api_keys']: if self.api_key_time_max_datetime >= datetime.datetime.strptime(key['time_created'], self.__iso_time_format): - self.cis_foundations_benchmark_2_0['1.8']['Status'] = False + self.cis_foundations_benchmark_3_0['1.8']['Status'] = False finding = { "user_name": user['name'], "user_id": user['id'], @@ -3753,18 +4095,18 @@ def __report_cis_analyze_tenancy_data(self): 'time_created': key['time_created'] } - self.cis_foundations_benchmark_2_0['1.8']['Findings'].append( + self.cis_foundations_benchmark_3_0['1.8']['Findings'].append( finding) # CIS Total 1.8 Adding - Customer Secrets to CIS Total - self.cis_foundations_benchmark_2_0['1.8']['Total'].append(key) + self.cis_foundations_benchmark_3_0['1.8']['Total'].append(key) # CIS 1.9 Check - Old Customer Secrets for user in self.__users: if user['customer_secret_keys']: for key in user['customer_secret_keys']: if self.api_key_time_max_datetime >= datetime.datetime.strptime(key['time_created'], self.__iso_time_format): - self.cis_foundations_benchmark_2_0['1.9']['Status'] = False + self.cis_foundations_benchmark_3_0['1.9']['Status'] = False finding = { "user_name": user['name'], "user_id": user['id'], @@ -3776,17 +4118,17 @@ def __report_cis_analyze_tenancy_data(self): 'time_expires': key['time_expires'] } - self.cis_foundations_benchmark_2_0['1.9']['Findings'].append(finding) + self.cis_foundations_benchmark_3_0['1.9']['Findings'].append(finding) # CIS Total 1.9 Adding - Customer Secrets to CIS Total - self.cis_foundations_benchmark_2_0['1.9']['Total'].append(key) + self.cis_foundations_benchmark_3_0['1.9']['Total'].append(key) # CIS 1.10 Check - Old Auth Tokens for user in self.__users: if user['auth_tokens']: for key in user['auth_tokens']: if self.api_key_time_max_datetime >= datetime.datetime.strptime(key['time_created'], self.__iso_time_format): # and key['lifecycle_state'] == 'ACTIVE': - self.cis_foundations_benchmark_2_0['1.10']['Status'] = False + self.cis_foundations_benchmark_3_0['1.10']['Status'] = False finding = { "user_name": user['name'], @@ -3800,50 +4142,71 @@ def __report_cis_analyze_tenancy_data(self): # "token": key['token'] } - self.cis_foundations_benchmark_2_0['1.10']['Findings'].append( + self.cis_foundations_benchmark_3_0['1.10']['Findings'].append( finding) # CIS Total 1.10 Adding - Keys to CIS Total - self.cis_foundations_benchmark_2_0['1.10']['Total'].append( + self.cis_foundations_benchmark_3_0['1.10']['Total'].append( key) + # CIS 1.11 Check - Old DB Password + #__iso_time_format1 = "%Y-%m-%dT%H:%M:%S.%fZ" + for user in self.__users: + if user['database_passwords']: + for key in user['database_passwords']: + if self.api_key_time_max_datetime >= datetime.datetime.strptime(key['time_created'], self.__iso_time_format): + self.cis_foundations_benchmark_3_0['1.11']['Status'] = False + + finding = { + "user_name": user['name'], + "user_id": user['id'], + "id": key['ocid'], + "description": key['description'], + # "expires-on": key['expires_on'] + } + + self.cis_foundations_benchmark_3_0['1.11']['Findings'].append(finding) + + # CIS Total 1.11 Adding - Keys to CIS Total + self.cis_foundations_benchmark_3_0['1.11']['Total'].append(key) + # CIS 1.12 Active Admins with API keys # Iterating through all users to see if they have API Keys and if they are active users for user in self.__users: if 'Administrators' in user['groups'] and user['api_keys'] and user['lifecycle_state']: - self.cis_foundations_benchmark_2_0['1.12']['Status'] = False - self.cis_foundations_benchmark_2_0['1.12']['Findings'].append( + self.cis_foundations_benchmark_3_0['1.12']['Status'] = False + self.cis_foundations_benchmark_3_0['1.12']['Findings'].append( user) # CIS Total 1.12 Adding - All IAM Users in Administrator group to CIS Total if 'Administrators' in user['groups'] and user['lifecycle_state']: - self.cis_foundations_benchmark_2_0['1.12']['Total'].append(user) + self.cis_foundations_benchmark_3_0['1.12']['Total'].append(user) # CIS 1.13 Check - This check is complete uses email verification # Iterating through all users to see if they have API Keys and if they are active users for user in self.__users: if user['external_identifier'] is None and user['lifecycle_state'] and not (user['email_verified']): - self.cis_foundations_benchmark_2_0['1.13']['Status'] = False - self.cis_foundations_benchmark_2_0['1.13']['Findings'].append( + self.cis_foundations_benchmark_3_0['1.13']['Status'] = False + self.cis_foundations_benchmark_3_0['1.13']['Findings'].append( user) # CIS Total 1.13 Adding - All IAM Users for to CIS Total - self.cis_foundations_benchmark_2_0['1.13']['Total'] = self.__users + self.cis_foundations_benchmark_3_0['1.13']['Total'] = self.__users # CIS 1.14 Check - Ensure Dynamic Groups are used for OCI instances, OCI Cloud Databases and OCI Function to access OCI resources # Iterating through all dynamic groups ensure there are some for fnfunc, instance or autonomous. Using reverse logic so starts as a false for dynamic_group in self.__dynamic_groups: if any(oci_resource.upper() in str(dynamic_group['matching_rule'].upper()) for oci_resource in self.cis_iam_checks['1.14']['resources']): - self.cis_foundations_benchmark_2_0['1.14']['Status'] = True + self.cis_foundations_benchmark_3_0['1.14']['Status'] = True else: - self.cis_foundations_benchmark_2_0['1.14']['Findings'].append( + self.cis_foundations_benchmark_3_0['1.14']['Findings'].append( dynamic_group) # Clearing finding - if self.cis_foundations_benchmark_2_0['1.14']['Status']: - self.cis_foundations_benchmark_2_0['1.14']['Findings'] = [] + if self.cis_foundations_benchmark_3_0['1.14']['Status']: + self.cis_foundations_benchmark_3_0['1.14']['Findings'] = [] # CIS Total 1.14 Adding - All Dynamic Groups for to CIS Total - self.cis_foundations_benchmark_2_0['1.14']['Total'] = self.__dynamic_groups + self.cis_foundations_benchmark_3_0['1.14']['Total'] = self.__dynamic_groups # CIS 1.15 Check - Ensure storage service-level admins cannot delete resources they manage. # Iterating through all policies @@ -3857,27 +4220,92 @@ def __report_cis_analyze_tenancy_data(self): clean_where_clause = split_statement[1].upper().replace(" ", "").replace("'", "") if all(permission.upper() in clean_where_clause for permission in self.cis_iam_checks['1.15'][resource]) and \ not(all(permission.upper() in clean_where_clause for permission in self.cis_iam_checks['1.15-storage-admin'][resource])): - debug("__report_cis_analyze_tenancy_data no permissions to delete storage: " + str(policy['name'])) + debug("__report_cis_analyze_tenancy_data CIS 1.15 no permissions to delete storage: " + str(policy['name'])) pass # Checking if this is the Storage admin with allowed elif all(permission.upper() in clean_where_clause for permission in self.cis_iam_checks['1.15-storage-admin'][resource]) and \ not(all(permission.upper() in clean_where_clause for permission in self.cis_iam_checks['1.15'][resource])): - debug("__report_cis_analyze_tenancy_data storage admin policy is: " + str(policy['name'])) + debug("__report_cis_analyze_tenancy_data CIS 1.15 storage admin policy is: " + str(policy['name'])) pass else: - self.cis_foundations_benchmark_2_0['1.15']['Findings'].append(policy) - debug("__report_cis_analyze_tenancy_data else policy is\n: " + str(policy['name'])) + self.cis_foundations_benchmark_3_0['1.15']['Findings'].append(policy) + debug("__report_cis_analyze_tenancy_data CIS 1.15 else policy is\n: " + str(policy['name'])) else: - self.cis_foundations_benchmark_2_0['1.15']['Findings'].append(policy) + self.cis_foundations_benchmark_3_0['1.15']['Findings'].append(policy) + + if self.cis_foundations_benchmark_3_0['1.15']['Findings']: + self.cis_foundations_benchmark_3_0['1.15']['Status'] = False + else: + self.cis_foundations_benchmark_3_0['1.15']['Status'] = True + + # CIS Total 1.15 Adding - All IAM Policies for to CIS Total + self.cis_foundations_benchmark_3_0['1.15']['Total'] = self.__policies + + # CIS 1.16 Check - Users with API Keys over 45 days + + for user in self.__users: + + login_over_45_days = None + api_key_over_45_days = None + if user['lifecycle_state']: # and not(user['is_federated']) and user['can_use_console_password']: + debug(f'__report_cis_analyze_tenancy_data CIS 1.16 Login Over 45 days is: {login_over_45_days}') + if user['last_successful_login_date']: + last_successful_login_date = user['last_successful_login_date'].split(".")[0] + if self.local_user_time_max_datetime > datetime.datetime.strptime(last_successful_login_date, self.__iso_time_format): + login_over_45_days = True + debug(f"__report_cis_analyze_tenancy_data CIS 1.16 Last login is {user['last_successful_login_date']} and max login is {self.local_user_time_max_datetime}") + else: + login_over_45_days = False + else: + debug("__report_cis_analyze_tenancy_data CIS 1.16 No Last login") + login_over_45_days = True - if self.cis_foundations_benchmark_2_0['1.15']['Findings']: - self.cis_foundations_benchmark_2_0['1.15']['Status'] = False + else: + debug("__report_cis_analyze_tenancy_data CIS 1.16 INACTIVE USE") + login_over_45_days = False + + if user['api_keys']: + debug("__report_cis_analyze_tenancy_data CIS 1.16 API Key Check") + for api_key in user['api_keys']: + if api_key['apikey_used_in_45_days']: + api_key_over_45_days = True + else: + debug("__report_cis_analyze_tenancy_data CIS 1.16 API Key used in under 45 days") + api_key_over_45_days = True + # else: + # api_key_over_45_days = False + + debug(f"__report_cis_analyze_tenancy_data CIS 1.16 User: {user['id']}") + debug(f'__report_cis_analyze_tenancy_data CIS 1.16 Over Login Over 45: {login_over_45_days}') + debug(f'__report_cis_analyze_tenancy_data CIS 1.16 Over API Key Over 45: {api_key_over_45_days}') + if login_over_45_days or api_key_over_45_days: + finding = user.copy() + finding['login_over_45_days'] = login_over_45_days + finding['api_key_over_45_days'] = api_key_over_45_days + self.cis_foundations_benchmark_3_0['1.16']['Findings'].append(finding) + + if self.cis_foundations_benchmark_3_0['1.16']['Findings']: + self.cis_foundations_benchmark_3_0['1.16']['Status'] = False else: - self.cis_foundations_benchmark_2_0['1.15']['Status'] = True + self.cis_foundations_benchmark_3_0['1.16']['Status'] = True # CIS Total 1.15 Adding - All IAM Policies for to CIS Total - self.cis_foundations_benchmark_2_0['1.15']['Total'] = self.__policies + self.cis_foundations_benchmark_3_0['1.16']['Total'] = self.__users + + + + # CIS 1.17 Check - Ensure there is only one active API Key for any single OCI IAM user + for user in self.__users: + if user['api_keys'] and not(len(user['api_keys']) < 2): + self.cis_foundations_benchmark_3_0['1.17']['Findings'].append(user) + + if self.cis_foundations_benchmark_3_0['1.17']['Findings']: + self.cis_foundations_benchmark_3_0['1.17']['Status'] = False + else: + self.cis_foundations_benchmark_3_0['1.17']['Status'] = True + # CIS Total 1.17 Adding - All IAM Policies for to CIS Total + self.cis_foundations_benchmark_3_0['1.17']['Total'] = self.__users # CIS 2.1, 2.2, & 2.5 Check - Security List Ingress from 0.0.0.0/0 on ports 22, 3389 for sl in self.__network_security_lists: @@ -3888,40 +4316,47 @@ def __report_cis_analyze_tenancy_data(self): port_max = irule['tcp_options']['destinationPortRange']['max'] ports_range = range(port_min, port_max + 1) if 22 in ports_range: - self.cis_foundations_benchmark_2_0['2.1']['Status'] = False - self.cis_foundations_benchmark_2_0['2.1']['Findings'].append(sl) + self.cis_foundations_benchmark_3_0['2.1']['Status'] = False + self.cis_foundations_benchmark_3_0['2.1']['Findings'].append(sl) if 3389 in ports_range: - self.cis_foundations_benchmark_2_0['2.2']['Status'] = False - self.cis_foundations_benchmark_2_0['2.2']['Findings'].append(sl) + self.cis_foundations_benchmark_3_0['2.2']['Status'] = False + self.cis_foundations_benchmark_3_0['2.2']['Findings'].append(sl) break else: # If TCP Options is null it includes all ports - self.cis_foundations_benchmark_2_0['2.1']['Status'] = False - self.cis_foundations_benchmark_2_0['2.1']['Findings'].append(sl) - self.cis_foundations_benchmark_2_0['2.2']['Status'] = False - self.cis_foundations_benchmark_2_0['2.2']['Findings'].append(sl) + self.cis_foundations_benchmark_3_0['2.1']['Status'] = False + self.cis_foundations_benchmark_3_0['2.1']['Findings'].append(sl) + self.cis_foundations_benchmark_3_0['2.2']['Status'] = False + self.cis_foundations_benchmark_3_0['2.2']['Findings'].append(sl) break elif irule['source'] == "0.0.0.0/0" and irule['protocol'] == 'all': # All Protocols allowed included TCP and all ports - self.cis_foundations_benchmark_2_0['2.1']['Status'] = False - self.cis_foundations_benchmark_2_0['2.1']['Findings'].append(sl) - self.cis_foundations_benchmark_2_0['2.2']['Status'] = False - self.cis_foundations_benchmark_2_0['2.2']['Findings'].append(sl) + self.cis_foundations_benchmark_3_0['2.1']['Status'] = False + self.cis_foundations_benchmark_3_0['2.1']['Findings'].append(sl) + self.cis_foundations_benchmark_3_0['2.2']['Status'] = False + self.cis_foundations_benchmark_3_0['2.2']['Findings'].append(sl) break # CIS Total 2.1, 2.2 Adding - All SLs for to CIS Total - self.cis_foundations_benchmark_2_0['2.1']['Total'] = self.__network_security_lists - self.cis_foundations_benchmark_2_0['2.2']['Total'] = self.__network_security_lists + self.cis_foundations_benchmark_3_0['2.1']['Total'] = self.__network_security_lists + self.cis_foundations_benchmark_3_0['2.2']['Total'] = self.__network_security_lists # CIS 2.5 Check - any rule with 0.0.0.0 where protocol not 1 (ICMP) # CIS Total 2.5 Adding - All Default Security List for to CIS Total for sl in self.__network_security_lists: if sl['display_name'].startswith("Default Security List for "): - self.cis_foundations_benchmark_2_0['2.5']['Total'].append(sl) - for irule in sl['ingress_security_rules']: - if irule['source'] == "0.0.0.0/0" and irule['protocol'] != '1': - self.cis_foundations_benchmark_2_0['2.5']['Status'] = False - self.cis_foundations_benchmark_2_0['2.5']['Findings'].append( + self.cis_foundations_benchmark_3_0['2.5']['Total'].append(sl) + for irule in sl['ingress_security_rules'] + sl['egress_security_rules']: + if 'source' in irule and irule['source'] == "0.0.0.0/0": + debug("__report_cis_analyze_tenancy_data: Security List has bad ingress rule") + self.cis_foundations_benchmark_3_0['2.5']['Status'] = False + self.cis_foundations_benchmark_3_0['2.5']['Findings'].append( + sl) + break + elif 'destination' in irule and irule['destination'] == "0.0.0.0/0" and irule['protocol'] != '1': + debug("Security List has bad egress rule") + self.cis_foundations_benchmark_3_0['2.5']['Status'] = False + self.cis_foundations_benchmark_3_0['2.5']['Findings'].append( sl) break @@ -3934,62 +4369,62 @@ def __report_cis_analyze_tenancy_data(self): port_max = rule['tcp_options'].destination_port_range.max ports_range = range(port_min, port_max + 1) if 22 in ports_range: - self.cis_foundations_benchmark_2_0['2.3']['Status'] = False - self.cis_foundations_benchmark_2_0['2.3']['Findings'].append( + self.cis_foundations_benchmark_3_0['2.3']['Status'] = False + self.cis_foundations_benchmark_3_0['2.3']['Findings'].append( nsg) if 3389 in ports_range: - self.cis_foundations_benchmark_2_0['2.4']['Status'] = False - self.cis_foundations_benchmark_2_0['2.4']['Findings'].append(nsg) + self.cis_foundations_benchmark_3_0['2.4']['Status'] = False + self.cis_foundations_benchmark_3_0['2.4']['Findings'].append(nsg) break else: # If TCP Options is null it includes all ports - self.cis_foundations_benchmark_2_0['2.3']['Status'] = False - self.cis_foundations_benchmark_2_0['2.3']['Findings'].append(nsg) - self.cis_foundations_benchmark_2_0['2.4']['Status'] = False - self.cis_foundations_benchmark_2_0['2.4']['Findings'].append(nsg) + self.cis_foundations_benchmark_3_0['2.3']['Status'] = False + self.cis_foundations_benchmark_3_0['2.3']['Findings'].append(nsg) + self.cis_foundations_benchmark_3_0['2.4']['Status'] = False + self.cis_foundations_benchmark_3_0['2.4']['Findings'].append(nsg) break elif rule['source'] == "0.0.0.0/0" and rule['protocol'] == 'all': # All Protocols allowed included TCP and all ports - self.cis_foundations_benchmark_2_0['2.3']['Status'] = False - self.cis_foundations_benchmark_2_0['2.3']['Findings'].append(nsg) - self.cis_foundations_benchmark_2_0['2.4']['Status'] = False - self.cis_foundations_benchmark_2_0['2.4']['Findings'].append(nsg) + self.cis_foundations_benchmark_3_0['2.3']['Status'] = False + self.cis_foundations_benchmark_3_0['2.3']['Findings'].append(nsg) + self.cis_foundations_benchmark_3_0['2.4']['Status'] = False + self.cis_foundations_benchmark_3_0['2.4']['Findings'].append(nsg) break # CIS Total 2.2 & 2.4 Adding - All NSGs Instances to CIS Total - self.cis_foundations_benchmark_2_0['2.3']['Total'] = self.__network_security_groups - self.cis_foundations_benchmark_2_0['2.4']['Total'] = self.__network_security_groups + self.cis_foundations_benchmark_3_0['2.3']['Total'] = self.__network_security_groups + self.cis_foundations_benchmark_3_0['2.4']['Total'] = self.__network_security_groups # CIS 2.6 - Ensure Oracle Integration Cloud (OIC) access is restricted to allowed sources # Iterating through OIC instance have network access rules and ensure 0.0.0.0/0 is not in the list for integration_instance in self.__integration_instances: if not (integration_instance['network_endpoint_details']): - self.cis_foundations_benchmark_2_0['2.6']['Status'] = False - self.cis_foundations_benchmark_2_0['2.6']['Findings'].append( + self.cis_foundations_benchmark_3_0['2.6']['Status'] = False + self.cis_foundations_benchmark_3_0['2.6']['Findings'].append( integration_instance) elif integration_instance['network_endpoint_details']: if "0.0.0.0/0" in str(integration_instance['network_endpoint_details']): - self.cis_foundations_benchmark_2_0['2.6']['Status'] = False - self.cis_foundations_benchmark_2_0['2.6']['Findings'].append( + self.cis_foundations_benchmark_3_0['2.6']['Status'] = False + self.cis_foundations_benchmark_3_0['2.6']['Findings'].append( integration_instance) # CIS Total 2.6 Adding - All OIC Instances to CIS Total - self.cis_foundations_benchmark_2_0['2.6']['Total'] = self.__integration_instances + self.cis_foundations_benchmark_3_0['2.6']['Total'] = self.__integration_instances # CIS 2.7 - Ensure Oracle Analytics Cloud (OAC) access is restricted to allowed sources or deployed within a VCN for analytics_instance in self.__analytics_instances: if analytics_instance['network_endpoint_type'].upper() == 'PUBLIC': if not (analytics_instance['network_endpoint_details'].whitelisted_ips): - self.cis_foundations_benchmark_2_0['2.7']['Status'] = False - self.cis_foundations_benchmark_2_0['2.7']['Findings'].append(analytics_instance) + self.cis_foundations_benchmark_3_0['2.7']['Status'] = False + self.cis_foundations_benchmark_3_0['2.7']['Findings'].append(analytics_instance) elif "0.0.0.0/0" in analytics_instance['network_endpoint_details'].whitelisted_ips: - self.cis_foundations_benchmark_2_0['2.7']['Status'] = False - self.cis_foundations_benchmark_2_0['2.7']['Findings'].append( + self.cis_foundations_benchmark_3_0['2.7']['Status'] = False + self.cis_foundations_benchmark_3_0['2.7']['Findings'].append( analytics_instance) # CIS Total 2.7 Adding - All OAC Instances to CIS Total - self.cis_foundations_benchmark_2_0['2.7']['Total'] = self.__analytics_instances + self.cis_foundations_benchmark_3_0['2.7']['Total'] = self.__analytics_instances # CIS 2.8 Check - Ensure Oracle Autonomous Shared Databases (ADB) access is restricted to allowed sources or deployed within a VCN # Iterating through ADB Checking for null NSGs, whitelisted ip or allowed IPs 0.0.0.0/0 @@ -3997,120 +4432,191 @@ def __report_cis_analyze_tenancy_data(self): for autonomous_database in self.__autonomous_databases: if autonomous_database['lifecycle_state'] not in [ oci.database.models.AutonomousDatabaseSummary.LIFECYCLE_STATE_TERMINATED, oci.database.models.AutonomousDatabaseSummary.LIFECYCLE_STATE_TERMINATING, oci.database.models.AutonomousDatabaseSummary.LIFECYCLE_STATE_UNAVAILABLE ]: if not (autonomous_database['whitelisted_ips']) and not (autonomous_database['subnet_id']): - self.cis_foundations_benchmark_2_0['2.8']['Status'] = False - self.cis_foundations_benchmark_2_0['2.8']['Findings'].append( + self.cis_foundations_benchmark_3_0['2.8']['Status'] = False + self.cis_foundations_benchmark_3_0['2.8']['Findings'].append( autonomous_database) elif autonomous_database['whitelisted_ips']: for value in autonomous_database['whitelisted_ips']: if '0.0.0.0/0' in str(autonomous_database['whitelisted_ips']): - self.cis_foundations_benchmark_2_0['2.8']['Status'] = False - self.cis_foundations_benchmark_2_0['2.8']['Findings'].append( + self.cis_foundations_benchmark_3_0['2.8']['Status'] = False + self.cis_foundations_benchmark_3_0['2.8']['Findings'].append( autonomous_database) # CIS Total 2.8 Adding - All ADBs to CIS Total - self.cis_foundations_benchmark_2_0['2.8']['Total'] = self.__autonomous_databases + self.cis_foundations_benchmark_3_0['2.8']['Total'] = self.__autonomous_databases # From CIS 2.0 CIS 4.1 Check - Ensure Audit log retention == 365 - Only checking in home region # if self.__audit_retention_period >= 365: - # self.cis_foundations_benchmark_2_0['4.1']['Status'] = True + # self.cis_foundations_benchmark_3_0['4.1']['Status'] = True for instance in self.__Instance: # CIS Check 3.1 Metadata Service v2 Enabled if instance['instance_options'] is None or not(instance['instance_options']['are_legacy_imds_endpoints_disabled']): debug(f"__report_cis_analyze_tenancy_data {instance['display_name']} doesn't disable IMDSv1") - self.cis_foundations_benchmark_2_0['3.1']['Status'] = False - self.cis_foundations_benchmark_2_0['3.1']['Findings'].append(instance) + self.cis_foundations_benchmark_3_0['3.1']['Status'] = False + self.cis_foundations_benchmark_3_0['3.1']['Findings'].append(instance) # CIS Check 3.2 Secure Boot enabled if instance['platform_config'] is None or not(instance['platform_config']['is_secure_boot_enabled']): debug(f"__report_cis_analyze_tenancy_data {instance['display_name']} doesn't enable secure boot") - self.cis_foundations_benchmark_2_0['3.2']['Status'] = False - self.cis_foundations_benchmark_2_0['3.2']['Findings'].append(instance) + self.cis_foundations_benchmark_3_0['3.2']['Status'] = False + self.cis_foundations_benchmark_3_0['3.2']['Findings'].append(instance) # CIS Check 3.3 Encryption in Transit enabled if instance['launch_options'] is None or not(instance['launch_options']['is_pv_encryption_in_transit_enabled']): debug(f"__report_cis_analyze_tenancy_data {instance['display_name']} doesn't enable encryption in transit") - self.cis_foundations_benchmark_2_0['3.3']['Status'] = False - self.cis_foundations_benchmark_2_0['3.3']['Findings'].append(instance) + self.cis_foundations_benchmark_3_0['3.3']['Status'] = False + self.cis_foundations_benchmark_3_0['3.3']['Findings'].append(instance) # CIS Total 3.1 Adding - All Instances to CIS Total - self.cis_foundations_benchmark_2_0['3.1']['Total'] = self.__Instance + self.cis_foundations_benchmark_3_0['3.1']['Total'] = self.__Instance # CIS Total 3.2 Adding - All Instances to CIS Total - self.cis_foundations_benchmark_2_0['3.2']['Total'] = self.__Instance + self.cis_foundations_benchmark_3_0['3.2']['Total'] = self.__Instance # CIS Total 3.3 Adding - All Instances to CIS Total - self.cis_foundations_benchmark_2_0['3.3']['Total'] = self.__Instance + self.cis_foundations_benchmark_3_0['3.3']['Total'] = self.__Instance # CIS Check 4.1 - Check for Default Tags in Root Compartment # Iterate through tags looking for ${iam.principal.name} for tag in self.__tag_defaults: if tag['value'] == "${iam.principal.name}": - self.cis_foundations_benchmark_2_0['4.1']['Status'] = True + self.cis_foundations_benchmark_3_0['4.1']['Status'] = True # CIS Total 4.1 Adding - All Tag Defaults to CIS Total - self.cis_foundations_benchmark_2_0['4.1']['Total'] = self.__tag_defaults + self.cis_foundations_benchmark_3_0['4.1']['Total'] = self.__tag_defaults # CIS Check 4.2 - Check for Active Notification and Subscription - if len(self.__subscriptions) > 0: - self.cis_foundations_benchmark_2_0['4.2']['Status'] = True + for sub in self.__subscriptions: + if sub['lifecycle_state'] == 'ACTIVE': + self.cis_foundations_benchmark_3_0['4.2']['Status'] = True + else: + self.cis_foundations_benchmark_3_0['4.2']['Findings'].append(sub) + + # CIS Check 4.2 Total - All Subscriptions to CIS Total - self.cis_foundations_benchmark_2_0['4.2']['Total'] = self.__subscriptions + self.cis_foundations_benchmark_3_0['4.2']['Total'] = self.__subscriptions - # CIS Checks 4.3 - 4.12 and 4.15 + # CIS Checks 4.3 - 4.12 and 4.15 and 4.18 # Iterate through all event rules for event in self.__event_rules: - # Convert Event Condition to dict - eventtype_jsonable_str = event['condition'].lower().replace("'", "\"") - try: - eventtype_dict = json.loads(eventtype_jsonable_str) - except Exception: - print("*** Invalid Event Condition for event (not in JSON format): " + event['display_name'] + " ***") - eventtype_dict = {} - # Issue 256: 'eventtype' not in eventtype_dict (i.e. missing in event condition) - if eventtype_dict and 'eventtype' in eventtype_dict: - for key, changes in self.cis_monitoring_checks.items(): - # Checking if all cis change list is a subset of event condition - try: - # Checking if each region has the required events - if (all(x in eventtype_dict['eventtype'] for x in changes)) and key in self.__cis_regional_checks: - self.__cis_regional_findings_data[key][event['region']] = True - - # Cloud Guard Check is only required in the Cloud Guard Reporting Region - elif key == "4.15" and event['region'] == self.__cloud_guard_config.reporting_region and \ - (all(x in eventtype_dict['eventtype'] for x in changes)): - self.cis_foundations_benchmark_2_0[key]['Status'] = True - - # For Checks that are home region based checking those - elif (all(x in eventtype_dict['eventtype'] for x in changes)) and \ - key not in self.__cis_regional_checks and event['region'] == self.__home_region: - self.cis_foundations_benchmark_2_0[key]['Status'] = True + if event['lifecycle_state'] == "ACTIVE": + # Convert Event Condition to dict + eventtype_jsonable_str = event['condition'].lower().replace("'", "\"") + try: + eventtype_dict = json.loads(eventtype_jsonable_str) + except Exception: + print("*** Invalid Event Condition for event (not in JSON format): " + event['display_name'] + " ***") + eventtype_dict = {} - except Exception: - print("*** Invalid Event Data for event: " + event['display_name'] + " ***") + # Issue 256: 'eventtype' not in eventtype_dict (i.e. missing in event condition) + if eventtype_dict and 'eventtype' in eventtype_dict: + for key, changes in self.cis_monitoring_checks.items(): + # Checking if all cis change list is a subset of event condition + try: + # Checking if each region has the required events + if (all(x in eventtype_dict['eventtype'] for x in changes)) and key in self.__cis_regional_checks: + self.__cis_regional_findings_data[key][event['region']] = True + + # Cloud Guard Check is only required in the Cloud Guard Reporting Region + elif self.__cloud_guard_config and key == "4.15" and \ + event['region'] == self.__cloud_guard_config.reporting_region and \ + (all(x in eventtype_dict['eventtype'] for x in changes)): + self.cis_foundations_benchmark_3_0[key]['Status'] = True + + # For Checks that are home region based checking those + elif (all(x in eventtype_dict['eventtype'] for x in changes)) and \ + key not in self.__cis_regional_checks and event['region'] == self.__home_region: + self.cis_foundations_benchmark_3_0[key]['Status'] = True + + except Exception as e: + print(e) + print("*** Invalid Event Data for event: " + event['display_name'] + " ***") # ******* Iterating through Regional Checks adding findings for key, findings in self.__cis_regional_findings_data.items(): if all(findings.values()): - self.cis_foundations_benchmark_2_0[key]['Status'] = True + self.cis_foundations_benchmark_3_0[key]['Status'] = True + ### Testing ### # CIS Check 4.13 - VCN FlowLog enable # Generate list of subnets IDs for subnet in self.__network_subnets: - if not (subnet['id'] in self.__subnet_logs): - self.cis_foundations_benchmark_2_0['4.13']['Status'] = False - self.cis_foundations_benchmark_2_0['4.13']['Findings'].append( - subnet) + vcn_id = subnet['vcn_id'] + try: + if self.__all_logs and 'flowlogs' in self.__all_logs and \ + 'vcn' in self.__all_logs['flowlogs'] and vcn_id in self.__all_logs['flowlogs']['vcn']: + + debug(f"__report_cis_analyze_tenancy_data: Flowlogs checking VCN {vcn_id} for Subnet: {subnet['id']} ") + if self.__all_logs['flowlogs']['vcn'][vcn_id]['capture_filter']: + capture_filter_id = self.__all_logs['flowlogs']['vcn'][vcn_id]['capture_filter'] + capture_filter = self.__network_capturefilters[capture_filter_id] + + if not(self.all_traffic_rules[0] in capture_filter['additional_details']['flowLogCaptureFilterRules'] or \ + self.all_traffic_rules[1] in capture_filter['additional_details']['flowLogCaptureFilterRules']): + # VCN is being logging but it is has a capture filter we need to check + debug(f"__report_cis_analyze_tenancy_data: Flowlogs Capture Filter {capture_filter_id} Rules not compliant.") + capture_filter = self.__network_capturefilters[capture_filter_id] + self.cis_foundations_benchmark_3_0['4.13']['Status'] = False + self.cis_foundations_benchmark_3_0['4.13']['Findings'].append(subnet) + + elif self.__all_logs and 'flowlogs' in self.__all_logs and \ + 'subnet' in self.__all_logs['flowlogs'] and subnet['id'] in self.__all_logs['flowlogs']['subnet']: + + debug(f"__report_cis_analyze_tenancy_data: Flowlogs checking Subnet {subnet['id']} in subnet") + debug(self.__all_logs['flowlogs']['subnet'][subnet['id']]['capture_filter']) + if self.__all_logs['flowlogs']['subnet'][subnet['id']]['capture_filter']: + debug(f"__report_cis_analyze_tenancy_data: Flowlogs checking Subnet {subnet['id']} capture filter in subnet") + capture_filter_id = self.__all_logs['flowlogs']['subnet'][subnet['id']]['capture_filter'] + capture_filter = self.__network_capturefilters[capture_filter_id] + if not(self.all_traffic_rules[0] in capture_filter['additional_details']['flowLogCaptureFilterRules'] or \ + self.all_traffic_rules[1] in capture_filter['additional_details']['flowLogCaptureFilterRules']): + # VCN is being logging but it is has a capture filter we need to check + debug(f"__report_cis_analyze_tenancy_data: Flowlogs Capture Filter {capture_filter_id} Rules not compliant.") + self.cis_foundations_benchmark_3_0['4.13']['Status'] = False + self.cis_foundations_benchmark_3_0['4.13']['Findings'].append(subnet) + + elif self.__all_logs and self.__all_logs['flowlogs'] and \ + 'all' in self.__all_logs['flowlogs'] and subnet['id'] in self.__all_logs['flowlogs']['all']: + + debug(f"__report_cis_analyze_tenancy_data: Flowlogs checking Subnet {subnet['id']} in all") + debug(self.__all_logs['flowlogs']['all'][subnet['id']]['capture_filter']) + if self.__all_logs['flowlogs']['all'][subnet['id']]['capture_filter']: + debug(f"__report_cis_analyze_tenancy_data: Flowlogs checking Subnet {subnet['id']} capture filter in all") + + capture_filter_id = self.__all_logs['flowlogs']['all'][subnet['id']]['capture_filter'] + capture_filter = self.__network_capturefilters[capture_filter_id] + if not(self.all_traffic_rules[0] in capture_filter['additional_details']['flowLogCaptureFilterRules'] or \ + self.all_traffic_rules[1] in capture_filter['additional_details']['flowLogCaptureFilterRules']): + # VCN is being logging but it is has a capture filter we need to check + debug(f"__report_cis_analyze_tenancy_data: Flowlogs Capture Filter {capture_filter_id} Rules not compliant.") + self.cis_foundations_benchmark_3_0['4.13']['Status'] = False + self.cis_foundations_benchmark_3_0['4.13']['Findings'].append(subnet) + + else: + debug(f"__report_cis_analyze_tenancy_data: Flowlogs count not find Subnet {subnet['id']}, it is a finding") + self.cis_foundations_benchmark_3_0['4.13']['Status'] = False + self.cis_foundations_benchmark_3_0['4.13']['Findings'].append(subnet) + except Exception as e: + self.cis_foundations_benchmark_3_0['4.13']['Status'] = False + if ".capturefilter." in str(e): + print(f"Unable to read capturefilter rules for: {str(e)}.\n*** Please ensure your auditor has permissions: 'to read capture-filters in tenancy. ***") + self.__errors.append({"id" : str(e), "error" : "Unable to read capturefilter rules *** Please ensure your auditor has permissions: 'to read capture-filters in tenancy'."}) + else: + msg = f'Unable to process all logs and capture filter rules: {str(e)}' + print(msg) + self.__errors.append({"id": "__network_subnet_logs", "error": msg}) + # CIS Check 4.13 Total - Adding All Subnets to total - self.cis_foundations_benchmark_2_0['4.13']['Total'] = self.__network_subnets + self.cis_foundations_benchmark_3_0['4.13']['Total'] = self.__network_subnets # CIS Check 4.14 - Cloud Guard enabled debug("__report_cis_analyze_tenancy_data Cloud Guard Check: " + str(self.__cloud_guard_config_status)) if self.__cloud_guard_config_status == 'ENABLED': - self.cis_foundations_benchmark_2_0['4.14']['Status'] = True + self.cis_foundations_benchmark_3_0['4.14']['Status'] = True else: - self.cis_foundations_benchmark_2_0['4.14']['Status'] = False + self.cis_foundations_benchmark_3_0['4.14']['Status'] = False # CIS Check 4.16 - Encryption keys over 365 # Generating list of keys @@ -4118,103 +4624,109 @@ def __report_cis_analyze_tenancy_data(self): try: if self.kms_key_time_max_datetime and self.kms_key_time_max_datetime >= datetime.datetime.strptime(key['currentKeyVersion_time_created'], self.__iso_time_format): - self.cis_foundations_benchmark_2_0['4.16']['Status'] = False - self.cis_foundations_benchmark_2_0['4.16']['Findings'].append( + self.cis_foundations_benchmark_3_0['4.16']['Status'] = False + self.cis_foundations_benchmark_3_0['4.16']['Findings'].append( key) if self.kms_key_time_max_datetime is None: - self.cis_foundations_benchmark_2_0['4.16']['Status'] = False - self.cis_foundations_benchmark_2_0['4.16']['Findings'].append( + self.cis_foundations_benchmark_3_0['4.16']['Status'] = False + self.cis_foundations_benchmark_3_0['4.16']['Findings'].append( key) except Exception: - self.cis_foundations_benchmark_2_0['4.16']['Status'] = False - self.cis_foundations_benchmark_2_0['4.16']['Findings'].append( + self.cis_foundations_benchmark_3_0['4.16']['Status'] = False + self.cis_foundations_benchmark_3_0['4.16']['Findings'].append( key) # CIS Check 4.16 Total - Adding Key to total - self.cis_foundations_benchmark_2_0['4.16']['Total'].append(key) + self.cis_foundations_benchmark_3_0['4.16']['Total'].append(key) # CIS Check 4.17 - Object Storage with Logs - # Generating list of buckets names - for bucket in self.__buckets: - if not (bucket['name'] in self.__write_bucket_logs): - self.cis_foundations_benchmark_2_0['4.17']['Status'] = False - self.cis_foundations_benchmark_2_0['4.17']['Findings'].append( - bucket) + # Generating list of buckets names and need to make sure they have write level bucekt logs + if self.__all_logs and 'objectstorage' in self.__all_logs and\ + 'write' in self.__all_logs['objectstorage']: + + for bucket in self.__buckets: + if not (bucket['name'] + "-" + bucket['region'] in self.__all_logs['objectstorage']['write']): + self.cis_foundations_benchmark_3_0['4.17']['Status'] = False + self.cis_foundations_benchmark_3_0['4.17']['Findings'].append( + bucket) + else: + self.cis_foundations_benchmark_3_0['4.17']['Status'] = False + self.cis_foundations_benchmark_3_0['4.17']['Findings'] +=self.__buckets # CIS Check 4.17 Total - Adding All Buckets to total - self.cis_foundations_benchmark_2_0['4.17']['Total'] = self.__buckets + self.cis_foundations_benchmark_3_0['4.17']['Total'] = self.__buckets # CIS Section 5.1 Bucket Checks # Generating list of buckets names for bucket in self.__buckets: if 'public_access_type' in bucket: if bucket['public_access_type'] != 'NoPublicAccess': - self.cis_foundations_benchmark_2_0['5.1.1']['Status'] = False - self.cis_foundations_benchmark_2_0['5.1.1']['Findings'].append( + self.cis_foundations_benchmark_3_0['5.1.1']['Status'] = False + self.cis_foundations_benchmark_3_0['5.1.1']['Findings'].append( bucket) if 'kms_key_id' in bucket: if not (bucket['kms_key_id']): - self.cis_foundations_benchmark_2_0['5.1.2']['Findings'].append( + self.cis_foundations_benchmark_3_0['5.1.2']['Findings'].append( bucket) - self.cis_foundations_benchmark_2_0['5.1.2']['Status'] = False + self.cis_foundations_benchmark_3_0['5.1.2']['Status'] = False if 'versioning' in bucket: if bucket['versioning'] != "Enabled": - self.cis_foundations_benchmark_2_0['5.1.3']['Findings'].append( + self.cis_foundations_benchmark_3_0['5.1.3']['Findings'].append( bucket) - self.cis_foundations_benchmark_2_0['5.1.3']['Status'] = False + self.cis_foundations_benchmark_3_0['5.1.3']['Status'] = False # CIS Check 5.1.1,5.1.2,5.1.3 Total - Adding All Buckets to total - self.cis_foundations_benchmark_2_0['5.1.1']['Total'] = self.__buckets - self.cis_foundations_benchmark_2_0['5.1.2']['Total'] = self.__buckets - self.cis_foundations_benchmark_2_0['5.1.3']['Total'] = self.__buckets + self.cis_foundations_benchmark_3_0['5.1.1']['Total'] = self.__buckets + self.cis_foundations_benchmark_3_0['5.1.2']['Total'] = self.__buckets + self.cis_foundations_benchmark_3_0['5.1.3']['Total'] = self.__buckets # CIS Section 5.2.1 Block Volume Checks # Generating list of block volumes names for volume in self.__block_volumes: if 'kms_key_id' in volume: if not (volume['kms_key_id']): - self.cis_foundations_benchmark_2_0['5.2.1']['Findings'].append( + self.cis_foundations_benchmark_3_0['5.2.1']['Findings'].append( volume) - self.cis_foundations_benchmark_2_0['5.2.1']['Status'] = False + self.cis_foundations_benchmark_3_0['5.2.1']['Status'] = False # CIS Check 5.2.1 Total - Adding All Block Volumes to total - self.cis_foundations_benchmark_2_0['5.2.1']['Total'] = self.__block_volumes + self.cis_foundations_benchmark_3_0['5.2.1']['Total'] = self.__block_volumes # CIS Section 5.2.2 Boot Volume Checks # Generating list of boot names for boot_volume in self.__boot_volumes: if 'kms_key_id' in boot_volume: if not (boot_volume['kms_key_id']): - self.cis_foundations_benchmark_2_0['5.2.2']['Findings'].append( + self.cis_foundations_benchmark_3_0['5.2.2']['Findings'].append( boot_volume) - self.cis_foundations_benchmark_2_0['5.2.2']['Status'] = False + self.cis_foundations_benchmark_3_0['5.2.2']['Status'] = False # CIS Check 4.2.2 Total - Adding All Block Volumes to total - self.cis_foundations_benchmark_2_0['5.2.2']['Total'] = self.__boot_volumes + self.cis_foundations_benchmark_3_0['5.2.2']['Total'] = self.__boot_volumes # CIS Section 5.3.1 FSS Checks # Generating list of FSS names for file_system in self.__file_storage_system: if 'kms_key_id' in file_system: if not (file_system['kms_key_id']): - self.cis_foundations_benchmark_2_0['5.3.1']['Findings'].append( + self.cis_foundations_benchmark_3_0['5.3.1']['Findings'].append( file_system) - self.cis_foundations_benchmark_2_0['5.3.1']['Status'] = False + self.cis_foundations_benchmark_3_0['5.3.1']['Status'] = False # CIS Check 4.3.1 Total - Adding All Block Volumes to total - self.cis_foundations_benchmark_2_0['5.3.1']['Total'] = self.__file_storage_system + self.cis_foundations_benchmark_3_0['5.3.1']['Total'] = self.__file_storage_system # CIS Section 6 Checks # Checking if more than one compartment because of the ManagedPaaS Compartment if len(self.__compartments) < 2: - self.cis_foundations_benchmark_2_0['6.1']['Status'] = False + self.cis_foundations_benchmark_3_0['6.1']['Status'] = False if len(self.__resources_in_root_compartment) > 0: for item in self.__resources_in_root_compartment: - self.cis_foundations_benchmark_2_0['6.2']['Status'] = False - self.cis_foundations_benchmark_2_0['6.2']['Findings'].append( + self.cis_foundations_benchmark_3_0['6.2']['Status'] = False + self.cis_foundations_benchmark_3_0['6.2']['Findings'].append( item) ########################################################################## @@ -4234,24 +4746,10 @@ def __get_children(self, parent, compartments): return kids ########################################################################## - # Analyzes Tenancy Data for Oracle Best Practices Report + # Initializes OBP Checks ########################################################################## - def __obp_analyze_tenancy_data(self): - - ####################################### - # Budget Checks - ####################################### - # Determines if a Budget Exists with an alert rule - if len(self.__budgets) > 0: - for budget in self.__budgets: - if budget['alert_rule_count'] > 0 and budget['target_compartment_id'] == self.__tenancy.id: - self.obp_foundations_checks['Cost_Tracking_Budgets']['Status'] = True - self.obp_foundations_checks['Cost_Tracking_Budgets']['OBP'].append(budget) - else: - self.obp_foundations_checks['Cost_Tracking_Budgets']['Findings'].append(budget) - - # Stores Regional Checks - for region_key, region_values in self.__regions.items(): + def __obp_init_regional_checks(self): + for region_key in self.__regions.keys(): self.__obp_regional_checks[region_key] = { "Audit": { "tenancy_level_audit": False, @@ -4277,16 +4775,21 @@ def __obp_analyze_tenancy_data(self): "status": False }, } - - ####################################### - # OCI Audit Log Compartments Checks - ####################################### - list_of_all_compartments = [] - dict_of_compartments = {} - for compartment in self.__compartments: - list_of_all_compartments.append(compartment.id) - - # Building a Hash Table of Parent Child Hieracrchy for Audit + + ########################################################################## + # OBP Budgets Check + ########################################################################## + def __obp_check_budget(self): + if len(self.__budgets) > 0: + for budget in self.__budgets: + if budget['alert_rule_count'] > 0 and budget['target_compartment_id'] == self.__tenancy.id: + self.obp_foundations_checks['Cost_Tracking_Budgets']['Status'] = True + self.obp_foundations_checks['Cost_Tracking_Budgets']['OBP'].append(budget) + else: + self.obp_foundations_checks['Cost_Tracking_Budgets']['Findings'].append(budget) + + def __obp_check_audit_log_compartments(self): + # Building a Hash Table of Parent Child Hierarchy for Audit dict_of_compartments = {} for compartment in self.__compartments: if "tenancy" not in compartment.id: @@ -4295,11 +4798,8 @@ def __obp_analyze_tenancy_data(self): except Exception: dict_of_compartments[compartment.compartment_id] = [] dict_of_compartments[compartment.compartment_id].append(compartment.id) - - # This is used for comparing compartments that are audit to the full list of compartments - set_of_all_compartments = set(list_of_all_compartments) - - # Collecting Servie Connectors Logs related to compartments + + # Collecting Service Connectors Logs related to compartments for sch_id, sch_values in self.__service_connectors.items(): # Only Active SCH with a target that is configured if sch_values['lifecycle_state'].upper() == "ACTIVE" and sch_values['target_kind']: @@ -4309,7 +4809,7 @@ def __obp_analyze_tenancy_data(self): if source['compartment_id'] == self.__tenancy.id and source['log_group_id'].upper() == "_Audit_Include_Subcompartment".upper(): self.__obp_regional_checks[sch_values['region']]['Audit']['tenancy_level_audit'] = True self.__obp_regional_checks[sch_values['region']]['Audit']['tenancy_level_include_sub_comps'] = True - + # Since it is not the Tenancy we should add the compartment to the list and check if sub compartment are included elif source['log_group_id'].upper() == "_Audit_Include_Subcompartment".upper(): self.__obp_regional_checks[sch_values['region']]['Audit']['compartments'] += self.__get_children(source['compartment_id'], dict_of_compartments) @@ -4318,10 +4818,13 @@ def __obp_analyze_tenancy_data(self): except Exception: # There can be empty log groups pass + # Analyzing Service Connector Audit Logs to see if each region has all compartments for region_key, region_values in self.__obp_regional_checks.items(): # Checking if I already found the tenancy ocid with all child compartments included if not region_values['Audit']['tenancy_level_audit']: + list_of_all_compartments = [compartment.id for compartment in self.__compartments] + set_of_all_compartments = set(list_of_all_compartments) audit_findings = set_of_all_compartments - set(region_values['Audit']['compartments']) # If there are items in the then it is not auditing everything in the tenancy if audit_findings: @@ -4329,20 +4832,20 @@ def __obp_analyze_tenancy_data(self): else: region_values['Audit']['tenancy_level_audit'] = True region_values['Audit']['findings'] = [] - + # Consolidating Audit findings into the OBP Checks for region_key, region_values in self.__obp_regional_checks.items(): # If this flag is set all compartments are not logged in region if not region_values['Audit']['tenancy_level_audit']: self.obp_foundations_checks['SIEM_Audit_Log_All_Comps']['Status'] = False - + # If this flag is set the region has the tenancy logging and all sub compartments flag checked if not region_values['Audit']['tenancy_level_include_sub_comps']: self.obp_foundations_checks['SIEM_Audit_Incl_Sub_Comp']['Status'] = False self.obp_foundations_checks['SIEM_Audit_Incl_Sub_Comp']['Findings'].append({"region_name": region_key}) else: self.obp_foundations_checks['SIEM_Audit_Incl_Sub_Comp']['OBP'].append({"region_name": region_key}) - + # Compartment Logs that are missed in the region for compartment in region_values['Audit']['findings']: try: @@ -4380,7 +4883,7 @@ def __obp_analyze_tenancy_data(self): exists_already = list(filter(lambda source: source['id'] == record['id'] and source['region'] == record['region'], self.obp_foundations_checks['SIEM_Audit_Log_All_Comps']['Findings'])) if not exists_already: self.obp_foundations_checks['SIEM_Audit_Log_All_Comps']['Findings'].append(record) - + # Compartment logs that are not missed in the region for compartment in region_values['Audit']['compartments']: try: @@ -4419,161 +4922,79 @@ def __obp_analyze_tenancy_data(self): if not exists_already: self.obp_foundations_checks['SIEM_Audit_Log_All_Comps']['OBP'].append(record) + def __obp_check_cloud_guard(self): ####################################### - # Subnet and Bucket Log Checks + # Cloud Guard Checks ####################################### - for sch_id, sch_values in self.__service_connectors.items(): - # Only Active SCH with a target that is configured - if sch_values['lifecycle_state'].upper() == "ACTIVE" and sch_values['target_kind']: - # Subnet Logs Checks - for subnet_id, log_values in self.__subnet_logs.items(): - - log_id = log_values['log_id'] - log_group_id = log_values['log_group_id'] - log_record = {"sch_id": sch_id, "sch_name": sch_values['display_name'], "id": subnet_id} - - subnet_log_group_in_sch = list(filter(lambda source: source['log_group_id'] == log_group_id, sch_values['log_sources'])) - subnet_log_in_sch = list(filter(lambda source: source['log_id'] == log_id, sch_values['log_sources'])) + cloud_guard_record = { + "cloud_guard_endable": True if self.__cloud_guard_config_status == 'ENABLED' else False, + "target_at_root": False, + "targert_configuration_detector": False, + "targert_configuration_detector_customer_owned": False, + "target_activity_detector": False, + "target_activity_detector_customer_owned": False, + "target_threat_detector": False, + "target_threat_detector_customer_owned": False, + "target_responder_recipes": False, + "target_responder_recipes_customer_owned": False, + "target_responder_event_rule": False, + } - # Checking if the Subnets's log group in is in SCH's log sources & the log_id is empty so it covers everything in the log group - if subnet_log_group_in_sch and not (subnet_log_in_sch): - self.__obp_regional_checks[sch_values['region']]['VCN']['subnets'].append(log_record) + try: + # Cloud Guard Target attached to the root compartment with activity, config, and threat detector plus a responder + if self.__cloud_guard_targets[self.__tenancy.id]: - # Checking if the Subnet's log id in is in the service connector's log sources if so I will add it - elif subnet_log_in_sch: - self.__obp_regional_checks[sch_values['region']]['VCN']['subnets'].append(log_record) + cloud_guard_record['target_at_root'] = True - # else: - # self.__obp_regional_checks[sch_values['region']]['VCN']['findings'].append(subnet_id) - - # Bucket Write Logs Checks - for bucket_name, log_values in self.__write_bucket_logs.items(): - log_id = log_values['log_id'] - log_group_id = log_values['log_group_id'] - log_record = {"sch_id": sch_id, "sch_name": sch_values['display_name'], "id": bucket_name} - log_region = log_values['region'] - - bucket_log_group_in_sch = list(filter(lambda source: source['log_group_id'] == log_group_id and sch_values['region'] == log_region, sch_values['log_sources'])) - bucket_log_in_sch = list(filter(lambda source: source['log_id'] == log_id and sch_values['region'] == log_region, sch_values['log_sources'])) - - # Checking if the Bucket's log group in is in SCH's log sources & the log_id is empty so it covers everything in the log group - if bucket_log_group_in_sch and not (bucket_log_in_sch): - self.__obp_regional_checks[sch_values['region']]['Write_Bucket']['buckets'].append(log_record) - - # Checking if the Bucket's log Group in is in the service connector's log sources if so I will add it - elif bucket_log_in_sch: - self.__obp_regional_checks[sch_values['region']]['Write_Bucket']['buckets'].append(log_record) - - # else: - # self.__obp_regional_checks[sch_values['region']]['Write_Bucket']['findings'].append(bucket_name) - - # Bucket Read Log Checks - - for bucket_name, log_values in self.__read_bucket_logs.items(): - log_id = log_values['log_id'] - log_group_id = log_values['log_group_id'] - log_record = {"sch_id": sch_id, "sch_name": sch_values['display_name'], "id": bucket_name} - - log_region = log_values['region'] - - bucket_log_group_in_sch = list(filter(lambda source: source['log_group_id'] == log_group_id and sch_values['region'] == log_region, sch_values['log_sources'])) - bucket_log_in_sch = list(filter(lambda source: source['log_id'] == log_id and sch_values['region'] == log_region, sch_values['log_sources'])) - - # Checking if the Bucket's log group in is in SCH's log sources & the log_id is empty so it covers everything in the log group - if bucket_log_group_in_sch and not (bucket_log_in_sch): - self.__obp_regional_checks[sch_values['region']]['Read_Bucket']['buckets'].append(log_record) - - # Checking if the Bucket's log id in is in the service connector's log sources if so I will add it - elif bucket_log_in_sch: - self.__obp_regional_checks[sch_values['region']]['Read_Bucket']['buckets'].append(log_record) - - # Consolidating regional SERVICE LOGGING findings into centralized finding report - for region_key, region_values in self.__obp_regional_checks.items(): - - for finding in region_values['VCN']['subnets']: - logged_subnet = list(filter(lambda subnet: subnet['id'] == finding['id'], self.__network_subnets)) - # Checking that the subnet has not already been written to OBP - existing_finding = list(filter(lambda subnet: subnet['id'] == finding['id'], self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['OBP'])) - if len(logged_subnet) != 0: - record = logged_subnet[0].copy() - record['sch_id'] = finding['sch_id'] - record['sch_name'] = finding['sch_name'] - - if logged_subnet and not (existing_finding): - self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['OBP'].append(record) - # else: - # print("Found this subnet being logged but the subnet does not exist: " + str(finding)) - - for finding in region_values['Write_Bucket']['buckets']: - logged_bucket = list(filter(lambda bucket: bucket['name'] == finding['id'], self.__buckets)) - if len(logged_bucket) != 0: - record = logged_bucket[0].copy() - record['sch_id'] = finding['sch_id'] - record['sch_name'] = finding['sch_name'] - - if logged_bucket: - self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['OBP'].append(record) - - for finding in region_values['Read_Bucket']['buckets']: - logged_bucket = list(filter(lambda bucket: bucket['name'] == finding['id'], self.__buckets)) - if len(logged_bucket) != 0: - record = logged_bucket[0].copy() - record['sch_id'] = finding['sch_id'] - record['sch_name'] = finding['sch_name'] - - if logged_bucket: - self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['OBP'].append(record) - - # Finding looking at all buckets and seeing if they meet one of the OBPs in one of the regions - for finding in self.__buckets: - read_logged_bucket = list(filter(lambda bucket: bucket['name'] == finding['name'] and bucket['region'] == finding['region'], self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['OBP'])) - if not (read_logged_bucket): - self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Findings'].append(finding) - - write_logged_bucket = list(filter(lambda bucket: bucket['name'] == finding['name'] and bucket['region'] == finding['region'], self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['OBP'])) - if not (write_logged_bucket): - self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['Findings'].append(finding) + if self.__cloud_guard_targets[self.__tenancy.id]: + if self.__cloud_guard_targets[self.__tenancy.id]['target_detector_recipes']: + for recipe in self.__cloud_guard_targets[self.__tenancy.id]['target_detector_recipes']: + if recipe.detector.upper() == 'IAAS_CONFIGURATION_DETECTOR': + cloud_guard_record['targert_configuration_detector'] = True + if recipe.owner.upper() == "CUSTOMER": + cloud_guard_record['targert_configuration_detector_customer_owned'] = True - # Finding looking at all subnet and seeing if they meet one of the OBPs in one of the regions - for finding in self.__network_subnets: - logged_subnet = list(filter(lambda subnet: subnet['id'] == finding['id'], self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['OBP'])) - if not (logged_subnet): - self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['Findings'].append(finding) + elif recipe.detector.upper() == 'IAAS_ACTIVITY_DETECTOR': + cloud_guard_record['target_activity_detector'] = True + if recipe.owner.upper() == "CUSTOMER": + cloud_guard_record['target_activity_detector_customer_owned'] = True - # Setting VCN Flow Logs Findings - if self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['Findings']: - self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['Status'] = False + elif recipe.detector.upper() == 'IAAS_THREAT_DETECTOR': + cloud_guard_record['target_threat_detector'] = True + if recipe.owner.upper() == "CUSTOMER": + cloud_guard_record['target_threat_detector_customer_owned'] = True - else: - self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['Status'] = True + if self.__cloud_guard_targets[self.__tenancy.id]['target_responder_recipes']: + cloud_guard_record['target_responder_recipes'] = True + for recipe in self.__cloud_guard_targets[self.__tenancy.id]['target_responder_recipes']: + if recipe.owner.upper() == 'CUSTOMER': + cloud_guard_record['target_responder_recipes_customer_owned'] = True - # Setting Write Bucket Findings - if self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['Findings']: - self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['Status'] = False + for rule in recipe.effective_responder_rules: + if rule.responder_rule_id.upper() == 'EVENT' and rule.details.is_enabled: + cloud_guard_record['target_responder_event_rule'] = True - elif not self.__service_connectors: - # If there are no service connectors then by default all buckets are not logged - self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['Status'] = False - self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['Findings'] += self.__buckets + cloud_guard_record['target_id'] = self.__cloud_guard_targets[self.__tenancy.id]['id'] + cloud_guard_record['target_name'] = self.__cloud_guard_targets[self.__tenancy.id]['display_name'] - else: - self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['Status'] = True + except Exception: + pass - # Setting Read Bucket Findings - if self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Findings']: - self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Status'] = False + all_cloud_guard_checks = True + for key, value in cloud_guard_record.items(): + if not (value): + all_cloud_guard_checks = False - elif not self.__service_connectors: - # If there are no service connectors then by default all buckets are not logged - self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Status'] = False - self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Findings'] += self.__buckets + self.obp_foundations_checks['Cloud_Guard_Config']['Status'] = all_cloud_guard_checks + if all_cloud_guard_checks: + self.obp_foundations_checks['Cloud_Guard_Config']['OBP'].append(cloud_guard_record) else: - self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Status'] = True - - ####################################### - # OBP Networking Checks - ####################################### + self.obp_foundations_checks['Cloud_Guard_Config']['Findings'].append(cloud_guard_record) + ####################################### + # OBP Networking Checks + ####################################### + def __obp_check_networking(self): # Fast Connect Connections for drg_id, drg_values in self.__network_drg_attachments.items(): @@ -4600,13 +5021,16 @@ def __obp_analyze_tenancy_data(self): elif attachment['network_type'].upper() == 'VIRTUAL_CIRCUIT': # Checking for Provision and BGP enabled Virtual Circuits and that it is associated - for virtual_circuit in self.__network_fastconnects[attachment['drg_id']]: - if attachment['network_id'] == virtual_circuit['id']: - if virtual_circuit['lifecycle_state'].upper() == 'PROVISIONED' and virtual_circuit['bgp_session_state'].upper() == "UP": - # Good VC to increment number of VCs and append the provider name - fast_connect_providers.add(virtual_circuit['provider_name']) - number_of_valid_fast_connect_circuits += 1 - + try: + for virtual_circuit in self.__network_fastconnects[attachment['drg_id']]: + if attachment['network_id'] == virtual_circuit['id']: + if virtual_circuit['lifecycle_state'].upper() == 'PROVISIONED' and virtual_circuit['bgp_session_state'].upper() == "UP": + # Good VC to increment number of VCs and append the provider name + fast_connect_providers.add(virtual_circuit['provider_name']) + number_of_valid_fast_connect_circuits += 1 + except Exception: + debug("__obp_analyze_tenancy_data: Fast Connect Connections check: DRG ID not found " + str(drg_id)) + self.__errors.append({"id" : str(drg_id), "error" : str("__obp_analyze_tenancy_data: Fast Connect Connections check: DRG ID not found")}) try: record = { "drg_id": drg_id, @@ -4657,78 +5081,10 @@ def __obp_analyze_tenancy_data(self): self.obp_foundations_checks["Networking_Connectivity"]["Findings"] += region_values["Network_Connectivity"]["findings"] self.obp_foundations_checks["Networking_Connectivity"]["OBP"] += region_values["Network_Connectivity"]["drgs"] - ####################################### - # Cloud Guard Checks - ####################################### - cloud_guard_record = { - "cloud_guard_endable": True if self.__cloud_guard_config_status == 'ENABLED' else False, - "target_at_root": False, - "targert_configuration_detector": False, - "targert_configuration_detector_customer_owned": False, - "target_activity_detector": False, - "target_activity_detector_customer_owned": False, - "target_threat_detector": False, - "target_threat_detector_customer_owned": False, - "target_responder_recipes": False, - "target_responder_recipes_customer_owned": False, - "target_responder_event_rule": False, - } - - try: - # Cloud Guard Target attached to the root compartment with activity, config, and threat detector plus a responder - if self.__cloud_guard_targets[self.__tenancy.id]: - - cloud_guard_record['target_at_root'] = True - - if self.__cloud_guard_targets[self.__tenancy.id]: - if self.__cloud_guard_targets[self.__tenancy.id]['target_detector_recipes']: - for recipe in self.__cloud_guard_targets[self.__tenancy.id]['target_detector_recipes']: - if recipe.detector.upper() == 'IAAS_CONFIGURATION_DETECTOR': - cloud_guard_record['targert_configuration_detector'] = True - if recipe.owner.upper() == "CUSTOMER": - cloud_guard_record['targert_configuration_detector_customer_owned'] = True - - elif recipe.detector.upper() == 'IAAS_ACTIVITY_DETECTOR': - cloud_guard_record['target_activity_detector'] = True - if recipe.owner.upper() == "CUSTOMER": - cloud_guard_record['target_activity_detector_customer_owned'] = True - - elif recipe.detector.upper() == 'IAAS_THREAT_DETECTOR': - cloud_guard_record['target_threat_detector'] = True - if recipe.owner.upper() == "CUSTOMER": - cloud_guard_record['target_threat_detector_customer_owned'] = True - - if self.__cloud_guard_targets[self.__tenancy.id]['target_responder_recipes']: - cloud_guard_record['target_responder_recipes'] = True - for recipe in self.__cloud_guard_targets[self.__tenancy.id]['target_responder_recipes']: - if recipe.owner.upper() == 'CUSTOMER': - cloud_guard_record['target_responder_recipes_customer_owned'] = True - - for rule in recipe.effective_responder_rules: - if rule.responder_rule_id.upper() == 'EVENT' and rule.details.is_enabled: - cloud_guard_record['target_responder_event_rule'] = True - - cloud_guard_record['target_id'] = self.__cloud_guard_targets[self.__tenancy.id]['id'] - cloud_guard_record['target_name'] = self.__cloud_guard_targets[self.__tenancy.id]['display_name'] - - except Exception: - pass - - all_cloud_guard_checks = True - for key, value in cloud_guard_record.items(): - if not (value): - all_cloud_guard_checks = False - - self.obp_foundations_checks['Cloud_Guard_Config']['Status'] = all_cloud_guard_checks - if all_cloud_guard_checks: - self.obp_foundations_checks['Cloud_Guard_Config']['OBP'].append(cloud_guard_record) - else: - self.obp_foundations_checks['Cloud_Guard_Config']['Findings'].append(cloud_guard_record) - - ####################################### - # Certificate Expiry Check - ####################################### - + ####################################### + # OBP Certificate Expiry Check + ####################################### + def __obp_check_certificates(self): for cert in self.__raw_oci_certificates: debug("\t__obp_analyze_tenancy_data: Iterating through certificates") @@ -4747,6 +5103,186 @@ def __obp_analyze_tenancy_data(self): else: self.obp_foundations_checks['Certificates_Near_Expiry']['Status'] = True + ####################################### + # OBP Subnet Log Checks + ####################################### + def __obp_check_subnet_logs(self): + cis_logged_subnets = set() + all_subnet_nets = set() + for subnet in self.cis_foundations_benchmark_3_0['4.13']['Findings']: + cis_logged_subnets.add(subnet['id']) + for subnet in self.cis_foundations_benchmark_3_0['4.13']['Total']: + all_subnet_nets.add(subnet['id']) + + list_of_properly_logged_subnets = all_subnet_nets - cis_logged_subnets + # need to check for no logs + for sch_id, sch_values in self.__service_connectors.items(): + if self.__all_logs and 'flowlogs' in self.__all_logs and \ + sch_values['lifecycle_state'].upper() == "ACTIVE" and sch_values['target_kind']: + for subnet_id in list_of_properly_logged_subnets: + log_values = None + if 'subnet' in self.__all_logs['flowlogs'] and subnet_id in self.__all_logs['flowlogs']['subnet']: + log_values = self.__all_logs['flowlogs']['subnet'][subnet_id] + elif 'all' in self.__all_logs['flowlogs'] and subnet_id in self.__all_logs['flowlogs']['all']: + log_values = self.__all_logs['flowlogs']['all'][subnet_id] + elif 'vcn' in self.__all_logs['flowlogs'] and self.__all_logs['flowlogs']['vcn']: + for vcn_id, vcn_values in self.__network_vcns.items(): + if subnet_id in vcn_values['subnets']: + log_values = self.__all_logs['flowlogs']['vcn'][vcn_id] + + log_id = log_values['id'] + log_group_id = log_values['log_group_id'] + log_record = {"sch_id": sch_id, "sch_name": sch_values['display_name'], "id": subnet_id} + + subnet_log_group_in_sch = list(filter(lambda source: source['log_group_id'] == log_group_id, sch_values['log_sources'])) + subnet_log_in_sch = list(filter(lambda source: source['log_id'] == log_id, sch_values['log_sources'])) + + if subnet_log_group_in_sch and not (subnet_log_in_sch): + self.__obp_regional_checks[sch_values['region']]['VCN']['subnets'].append(log_record) + elif subnet_log_in_sch: + self.__obp_regional_checks[sch_values['region']]['VCN']['subnets'].append(log_record) + + for region_values in self.__obp_regional_checks.values(): + for finding in region_values['VCN']['subnets']: + logged_subnet = list(filter(lambda subnet: subnet['id'] == finding['id'], self.__network_subnets)) + existing_finding = list(filter(lambda subnet: subnet['id'] == finding['id'], self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['OBP'])) + if len(logged_subnet) != 0: + record = logged_subnet[0].copy() + record['sch_id'] = finding['sch_id'] + record['sch_name'] = finding['sch_name'] + if logged_subnet and not (existing_finding): + self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['OBP'].append(record) + + for finding in self.__network_subnets: + logged_subnet = list(filter(lambda subnet: subnet['id'] == finding['id'], self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['OBP'])) + if not (logged_subnet): + self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['Findings'].append(finding) + + if self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['Findings']: + self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['Status'] = False + else: + self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['Status'] = True + + ####################################### + # OBP Subnet and Bucket Log Checks + ####################################### + def __obp_check_bucket_logs(self): + for sch_id, sch_values in self.__service_connectors.items(): + if self.__all_logs and 'objectstorage' in self.__all_logs and \ + sch_values['lifecycle_state'].upper() == "ACTIVE" and sch_values['target_kind']: + + # Bucket Write Logs Checks + # for bucket_name, log_values in self.__write_bucket_logs.items(): + if 'write' in self.__all_logs['objectstorage']: + for bucket_name, log_values in self.__all_logs['objectstorage']['write'].items(): + log_id = log_values['id'] + log_group_id = log_values['log_group_id'] + log_record = {"sch_id": sch_id, "sch_name": sch_values['display_name'], "id": bucket_name} + log_region = log_values['region'] + + bucket_log_group_in_sch = any(source['log_group_id'] == log_group_id and sch_values['region'] == log_region for source in sch_values['log_sources']) + bucket_log_in_sch = any(source['log_id'] == log_id and sch_values['region'] == log_region for source in sch_values['log_sources']) + + # Checking if the Bucket's log group in is in SCH's log sources & the log_id is empty so it covers everything in the log group + if bucket_log_group_in_sch and not (bucket_log_in_sch): + self.__obp_regional_checks[sch_values['region']]['Write_Bucket']['buckets'].append(log_record) + + # Checking if the Bucket's log Group in is in the service connector's log sources if so I will add it + elif bucket_log_in_sch: + self.__obp_regional_checks[sch_values['region']]['Write_Bucket']['buckets'].append(log_record) + + # else: + # self.__obp_regional_checks[sch_values['region']]['Write_Bucket']['findings'].append(bucket_name) + + # Bucket Read Log Checks + if 'read' in self.__all_logs['objectstorage']: + for bucket_name, log_values in self.__all_logs['objectstorage']['read'].items(): + log_id = log_values['id'] + log_group_id = log_values['log_group_id'] + log_record = {"sch_id": sch_id, "sch_name": sch_values['display_name'], "id": bucket_name} + + log_region = log_values['region'] + + bucket_log_group_in_sch = list(filter(lambda source: source['log_group_id'] == log_group_id and sch_values['region'] == log_region, sch_values['log_sources'])) + bucket_log_in_sch = list(filter(lambda source: source['log_id'] == log_id and sch_values['region'] == log_region, sch_values['log_sources'])) + + # Checking if the Bucket's log group in is in SCH's log sources & the log_id is empty so it covers everything in the log group + if bucket_log_group_in_sch and not (bucket_log_in_sch): + self.__obp_regional_checks[sch_values['region']]['Read_Bucket']['buckets'].append(log_record) + + # Checking if the Bucket's log id in is in the service connector's log sources if so I will add it + elif bucket_log_in_sch: + self.__obp_regional_checks[sch_values['region']]['Read_Bucket']['buckets'].append(log_record) + + # Consolidating regional SERVICE LOGGING findings into centralized finding report + for region_values in self.__obp_regional_checks.values(): + for finding in region_values['Write_Bucket']['buckets']: + logged_bucket = list(filter(lambda bucket: bucket['source_resource'] == finding['id'], self.__buckets)) + if len(logged_bucket) != 0: + record = logged_bucket[0].copy() + record['sch_id'] = finding['sch_id'] + record['sch_name'] = finding['sch_name'] + + if logged_bucket: + self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['OBP'].append(record) + + for finding in region_values['Read_Bucket']['buckets']: + logged_bucket = list(filter(lambda bucket: bucket['source_resource'] == finding['id'], self.__buckets)) + if len(logged_bucket) != 0: + record = logged_bucket[0].copy() + record['sch_id'] = finding['sch_id'] + record['sch_name'] = finding['sch_name'] + + if logged_bucket: + self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['OBP'].append(record) + + # Finding looking at all buckets and seeing if they meet one of the OBPs in one of the regions + for finding in self.__buckets: + read_logged_bucket = list(filter(lambda bucket: bucket['name'] == finding['name'] and bucket['region'] == finding['region'], self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['OBP'])) + if not (read_logged_bucket): + self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Findings'].append(finding) + + write_logged_bucket = list(filter(lambda bucket: bucket['name'] == finding['name'] and bucket['region'] == finding['region'], self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['OBP'])) + if not (write_logged_bucket): + self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['Findings'].append(finding) + # Setting Write Bucket Findings + if self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['Findings']: + self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['Status'] = False + + elif not self.__service_connectors: + # If there are no service connectors then by default all buckets are not logged + self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['Status'] = False + self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['Findings'] += self.__buckets + + else: + self.obp_foundations_checks['SIEM_Write_Bucket_Logs']['Status'] = True + + # Setting Read Bucket Findings + if self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Findings']: + self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Status'] = False + + elif not self.__service_connectors: + # If there are no service connectors then by default all buckets are not logged + self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Status'] = False + self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Findings'] += self.__buckets + else: + self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Status'] = True + + + ########################################################################## + # Analyzes Tenancy Data for Oracle Best Practices Report + ########################################################################## + def __obp_analyze_tenancy_data(self): + self.__obp_init_regional_checks() + self.__obp_check_budget() + self.__obp_check_audit_log_compartments() + self.__obp_check_cloud_guard() + self.__obp_check_networking() + self.__obp_check_certificates() + self.__obp_check_bucket_logs() + self.__obp_check_subnet_logs() + + ########################################################################## # Orchestrates data collection and CIS report generation ########################################################################## @@ -4755,16 +5291,23 @@ def __report_generate_cis_report(self, level): # Creating summary report summary_report = [] - for key, recommendation in self.cis_foundations_benchmark_2_0.items(): + for key, recommendation in self.cis_foundations_benchmark_3_0.items(): if recommendation['Level'] <= level: - report_filename = "cis" + " " + recommendation['section'] + "_" + recommendation['recommendation_#'] + report_filename = f'{self.__report_prefix}cis {recommendation["section"]}_{recommendation["recommendation_#"]}' report_filename = report_filename.replace(" ", "_").replace(".", "-").replace("_-_", "_") + ".csv" if recommendation['Status']: compliant_output = "Yes" + compliance_percentage = "100%" elif recommendation['Status'] is None: compliant_output = "Not Applicable" + compliance_percentage = "N/A" else: compliant_output = "No" + compliance_percentage = ( + str(int(( + (len(recommendation.get('Total') or []) - len(recommendation.get('Findings') or [])) + / len(recommendation.get('Total') or []) * 100 + )) if len(recommendation.get('Total') or []) > 0 else 0) + "%") record = { "Recommendation #": f"{key}", "Section": recommendation['section'], @@ -4773,6 +5316,7 @@ def __report_generate_cis_report(self, level): "Findings": (str(len(recommendation['Findings'])) if len(recommendation['Findings']) > 0 else " "), "Compliant Items": str(len(recommendation['Total']) - len(recommendation['Findings'])), "Total": (str(len(recommendation['Total'])) if len(recommendation['Total']) > 0 else " "), + "Compliance Percentage Per Recommendation": compliance_percentage, "Title": recommendation['Title'], "CIS v8": recommendation['CISv8'], "CCCS Guard Rail": recommendation['CCCS Guard Rail'], @@ -4786,7 +5330,7 @@ def __report_generate_cis_report(self, level): # self.__print_to_csv_file("cis", recommendation['section'] + "_" + recommendation['recommendation_#'], recommendation['Findings']) # Screen output for CIS Summary Report - print_header("CIS Foundations Benchmark 2.0.0 Summary Report") + print_header("CIS Foundations Benchmark 3.0.0 Summary Report") print('Num' + "\t" + "Level " + "\t" "Compliant" + "\t" + "Findings " + "\t" + "Total " + "\t\t" + 'Title') print('#' * 90) @@ -4814,30 +5358,120 @@ def __report_generate_cis_report(self, level): summary_file_name = self.__report_generate_html_summary_report("cis", "html_summary_report", summary_report) summary_files.append(summary_file_name) + if OUTPUT_DIAGRAMS: + diagram_file_name = self.__generate_compliance_diagram("cis", "summary_compliance", summary_report) + summary_files.append(diagram_file_name) + diagram_file_name = self.__generate_compliance_by_area_diagram("cis", "summary_compliance_by_focus_area", summary_report) + summary_files.append(diagram_file_name) + # Outputing to a bucket if I have one if summary_files and self.__output_bucket: for summary_file in summary_files: self.__os_copy_report_to_object_storage( self.__output_bucket, summary_file) - for key, recommendation in self.cis_foundations_benchmark_2_0.items(): + for key, recommendation in self.cis_foundations_benchmark_3_0.items(): if recommendation['Level'] <= level: report_file_name = self.__print_to_csv_file("cis", recommendation['section'] + "_" + recommendation['recommendation_#'], recommendation['Findings']) if report_file_name and self.__output_bucket: self.__os_copy_report_to_object_storage( self.__output_bucket, report_file_name) + ########################################################################## + # Generate summary diagrams + ########################################################################## + diagram_colors = ['#4C825C','#C74634'] + diagram_values = ['Compliant', 'Non-compliant'] + diagram_sections = ( + 'Identity and Access Management', + 'Networking', + 'Compute', + 'Logging and Monitoring', + 'Storage', + 'Asset Management' + ) + diagram_fontweight = 'bold' + diagram_fontcolor_reverse = 'white' + + ########################################################################## + # __cis_compliance + ########################################################################## + def __cis_compliance(self, filename, title, values=None): + plt.close('all') + plt.figure(figsize=(6,5)) + wegdes, labels, pcttexts = plt.pie(values, labels=self.diagram_values, colors=self.diagram_colors, autopct='%.0f%%', wedgeprops={'linewidth': 3.0, 'edgecolor': 'white'}, startangle=90, counterclock=False, radius=1.1) + for t in labels: + t.set_fontweight(self.diagram_fontweight) + for p in pcttexts: + p.set_fontweight(self.diagram_fontweight) + p.set_color(self.diagram_fontcolor_reverse) + plt.title(title, fontweight=self.diagram_fontweight, pad=30.0) + plt.savefig(filename) + + ########################################################################## + # __cis_compliance_by_area + ########################################################################## + def __cis_compliance_by_area(self, filename, title, section_values=None): + plt.close('all') + height = 0.4 + fig, ax = plt.subplots(figsize=(10,5), layout='compressed') + y = np.arange(len(self.diagram_sections)) + p = ax.barh(y - height/2, section_values[self.diagram_values[0]], height, color=self.diagram_colors[0]) + ax.bar_label(p, padding=-16, color=self.diagram_fontcolor_reverse, fontweight=self.diagram_fontweight) + p = ax.barh(y + (height/2), section_values[self.diagram_values[1]], height, color=self.diagram_colors[1]) + ax.bar_label(p, padding=-16, color=self.diagram_fontcolor_reverse, fontweight=self.diagram_fontweight) + ax.set_frame_on(False) + ax.set_title(title, fontweight=self.diagram_fontweight, loc='left') + ax.set_yticks(y) + ax.set_yticklabels(self.diagram_sections, fontweight=self.diagram_fontweight) + ax.invert_yaxis() + plt.tick_params(left=False, right=False, labelbottom=False, bottom=False) + plt.savefig(filename) + + ########################################################################## + # __generate_compliance_diagram + ########################################################################## + def __generate_compliance_diagram(self, header, file_subject, data): + compliant = 0 + non_compliant = 0 + for finding in data: + if finding['Compliant'] == 'Yes': + compliant += 1 + else: + non_compliant += 1 + cis_compliance_file = self.__get_output_file_path(header, file_subject, '.png') + self.__cis_compliance(cis_compliance_file, 'CIS Recommendation Compliance', [compliant, non_compliant]) + return cis_compliance_file + + ########################################################################## + # __generate_compliance_by_area_diagram + ########################################################################## + def __generate_compliance_by_area_diagram(self, header, file_subject, data): + compliants = [] + non_compliants = [] + for section in self.diagram_sections: + compliant = 0 + non_compliant = 0 + for finding in data: + if section in finding['Section']: + if finding['Compliant'] == 'Yes': + compliant += 1 + else: + non_compliant += 1 + compliants.append(compliant) + non_compliants.append(non_compliant) + + cis_compliance_by_area_file = self.__get_output_file_path(header, file_subject, '.png') + self.__cis_compliance_by_area(cis_compliance_by_area_file, 'CIS Recommendation Compliance per Focus Area', { + self.diagram_values[0]: compliants, + self.diagram_values[1]: non_compliants, + }) + return cis_compliance_by_area_file + ########################################################################## # Generates an HTML report ########################################################################## def __report_generate_html_summary_report(self, header, file_subject, data): - try: - # Creating report directory - if not os.path.isdir(self.__report_directory): - os.mkdir(self.__report_directory) - - except Exception as e: - raise Exception("Error in creating report directory: " + str(e.args)) try: # if no data @@ -4845,9 +5479,7 @@ def __report_generate_html_summary_report(self, header, file_subject, data): return None # get the file name of the HTML - file_name = header + "_" + file_subject - file_name = (file_name.replace(" ", "_")).replace(".", "-").replace("_-_", "_") + ".html" - file_path = os.path.join(self.__report_directory, f'{self.__report_prefix}{file_name}') + file_path = self.__get_output_file_path(header, file_subject, '.html') # add report_datetimeto each dictionary result = [dict(item, extract_date=self.start_time_str) @@ -4873,7 +5505,7 @@ def __report_generate_html_summary_report(self, header, file_subject, data): # generate fields fields = ['Recommendation #', 'Compliant', 'Section', 'Details'] - html_title = 'CIS OCI Foundations Benchmark 2.0.0 - Compliance Report' + html_title = 'CIS OCI Foundations Benchmark 3.0.0 - Compliance Report' with open(file_path, mode='w') as html_file: # Creating table header html_file.write('') @@ -4911,7 +5543,11 @@ def __report_generate_html_summary_report(self, header, file_subject, data): r = result[0] extract_date = r['extract_date'].replace('T',' ') html_file.write(f'
Extract Date: {extract_date} UTC
') - html_file.write("") + html_file.write('') + if OUTPUT_DIAGRAMS: + # Include dashboard + html_file.write(f'
') + html_file.write(f'
 
') # Navigation html_file.write('
') html_file.write('
') @@ -4965,7 +5601,7 @@ def __report_generate_html_summary_report(self, header, file_subject, data): if int(total) > 1: tmp += 's' html_file.write(f'{str(compliant)}{tmp}\n') - html_file.write(f"{str(row['Section'])}\n") + html_file.write(f'{str(row["Section"])}\n') # Details html_file.write('') html_file.write(f'') @@ -4990,9 +5626,7 @@ def __report_generate_html_summary_report(self, header, file_subject, data): html_file.write("
Title{str(row["Title"])}
\n") # Line html_file.write('
') - html_file.write('
') - html_file.write('

Non-compliant Recommendations

') - html_file.write('
') + html_file.write('

Non-compliant Recommendations

') html_file.write('
') html_file.write('
') html_file.write('
') @@ -5028,7 +5662,7 @@ def __report_generate_html_summary_report(self, header, file_subject, data): html_file.write(f'{v}\n') else: html_file.write(f'{v}\n') - f = row['Findings'] + f = row['Findings'] t = row['Total'] tmp = '' if t != ' ': @@ -5047,14 +5681,14 @@ def __report_generate_html_summary_report(self, header, file_subject, data): html_file.write('CCCS Guard Rail') html_file.write('File') html_file.write(f'{str(row["Level"])}') - cis_v8 = str(row["CIS v8"]).replace("[","").replace("]","").replace("'","") + cis_v8 = str(row["CIS v8"]).replace("[", "").replace("]", "").replace("'", "") html_file.write(f'{cis_v8}') html_file.write(f'{str(row["CCCS Guard Rail"])}') v = str(row['Filename']) if v == ' ': html_file.write(' ') else: - html_file.write(f'{v}') + html_file.write(f'{v}') html_file.write('') html_file.write("") @@ -5064,7 +5698,7 @@ def __report_generate_html_summary_report(self, header, file_subject, data): html_file.write('
') # Creating appendix for the report for finding in html_appendix: - fing = self.cis_foundations_benchmark_2_0[finding] + fing = self.cis_foundations_benchmark_3_0[finding] html_file.write(f'

{finding} – {fing["Title"]}

\n') for item_key, item_value in self.cis_report_data[finding].items(): if item_value != "": @@ -5085,20 +5719,19 @@ def __report_generate_html_summary_report(self, header, file_subject, data): html_file.write(f"

{v}

\n") html_file.write("
\n") # Closing HTML + report_year = str(self.start_datetime.strftime('%Y')) html_file.write("""
\n") +
\n') print("HTML: " + file_subject.ljust(22) + " --> " + file_path) # Used by Upload @@ -5223,7 +5856,9 @@ def __collect_tenancy_data(self): self.__ons_read_subscriptions, self.__network_read_network_security_lists, self.__network_read_network_security_groups_rules, + self.__network_read_network_vcns, self.__network_read_network_subnets, + self.__network_read_network_capturefilters, self.__adb_read_adbs, self.__oic_read_oics, self.__oac_read_oacs, @@ -5231,7 +5866,6 @@ def __collect_tenancy_data(self): self.__boot_volume_read_boot_volumes, self.__fss_read_fsss, self.__core_instance_read_compute, - self.__certificates_read_certificates ] # Oracle Best practice functions @@ -5242,8 +5876,8 @@ def __collect_tenancy_data(self): self.__network_read_drgs, self.__network_read_drg_attachments, self.__sch_read_service_connectors, - self.__network_topology_dump - ] + self.__certificates_read_certificates + ] else: obp_functions = [] @@ -5251,7 +5885,8 @@ def __collect_tenancy_data(self): if self.__all_resources: all_resources = [ - self.__search_resources_all_resources_in_tenancy, + self.__network_topology_dump, + self.__search_resources_all_resources_in_tenancy ] else: all_resources = [] @@ -5269,6 +5904,9 @@ def execute_function(func): for future in concurrent.futures.as_completed(futures): future.result() + if obp_functions: + self.__unify_network_data() + ########################################################################## # Generate Raw Data Output ########################################################################## @@ -5288,6 +5926,8 @@ def __report_generate_raw_data_output(self): "network_security_groups": self.__network_security_groups, "network_security_lists": self.__network_security_lists, "network_subnets": self.__network_subnets, + "network_vcns": list(self.__network_vcns.values()), + "network_capture_filters": list(self.__network_capturefilters.values()), "autonomous_databases": self.__autonomous_databases, "analytics_instances": self.__analytics_instances, "integration_instances": self.__integration_instances, @@ -5353,18 +5993,26 @@ def __os_copy_report_to_object_storage(self, bucketname, filename): "Error opening file os_copy_report_to_object_storage: " + str(e.args)) ########################################################################## - # Print to CSV + # Get output file path with suffix ########################################################################## - def __print_to_csv_file(self, header, file_subject, data): - debug("__print_to_csv_file: " + header + "_" + file_subject) + def __get_output_file_path(self, header, file_subject, suffix): try: # Creating report directory if not os.path.isdir(self.__report_directory): os.mkdir(self.__report_directory) except Exception as e: - raise Exception( - "Error in creating report directory: " + str(e.args)) + raise Exception(f'Error in creating report directory: {str(e.args)}') + + file_name = f'{header}_{file_subject}' + file_name = f'{file_name.replace(" ", "_").replace(".", "-").replace("_-_", "_")}{suffix}' + return os.path.join(self.__report_directory, f'{self.__report_prefix}{file_name}') + + ########################################################################## + # Print to CSV + ########################################################################## + def __print_to_csv_file(self, header, file_subject, data): + debug("__print_to_csv_file: " + header + "_" + file_subject) try: # if no data @@ -5372,10 +6020,7 @@ def __print_to_csv_file(self, header, file_subject, data): return None # get the file name of the CSV - - file_name = header + "_" + file_subject - file_name = (file_name.replace(" ", "_")).replace(".", "-").replace("_-_", "_") + ".csv" - file_path = os.path.join(self.__report_directory, f'{self.__report_prefix}{file_name}') + file_path = self.__get_output_file_path(header, file_subject, '.csv') # add report_datetimeto each dictionary result = [dict(item, extract_date=self.start_time_str) @@ -5423,26 +6068,14 @@ def __print_to_csv_file(self, header, file_subject, data): # Print to JSON ########################################################################## def __print_to_json_file(self, header, file_subject, data): - try: - # Creating report directory - if not os.path.isdir(self.__report_directory): - os.mkdir(self.__report_directory) - - except Exception as e: - raise Exception( - "Error in creating report directory: " + str(e.args)) try: # if no data if len(data) == 0: return None - # get the file name of the CSV - - file_name = header + "_" + file_subject - file_name = (file_name.replace(" ", "_") - ).replace(".", "-").replace("_-_","_") + ".json" - file_path = os.path.join(self.__report_directory, f'{self.__report_prefix}{file_name}') + # get the file name of the JSON + file_path = self.__get_output_file_path(header, file_subject, '.json') # Serializing JSON to string json_object = json.dumps(data, indent=4) @@ -5470,26 +6103,14 @@ def __print_to_json_file(self, header, file_subject, data): # Print to PKL ########################################################################## def __print_to_pkl_file(self, header, file_subject, data): - try: - # Creating report directory - if not os.path.isdir(self.__report_directory): - os.mkdir(self.__report_directory) - - except Exception as e: - raise Exception( - "Error in creating report directory: " + str(e.args)) try: # if no data if len(data) == 0: return None - # get the file name of the CSV - - file_name = header + "_" + file_subject - file_name = (file_name.replace(" ", "_") - ).replace(".", "-").replace("_-_","_") + ".pkl" - file_path = os.path.join(self.__report_directory, f'{self.__report_prefix}{file_name}') + # get the file name of the PKL + file_path = self.__get_output_file_path(header, file_subject, '.pkl') # Writing to json file with open(file_path, 'wb') as pkl_file: @@ -5504,9 +6125,7 @@ def __print_to_pkl_file(self, header, file_subject, data): except Exception as e: raise Exception("Error in __print_to_pkl_file: " + str(e.args)) - - - + ########################################################################## # Orchestrates Data collection and reports ########################################################################## @@ -5739,7 +6358,9 @@ def execute_report(): help='Checks for OCI best practices.') parser.add_argument('--all-resources', action='store_true', default=False, help='Uses Advanced Search Service to query all resources in the tenancy and outputs to a JSON. This also enables OCI Best Practice Checks (--obp) and All resource to csv (--raw) flags.') - parser.add_argument('--redact_output', action='store_true', default=False, + parser.add_argument('--disable-api-usage-check', action='store_true', default=False, + help='Disables the checking of OCI API unused for 45 days or more.') + parser.add_argument('--redact-output', action='store_true', default=False, help='Redacts OCIDs in output CSV and JSON files.') parser.add_argument('--deeplink-url-override', default=None, dest='oci_url', help='Replaces the base OCI URL (https://cloud.oracle.com) for deeplinks (i.e. https://oc10.cloud.oracle.com).') @@ -5752,7 +6373,7 @@ def execute_report(): parser.add_argument('-v', action='store_true', default=False, dest='version', help='Show the version of the script and exit.') parser.add_argument('--debug', action='store_true', default=False, - dest='debug', help='Enables debugging messages. This feature is in beta.') + dest='debug', help='Enables debugging messages printed to screen.') cmd = parser.parse_args() if cmd.version: @@ -5762,14 +6383,23 @@ def execute_report(): config, signer = create_signer(cmd.file_location, cmd.config_profile, cmd.is_instance_principals, cmd.is_delegation_token, cmd.is_security_token) config['retry_strategy'] = oci.retry.DEFAULT_RETRY_STRATEGY report = CIS_Report(config, signer, cmd.proxy, cmd.output_bucket, cmd.report_directory, cmd.report_prefix, cmd.report_summary_json, cmd.print_to_screen, \ - cmd.regions, cmd.raw, cmd.obp, cmd.redact_output, oci_url=cmd.oci_url, debug=cmd.debug, all_resources=cmd.all_resources) + cmd.regions, cmd.raw, cmd.obp, cmd.redact_output, oci_url=cmd.oci_url, debug=cmd.debug, all_resources=cmd.all_resources, disable_api_keys=cmd.disable_api_usage_check) csv_report_directory = report.generate_reports(int(cmd.level)) - try: - if OUTPUT_TO_XLSX: + if OUTPUT_TO_XLSX: + try: report_prefix = f'{cmd.report_prefix}_' if cmd.report_prefix else '' workbook = Workbook(f'{csv_report_directory}/{report_prefix}Consolidated_Report.xlsx', {'in_memory': True}) - for csvfile in glob.glob(f'{csv_report_directory}/{report_prefix}*.csv'): + if OUTPUT_DIAGRAMS: + try: + worksheet = workbook.add_worksheet('cis_summary_charts') + worksheet.insert_image('B2', f'{csv_report_directory}/{report_prefix}cis_summary_compliance.png') + worksheet.insert_image('L2', f'{csv_report_directory}/{report_prefix}cis_summary_compliance_by_focus_area.png') + except Exception: + pass + csvfiles = glob.glob(f'{csv_report_directory}/{report_prefix}*.csv') + csvfiles.sort() + for csvfile in csvfiles: worksheet_name = csvfile.split(os.path.sep)[-1].replace(report_prefix, "").replace(".csv", "").replace("raw_data_", "raw_").replace("Findings", "fds").replace("Best_Practices", "bps") @@ -5797,10 +6427,13 @@ def execute_report(): # Skipping the deep link due to formating errors in xlsx if "=HYPERLINK" not in col: worksheet.write(r, c, col) + worksheet.autofilter(0, 0, r - 1, c - 1) + worksheet.autofit() + workbook.close() - except Exception as e: - print("** Failed to output to excel. Please use CSV files. **") - print(e) + except Exception as e: + print("** Failed to output to Excel. Please use CSV files. **") + print(e) ##########################################################################