diff --git a/datasource.tf b/datasource.tf index f6fada1e..4f893a4a 100644 --- a/datasource.tf +++ b/datasource.tf @@ -20,3 +20,33 @@ data "ibm_is_subnet" "itself" { identifier = local.subnets[count.index]["id"] } */ + +data "ibm_resource_group" "resource_group" { + count = var.resource_group == null ? 0 : 1 + name = var.resource_group +} + +data "ibm_is_subnet" "existing_compute_subnets" { + count = var.vpc != null && var.compute_subnets != null ? 1 : 0 + name = var.compute_subnets[count.index] +} + +data "ibm_is_subnet" "existing_storage_subnets" { + count = var.vpc != null && var.storage_subnets != null ? 1 : 0 + name = var.storage_subnets[count.index] +} + +data "ibm_is_subnet" "existing_protocol_subnets" { + count = var.vpc != null && var.protocol_subnets != null ? 1 : 0 + name = var.protocol_subnets[count.index] +} + +data "ibm_is_subnet" "existing_client_subnets" { + count = var.vpc != null && var.client_subnets != null ? 1 : 0 + name = var.client_subnets[count.index] +} + +data "ibm_is_subnet" "existing_bastion_subnets" { + count = var.vpc != null && var.bastion_subnets != null ? 1 : 0 + name = var.bastion_subnets[count.index] +} \ No newline at end of file diff --git a/locals.tf b/locals.tf index 537c3d38..11f5390d 100644 --- a/locals.tf +++ b/locals.tf @@ -15,12 +15,13 @@ locals { # locals needed for deployer locals { # dependency: landing_zone -> deployer - vpc_id = var.vpc == null ? one(module.landing_zone.vpc_id) : var.vpc + vpc_id = var.vpc == null ? one(module.landing_zone.vpc_id) : var.vpc_id + vpc = var.vpc == null ? one(module.landing_zone.vpc_name) : var.vpc bastion_subnets = module.landing_zone.bastion_subnets kms_encryption_enabled = var.key_management != null ? true : false boot_volume_encryption_key = var.key_management != null ? one(module.landing_zone.boot_volume_encryption_key)["crn"] : null existing_kms_instance_guid = var.key_management != null ? module.landing_zone.key_management_guid : null - cos_data = module.landing_zone.cos_buckets_data + cos_data = var.enable_bastion ? [] : module.landing_zone.cos_buckets_data # Future use # skip_iam_authorization_policy = true } @@ -29,14 +30,78 @@ locals { # locals needed for landing_zone_vsi locals { # dependency: landing_zone -> deployer -> landing_zone_vsi - bastion_security_group_id = module.deployer.bastion_security_group_id - bastion_public_key_content = module.deployer.bastion_public_key_content + bastion_security_group_id = module.deployer.bastion_security_group_id + bastion_public_key_content = module.deployer.bastion_public_key_content + bastion_private_key_content = module.deployer.bastion_private_key_content + + deployer_hostname = var.enable_bastion ? flatten(module.deployer.deployer_vsi_data[*].list)[0].name : "" + deployer_ip = module.deployer.deployer_ip + + compute_public_key_contents = module.deployer.compute_public_key_content + compute_private_key_contents = module.deployer.compute_private_key_content + + # Existing subnets details + existing_compute_subnets = [ + for subnet in data.ibm_is_subnet.existing_compute_subnets : + { + cidr = subnet.ipv4_cidr_block + id = subnet.id + name = subnet.name + zone = subnet.zone + } + ] + + existing_storage_subnets = [ + for subnet in data.ibm_is_subnet.existing_storage_subnets : + { + cidr = subnet.ipv4_cidr_block + id = subnet.id + name = subnet.name + zone = subnet.zone + } + ] + + existing_protocol_subnets = [ + for subnet in data.ibm_is_subnet.existing_protocol_subnets : + { + cidr = subnet.ipv4_cidr_block + id = subnet.id + name = subnet.name + zone = subnet.zone + } + ] + + existing_client_subnets = [ + for subnet in data.ibm_is_subnet.existing_client_subnets : + { + cidr = subnet.ipv4_cidr_block + id = subnet.id + name = subnet.name + zone = subnet.zone + } + ] + + existing_bastion_subnets = [ + for subnet in data.ibm_is_subnet.existing_bastion_subnets : + { + cidr = subnet.ipv4_cidr_block + id = subnet.id + name = subnet.name + zone = subnet.zone + } + ] # dependency: landing_zone -> landing_zone_vsi - client_subnets = module.landing_zone.client_subnets - compute_subnets = module.landing_zone.compute_subnets - storage_subnets = module.landing_zone.storage_subnets - protocol_subnets = module.landing_zone.protocol_subnets + client_subnets = var.vpc != null && var.client_subnets != null ? local.existing_client_subnets : module.landing_zone.client_subnets + compute_subnets = var.vpc != null && var.compute_subnets != null ? local.existing_compute_subnets : module.landing_zone.compute_subnets + storage_subnets = var.vpc != null && var.storage_subnets != null ? local.existing_storage_subnets : module.landing_zone.storage_subnets + protocol_subnets = var.vpc != null && var.protocol_subnets != null ? local.existing_protocol_subnets : module.landing_zone.protocol_subnets + + storage_subnet = [for subnet in local.storage_subnets : subnet.name] + protocol_subnet = [for subnet in local.protocol_subnets : subnet.name] + compute_subnet = [for subnet in local.compute_subnets : subnet.name] + client_subnet = [for subnet in local.client_subnets : subnet.name] + bastion_subnet = [for subnet in local.bastion_subnets : subnet.name] #boot_volume_encryption_key = var.key_management != null ? one(module.landing_zone.boot_volume_encryption_key)["crn"] : null #skip_iam_authorization_policy = true @@ -52,8 +117,8 @@ locals { #boot_volume_encryption_key = var.key_management != null ? one(module.landing_zone.boot_volume_encryption_key)["crn"] : null # dependency: landing_zone_vsi -> file-share - compute_subnet_id = local.compute_subnets[0].id - compute_security_group_id = module.landing_zone_vsi.compute_sg_id + compute_subnet_id = var.vpc == null && var.compute_subnets == null ? local.compute_subnets[0].id : [for subnet in data.ibm_is_subnet.existing_compute_subnets : subnet.id][0] + compute_security_group_id = var.enable_deployer ? [] : module.landing_zone_vsi[0].compute_sg_id management_instance_count = sum(var.management_instances[*]["count"]) default_share = local.management_instance_count > 0 ? [ { @@ -77,39 +142,65 @@ locals { # locals needed for DNS locals { # dependency: landing_zone -> DNS - resource_group_id = one(values(one(module.landing_zone.resource_group_id))) + resource_group = var.resource_group == null ? "workload-rg" : var.resource_group + resource_group_ids = { + # management_rg = var.resource_group == null ? module.landing_zone.resource_group_id[0]["management-rg"] : one(values(one(module.landing_zone.resource_group_id))) + service_rg = var.resource_group == null ? module.landing_zone.resource_group_id[0]["service-rg"] : data.ibm_resource_group.resource_group[0].id + workload_rg = var.resource_group == null ? module.landing_zone.resource_group_id[0]["workload-rg"] : data.ibm_resource_group.resource_group[0].id + } + # resource_group_id = one(values(one(module.landing_zone.resource_group_id))) vpc_crn = var.vpc == null ? one(module.landing_zone.vpc_crn) : one(data.ibm_is_vpc.itself[*].crn) # TODO: Fix existing subnet logic #subnets_crn = var.vpc == null ? module.landing_zone.subnets_crn : ### + existing_compute_subnet_crns = [for subnet in data.ibm_is_subnet.existing_compute_subnets : subnet.crn] + existing_storage_subnet_crns = [for subnet in data.ibm_is_subnet.existing_storage_subnets : subnet.crn] + existing_protocol_subnet_crns = [for subnet in data.ibm_is_subnet.existing_protocol_subnets : subnet.crn] + existing_client_subnet_crns = [for subnet in data.ibm_is_subnet.existing_client_subnets : subnet.crn] + existing_bastion_subnet_crns = [for subnet in data.ibm_is_subnet.existing_bastion_subnets : subnet.crn] + subnets_crn = concat(local.existing_compute_subnet_crns, local.existing_storage_subnet_crns, local.existing_protocol_subnet_crns, local.existing_client_subnet_crns, local.existing_bastion_subnet_crns) + # subnets_crn = var.vpc == null && var.compute_subnets == null ? module.landing_zone.subnets_crn : concat(local.existing_subnet_crns, module.landing_zone.subnets_crn) #subnets = flatten([local.compute_subnets, local.storage_subnets, local.protocol_subnets]) #subnets_crns = data.ibm_is_subnet.itself[*].crn - subnets_crn = module.landing_zone.subnets_crn + # subnets_crn = module.landing_zone.subnets_crn #boot_volume_encryption_key = var.key_management != null ? one(module.landing_zone.boot_volume_encryption_key)["crn"] : null # dependency: landing_zone_vsi -> file-share } +data "external" "get_hostname" { + program = ["sh", "-c", "echo '{\"name\": \"'$(hostname)'\", \"ipv4_address\": \"'$(hostname -I | awk '{print $1}')'\"}'"] +} + + # locals needed for dns-records locals { # dependency: dns -> dns-records - dns_instance_id = module.dns.dns_instance_id + dns_instance_id = var.enable_deployer ? "" : module.dns[0].dns_instance_id + dns_custom_resolver_id = var.enable_deployer ? "" : module.dns[0].dns_custom_resolver_id + dns_zone_map_list = var.enable_deployer ? [] : module.dns[0].dns_zone_maps compute_dns_zone_id = one(flatten([ - for dns_zone in module.dns.dns_zone_maps : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["compute"] + for dns_zone in local.dns_zone_map_list : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["compute"] ])) storage_dns_zone_id = one(flatten([ - for dns_zone in module.dns.dns_zone_maps : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["storage"] + for dns_zone in local.dns_zone_map_list : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["storage"] ])) protocol_dns_zone_id = one(flatten([ - for dns_zone in module.dns.dns_zone_maps : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["protocol"] + for dns_zone in local.dns_zone_map_list : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["protocol"] ])) # dependency: landing_zone_vsi -> dns-records - compute_instances = flatten([module.landing_zone_vsi.management_vsi_data, module.landing_zone_vsi.compute_vsi_data]) - storage_instances = flatten([module.landing_zone_vsi.storage_vsi_data, module.landing_zone_vsi.protocol_vsi_data]) - protocol_instances = flatten([module.landing_zone_vsi.protocol_vsi_data]) + compute_instances = var.enable_deployer ? [] : flatten([module.landing_zone_vsi[0].management_vsi_data, module.landing_zone_vsi[0].compute_vsi_data]) + storage_instances = var.enable_deployer ? [] : flatten([module.landing_zone_vsi[0].storage_vsi_data, module.landing_zone_vsi[0].protocol_vsi_data]) + protocol_instances = var.enable_deployer ? [] : flatten([module.landing_zone_vsi[0].protocol_vsi_data]) + deployer_instances = [ + { + name = var.deployer_hostname + ipv4_address = var.deployer_ip + } + ] compute_dns_records = [ - for instance in local.compute_instances : + for instance in concat(local.compute_instances, local.deployer_instances): { name = instance["name"] rdata = instance["ipv4_address"] @@ -133,22 +224,63 @@ locals { # locals needed for inventory locals { - compute_hosts = try([for name in local.compute_instances[*]["name"] : "${name}.${var.dns_domain_names["compute"]}"], []) #local.compute_instances[*]["ipv4_address"] - storage_hosts = try([for name in local.storage_instances[*]["name"] : "${name}.${var.dns_domain_names["storage"]}"], []) #local.storage_instances[*]["ipv4_address"] - compute_inventory_path = "./../../modules/ansible-roles/compute.ini" - storage_inventory_path = "./../../modules/ansible-roles/storage.ini" + compute_hosts = try([for name in local.compute_instances[*]["name"] : "${name}.${var.dns_domain_names["compute"]}"], []) #concat(["${data.external.get_hostname.result["name"]}.${var.dns_domain_names["compute"]}"], try([for name in local.compute_instances[*]["name"] : "${name}.${var.dns_domain_names["compute"]}"], [])) + storage_hosts = try([for name in local.storage_instances[*]["name"] : "${name}.${var.dns_domain_names["storage"]}"], []) + compute_inventory_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/compute.ini" : "${path.root}/modules/ansible-roles/compute.ini" + storage_inventory_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/storage.ini" : "${path.root}/modules/ansible-roles/storage.ini" } # locals needed for playbook locals { bastion_fip = module.deployer.bastion_fip - compute_private_key_path = "./../../modules/ansible-roles/compute_id_rsa" #checkov:skip=CKV_SECRET_6 - storage_private_key_path = "./../../modules/ansible-roles/storage_id_rsa" #checkov:skip=CKV_SECRET_6 - compute_playbook_path = "./../../modules/ansible-roles/compute_ssh.yaml" - storage_playbook_path = "./../../modules/ansible-roles/storage_ssh.yaml" + compute_private_key_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/compute_id_rsa" : "${path.root}/modules/ansible-roles/compute_id_rsa" #checkov:skip=CKV_SECRET_6 + storage_private_key_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/storage_id_rsa" : "${path.root}/modules/ansible-roles/storage_id_rsa" #checkov:skip=CKV_SECRET_6 + compute_playbook_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/compute_ssh.yaml" : "${path.root}/modules/ansible-roles/compute_ssh.yaml" + storage_playbook_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/storage_ssh.yaml" : "${path.root}/modules/ansible-roles/storage_ssh.yaml" } # file Share OutPut locals { - fileshare_name_mount_path_map = module.file_storage.name_mount_path_map + fileshare_name_mount_path_map = var.enable_deployer ? {} : module.file_storage[0].name_mount_path_map +} + +# details needed for json file +locals { + json_inventory_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/all.json" : "${path.root}/modules/ansible-roles/all.json" + management_nodes = var.enable_deployer ? [] : (flatten([module.landing_zone_vsi[0].management_vsi_data]))[*]["name"] + compute_nodes = var.enable_deployer ? [] : (flatten([module.landing_zone_vsi[0].compute_vsi_data]))[*]["name"] + client_nodes = var.enable_deployer ? [] : (flatten([module.landing_zone_vsi[0].client_vsi_data]))[*]["name"] + gui_hosts = var.enable_deployer ? [] : [local.management_nodes[0]] # Without Pac HA + db_hosts = var.enable_deployer ? [] : [local.management_nodes[0]] # Without Pac HA + ha_shared_dir = "/mnt/lsf/shared" + nfs_install_dir = "none" + Enable_Monitoring = false + lsf_deployer_hostname = var.deployer_hostname #data.external.get_hostname.result.name #var.enable_bastion ? "" : flatten(module.deployer.deployer_vsi_data[*].list)[0].name } + +locals { + schematics_inputs_path = "/tmp/.schematics/solution_terraform.auto.tfvars.json" + remote_inputs_path = format("%s/terraform.tfvars.json", "/tmp") + deployer_path = "/opt/ibm" + remote_terraform_path = format("%s/terraform-ibm-hpc", local.deployer_path) + remote_ansible_path = format("%s/terraform-ibm-hpc", local.deployer_path) + da_hpc_repo_url = "https://github.com/terraform-ibm-modules/terraform-ibm-hpc.git" + da_hpc_repo_tag = "jay_lsf_scale_deployer" ###### change it to main in future + zones = jsonencode(var.zones) + list_compute_ssh_keys = jsonencode(local.compute_ssh_keys) + list_storage_ssh_keys = jsonencode(local.storage_ssh_keys) + list_storage_instances = jsonencode(var.storage_instances) + list_management_instances = jsonencode(var.management_instances) + list_protocol_instances = jsonencode(var.protocol_instances) + list_compute_instances = jsonencode(var.static_compute_instances) + list_client_instances = jsonencode(var.client_instances) + allowed_cidr = jsonencode(var.allowed_cidr) + list_storage_subnets = jsonencode(length(local.storage_subnet) == 0 ? null : local.storage_subnet) + list_protocol_subnets = jsonencode(length(local.protocol_subnet) == 0 ? null : local.protocol_subnet) + list_compute_subnets = jsonencode(length(local.compute_subnet) == 0 ? null : local.compute_subnet) + list_client_subnets = jsonencode(length(local.client_subnet) == 0 ? null : local.client_subnet) + list_bastion_subnets = jsonencode(length(local.bastion_subnet) == 0 ? null : local.bastion_subnet) + dns_domain_names = jsonencode(var.dns_domain_names) + compute_public_key_content = local.compute_public_key_contents != null ? jsonencode(base64encode(local.compute_public_key_contents)) : "" + compute_private_key_content = local.compute_private_key_contents != null ? jsonencode(base64encode(local.compute_private_key_contents)) : "" +} \ No newline at end of file diff --git a/main.tf b/main.tf index 43846e34..c34e1a7a 100644 --- a/main.tf +++ b/main.tf @@ -1,5 +1,6 @@ module "landing_zone" { source = "./modules/landing_zone" + enable_landing_zone = var.enable_landing_zone allowed_cidr = var.allowed_cidr compute_subnets_cidr = var.compute_subnets_cidr clusters = var.clusters @@ -55,16 +56,23 @@ module "deployer" { boot_volume_encryption_key = local.boot_volume_encryption_key existing_kms_instance_guid = local.existing_kms_instance_guid skip_iam_authorization_policy = var.skip_iam_authorization_policy + static_compute_instances = var.static_compute_instances + management_instances = var.management_instances + dns_domain_names = var.dns_domain_names + } module "landing_zone_vsi" { + count = var.enable_deployer == false ? 1 : 0 source = "./modules/landing_zone_vsi" resource_group = var.resource_group prefix = var.prefix zones = var.zones vpc_id = local.vpc_id - bastion_security_group_id = local.bastion_security_group_id + bastion_security_group_id = var.bastion_security_group_id bastion_public_key_content = local.bastion_public_key_content + compute_public_key_content = var.compute_public_key_content + compute_private_key_content= var.compute_private_key_content client_subnets = local.client_subnets client_ssh_keys = local.client_ssh_keys client_instances = var.client_instances @@ -82,12 +90,126 @@ module "landing_zone_vsi" { dns_domain_names = var.dns_domain_names kms_encryption_enabled = local.kms_encryption_enabled boot_volume_encryption_key = local.boot_volume_encryption_key + enable_bastion = var.enable_bastion +} + + +resource "local_sensitive_file" "prepare_tf_input" { + count = var.enable_deployer == true ? 1 : 0 + content = <[:]/?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT + # + JDBC_string: none + + # (Optional) Primary_LSF_ADMIN is a primary LSF administrator. + # + # The primary LSF administrator account must exist + # on all hosts in the cluster before you install LSF. + # + # Note that the primary LSF administrator is lsfadmin by default. + # if the lsfadmin user does not exist, the installation creates lsfadmin + # with UID 495. The primary administrator owns the LSF configuration files + # and log files for job events. + # + #Primary_LSF_ADMIN: user_name + + # (Optional) Secondary_LSF_ADMINS is a space delimited + # list of secondary LSF administrators. + # + # LSF administrators have permission to reconfigure LSF + # and to control batch jobs that are submitted by other users. + # Secondary administrators typically do not have permission to start LSF + # daemons. Usually, only root has permission to start LSF daemons. + # + # All secondary LSF administrator accounts must exist + # on all hosts in the cluster before you install LSF. + # + #Secondary_LSF_ADMINS: user_name1 user_name2 ... + + # (Optional) LSF_Ports is a space delimited + # list of LSF port numbers. + # + # LSF has the following default port number values listed below. + # All are used for TCP, except LSF_LIM_PORT which also uses UDP. + #LSF_LIM_PORT: 7869 + #LSF_RES_PORT: 6878 + #LSB_SBD_PORT: 6882 + #LSB_MBD_PORT: 6881 + #LSB_QUERY_PORT: 6891 + #LSF_DATA_PORT: 9998 + #EGO_KD_PORT: 7870 + #EGO_PEM_PORT: 7871 + #ESC_PORT: 7872 + # + # Specify either nine individual port numbers or + # a range with the starting port number followed by '-'. + # A port number must be an integer in the range 1024 - 65535 + # except for any of the following: + # 1966, 1967, 1968, 5000, 6080, 8046, 8047, 8048, 8080, 8081, 8443, 8444. + # For example, + # LSF_Ports: 9991 9992 9993 9994 9995 9996 9997 9998 9999 + # LSF_Ports: 9991 - + # where '-' indicates eight consecutive numbers from the leading number + # + #LSF_Ports: port_number1 [ - | port_number2 ... port_number9 ] + + # (Optional) Private_IPv4_Range allows you to specify a range of private IPv4 + # addresses used by LSF hosts. + # This parameter can be used in scenarios where the LSF master host has both + # public and private IP addresses, but the compute nodes have only private IP + # addresses. + # Specify a range of IPv4 addresses in the form of a Classless Inter-Domain + # Routing (CIDR) block. + # For example, Private_IPv4_Range: 10.10.99.0/8 + #Private_IPv4_Range: none + + # The ES_SERVERS parameter sets the list of the Elasticsearch servers where + # Elasticsearch 7.2, or later, is running. + # + # It contains the URLs to which Elasticsearch is listening and enables + # LSF Explorer to use the Elasticsearch features. + # + # Enter a space-separated list of the Elasticserch server URLs in the following format: + # http:// or https:// IP_address or host_name : port number + # For example, + # ES_SERVERS: "http://hostA:9200 http://hostB:9200" + # + # If the first server does not respond, the next server in the list is contacted. + # + #ES_SERVERS: "http://ES_host_ip:9200" + + # ES_SSL_CERTIFICATE_PATH must be configured when + # the protocol Elasticsearch (scheme of the URL) is https. + # The file path must be accessible on the deployer machine. + # For example, + #ES_SSL_CERTIFICATE_PATH: /path/to/cert_file.crt + + # LOGSTASH_TOP is the top directory of the Logstash installation. + # Use this parameter to enable the energy accounting and the gpfsio-collector service. + # When LOGSTASH_TOP is defined, ES_SERVERS must also be defined in lsf-config.yml. + # + # The directory path must point to the top directory of Logstash on a host in the GUI_Hosts role. + # For example, LOGSTASH_TOP: /path/to/logstash/top/directory + # + # If the Logstash path.settings is not set to LOGSTASH_TOP/config, + # make a symbolic link for LOGSTASH_TOP/config to the Logstash path.settings directory. + # For example, + # ln -s /etc/logstash LOGSTASH_TOP/config + # + #LOGSTASH_TOP: none + +# END OF LSF-CONFIG.YML \ No newline at end of file diff --git a/modules/ansible-roles/roles/lsf/templates/lsf-inventory.j2 b/modules/ansible-roles/roles/lsf/templates/lsf-inventory.j2 new file mode 100644 index 00000000..51039957 --- /dev/null +++ b/modules/ansible-roles/roles/lsf/templates/lsf-inventory.j2 @@ -0,0 +1,56 @@ +# LSF Suites Inventory file +[local] +localhost +# Use this file to define the machines there roles +# A machine can belong to more than one role, but +# should only belong to one LSF_xxxxx role. + +# LSF_Masters are the machines LSF will use to run the +# management processes. For HA there must be 2 or more, +# and the shared filesystem must be available. +[LSF_Masters] +{% for host in lsf_masters %} +{{ host }} +{% endfor %} + +# LSF_Servers are machines that LSF will use to run jobs. +# Expressions can be used to represent a number of +# machines e.g. +# host[1:100] == host1, host2, host3, ... host100 +# host[a:f] == hosta, hostb, hostc, ... hostf +[LSF_Servers] +{% for host in lsf_servers %} +{{ host }} +{% endfor %} + +# LSF_Clients are machines that cannot run work, but can +# submit jobs via the CLI, and query the cluster. +# These are optional. +[LSF_Clients] +{% for host in lsf_clients %} +{{ host }} +{% endfor %} + +# GUI_Hosts are machines that will run the GUI and +# other supporting services. A minimum of 1 machine +# needs to be a GUI host. If there is no HA_shared_dir +# then this must be set to the LSF_Masters host. +# Use public (external) host name if the machine has +# multiple NICs. Make sure the machine can be ping-able +# by using both its public IP address and the host +# name reported by the hostname command and vice versa. +[GUI_Hosts] +{% for host in gui_hosts %} +{{ host }} +{% endfor %} + +# DB_HOST is optional, and is the machine that hosts the database +# used by the Application Center component in LSF Suite. +# However, this database is not configured for High Availability (HA). +# To enable HA for this database, manually create the database using MariaDB +# and configure it to be HA-ready, then set the JDBC_string parameter +# in the /opt/ibm/lsf_installer/playbook/lsf-config.yml file to specify the database connection. +[DB_Host] +{% for host in db_hosts %} +{{ host }} +{% endfor %} \ No newline at end of file diff --git a/modules/ansible-roles/roles/lsf/vars/main.yml b/modules/ansible-roles/roles/lsf/vars/main.yml new file mode 100644 index 00000000..1927972f --- /dev/null +++ b/modules/ansible-roles/roles/lsf/vars/main.yml @@ -0,0 +1,6 @@ +--- + +# Static Variables + +inventory_path: "/opt/ibm/lsf_installer/playbook/" +lsf_dir: "{{ HA_shared_dir | basename }}" \ No newline at end of file diff --git a/modules/ansible-roles/roles/prerequisite/tasks/main.yml b/modules/ansible-roles/roles/prerequisite/tasks/main.yml deleted file mode 100644 index 55479781..00000000 --- a/modules/ansible-roles/roles/prerequisite/tasks/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -# tasks file for fileshare mount -- import_tasks: vpc-fileshare-mount.yml \ No newline at end of file diff --git a/modules/ansible-roles/roles/prerequisite/handlers/main.yml b/modules/ansible-roles/roles/vpc_fileshare_configure/handlers/main.yml similarity index 100% rename from modules/ansible-roles/roles/prerequisite/handlers/main.yml rename to modules/ansible-roles/roles/vpc_fileshare_configure/handlers/main.yml diff --git a/modules/ansible-roles/roles/vpc_fileshare_configure/tasks/main.yml b/modules/ansible-roles/roles/vpc_fileshare_configure/tasks/main.yml new file mode 100644 index 00000000..f8f3dbde --- /dev/null +++ b/modules/ansible-roles/roles/vpc_fileshare_configure/tasks/main.yml @@ -0,0 +1,4 @@ +--- + +# tasks file for fileshare mount +- import_tasks: vpc_fileshare_configure.yml \ No newline at end of file diff --git a/modules/ansible-roles/roles/prerequisite/tasks/vpc-fileshare-mount.yml b/modules/ansible-roles/roles/vpc_fileshare_configure/tasks/vpc_fileshare_configure.yml similarity index 93% rename from modules/ansible-roles/roles/prerequisite/tasks/vpc-fileshare-mount.yml rename to modules/ansible-roles/roles/vpc_fileshare_configure/tasks/vpc_fileshare_configure.yml index ca1f1638..aa991f49 100644 --- a/modules/ansible-roles/roles/prerequisite/tasks/vpc-fileshare-mount.yml +++ b/modules/ansible-roles/roles/vpc_fileshare_configure/tasks/vpc_fileshare_configure.yml @@ -20,6 +20,9 @@ with_dict: "{{ name_mount_path_map }}" notify: Mount NFS +- name: Flush handlers immediately + meta: flush_handlers + - name: Verify mounted filesystems command: df -h register: df_output diff --git a/modules/deployer/locals.tf b/modules/deployer/locals.tf index 7c83b9b6..2768edcc 100644 --- a/modules/deployer/locals.tf +++ b/modules/deployer/locals.tf @@ -59,8 +59,8 @@ locals { port_max = 22 } }], - [for cidr in local.bastion_sg_variable_cidr : { - name = format("allow-variable-outbound-%s", index(local.bastion_sg_variable_cidr, cidr) + 1) + [for cidr in concat(local.bastion_sg_variable_cidr, ["0.0.0.0/0"]) : { + name = format("allow-variable-outbound-%s", index(concat(local.bastion_sg_variable_cidr, ["0.0.0.0/0"]), cidr) + 1) direction = "outbound" remote = cidr }] @@ -73,3 +73,13 @@ locals { # Subnets bastion_subnets = var.bastion_subnets } + +locals { + vsi_interfaces = ["eth0", "eth1"] + compute_interfaces = local.vsi_interfaces[0] + compute_dns_domain = var.dns_domain_names["compute"] + + management_instance_count = sum(var.management_instances[*]["count"]) + static_compute_instance_count = sum(var.static_compute_instances[*]["count"]) + enable_compute = local.management_instance_count > 0 || local.static_compute_instance_count > 0 +} \ No newline at end of file diff --git a/modules/deployer/main.tf b/modules/deployer/main.tf index 34c2a8d4..48fb77dd 100644 --- a/modules/deployer/main.tf +++ b/modules/deployer/main.tf @@ -1,3 +1,9 @@ +module "compute_key" { + count = local.enable_deployer && local.enable_compute ? 1 : 0 + source = "./../key" + private_key_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/compute_id_rsa" : "${path.root}/modules/ansible-roles/compute_id_rsa" #checkov:skip=CKV_SECRET_6 +} + module "ssh_key" { count = local.enable_bastion ? 1 : 0 source = "./../key" diff --git a/modules/deployer/outputs.tf b/modules/deployer/outputs.tf index 1e288f77..6ecf495c 100644 --- a/modules/deployer/outputs.tf +++ b/modules/deployer/outputs.tf @@ -26,3 +26,21 @@ output "deployer_ip" { description = "Deployer IP" value = one(module.deployer_vsi[*]["list"][0]["ipv4_address"]) } + +output "bastion_private_key_content" { + description = "Bastion private key content" + sensitive = true + value = one(module.ssh_key[*].private_key_content) +} + +output "compute_public_key_content" { + description = "Compute public key content" + sensitive = true + value = one(module.compute_key[*].public_key_content) +} + +output "compute_private_key_content" { + description = "Compute private key content" + sensitive = true + value = one(module.compute_key[*].private_key_content) +} \ No newline at end of file diff --git a/modules/deployer/template_files.tf b/modules/deployer/template_files.tf index ecbd67a9..7569a41a 100644 --- a/modules/deployer/template_files.tf +++ b/modules/deployer/template_files.tf @@ -8,6 +8,11 @@ data "template_file" "bastion_user_data" { data "template_file" "deployer_user_data" { template = file("${path.module}/templates/deployer_user_data.tpl") vars = { - bastion_public_key_content = local.enable_bastion ? module.ssh_key[0].public_key_content : "" + bastion_public_key_content = local.enable_bastion ? module.ssh_key[0].public_key_content : "" + compute_dns_domain = local.enable_bastion ? local.compute_dns_domain : "" + compute_interfaces = local.enable_bastion ? local.compute_interfaces : "" + compute_public_key_content = local.enable_bastion ? module.compute_key[0].public_key_content : "" + compute_private_key_content = local.enable_bastion ? module.compute_key[0].private_key_content : "" + } } diff --git a/modules/deployer/templates/deployer_user_data.tpl b/modules/deployer/templates/deployer_user_data.tpl index a5782268..5648dba2 100644 --- a/modules/deployer/templates/deployer_user_data.tpl +++ b/modules/deployer/templates/deployer_user_data.tpl @@ -14,32 +14,41 @@ then USER=ubuntu fi sed -i -e "s/^/no-port-forwarding,no-agent-forwarding,no-X11-forwarding,command=\"echo \'Please login as the user \\\\\"$USER\\\\\" rather than the user \\\\\"root\\\\\".\';echo;sleep 5; exit 142\" /" /root/.ssh/authorized_keys +echo "DOMAIN=${compute_dns_domain}" >> "/etc/sysconfig/network-scripts/ifcfg-${compute_interfaces}" +echo "MTU=9000" >> "/etc/sysconfig/network-scripts/ifcfg-${compute_interfaces}" +chage -I -1 -m 0 -M 99999 -E -1 -W 14 vpcuser +120 +systemctl restart NetworkManager # input parameters echo "${bastion_public_key_content}" >> /home/$USER/.ssh/authorized_keys echo "StrictHostKeyChecking no" >> /home/$USER/.ssh/config +echo "StrictHostKeyChecking no" >> ~/.ssh/config +echo "${compute_public_key_content}" >> ~/.ssh/authorized_keys +echo "${compute_private_key_content}" > ~/.ssh/id_rsa +chmod 600 ~/.ssh/id_rsa -# setup env -# TODO: Conditional installation (python3, terraform & ansible) -if grep -E -q "CentOS|Red Hat" /etc/os-release -then - # TODO: Terraform Repo access - #yum install -y yum-utils - #yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo - #if (which terraform); then echo "Terraform exists, skipping the installation"; else (yum install -y terraform - if (which python3); then echo "Python3 exists, skipping the installation"; else (yum install -y python38); fi - if (which ansible-playbook); then echo "Ansible exists, skipping the installation"; else (yum install -y ansible); fi -elif grep -q "Ubuntu" /etc/os-release -then - apt update - # TODO: Terraform Repo access - #apt-get update && sudo apt-get install -y gnupg software-properties-common - #wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg - #gpg --no-default-keyring --keyring /usr/share/keyrings/hashicorp-archive-keyring.gpg --fingerprint - apt install software-properties-common - apt-add-repository --yes --update ppa:ansible/ansible - if (which python3); then echo "Python3 exists, skipping the installation"; else (apt install python38); fi - if (which ansible-playbook); then echo "Ansible exists, skipping the installation"; else (apt install ansible); fi -fi +# # setup env +# # TODO: Conditional installation (python3, terraform & ansible) +# if grep -E -q "CentOS|Red Hat" /etc/os-release +# then +# # TODO: Terraform Repo access +# #yum install -y yum-utils +# #yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo +# #if (which terraform); then echo "Terraform exists, skipping the installation"; else (yum install -y terraform +# if (which python3); then echo "Python3 exists, skipping the installation"; else (yum install -y python38); fi +# if (which ansible-playbook); then echo "Ansible exists, skipping the installation"; else (yum install -y ansible); fi +# elif grep -q "Ubuntu" /etc/os-release +# then +# apt update +# # TODO: Terraform Repo access +# #apt-get update && sudo apt-get install -y gnupg software-properties-common +# #wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg +# #gpg --no-default-keyring --keyring /usr/share/keyrings/hashicorp-archive-keyring.gpg --fingerprint +# apt install software-properties-common +# apt-add-repository --yes --update ppa:ansible/ansible +# if (which python3); then echo "Python3 exists, skipping the installation"; else (apt install python38); fi +# if (which ansible-playbook); then echo "Ansible exists, skipping the installation"; else (apt install ansible); fi +# fi # TODO: run terraform diff --git a/modules/deployer/variables.tf b/modules/deployer/variables.tf index 1c9ba1e2..1b89ae38 100644 --- a/modules/deployer/variables.tf +++ b/modules/deployer/variables.tf @@ -130,3 +130,34 @@ variable "skip_iam_authorization_policy" { default = false description = "Set to false if authorization policy is required for VPC block storage volumes to access kms. This can be set to true if authorization policy already exists. For more information on how to create authorization policy manually, see [creating authorization policies for block storage volume](https://cloud.ibm.com/docs/vpc?topic=vpc-block-s2s-auth&interface=ui)." } + +variable "management_instances" { + type = list( + object({ + profile = string + count = number + image = string + }) + ) + description = "Number of instances to be launched for management." +} + +variable "static_compute_instances" { + type = list( + object({ + profile = string + count = number + image = string + }) + ) + description = "Total Number of instances to be launched for compute cluster." +} + +variable "dns_domain_names" { + type = object({ + compute = string + storage = string + protocol = string + }) + description = "IBM Cloud HPC DNS domain names." +} diff --git a/modules/file_storage/outputs.tf b/modules/file_storage/outputs.tf index c8c27cc3..7f265e54 100644 --- a/modules/file_storage/outputs.tf +++ b/modules/file_storage/outputs.tf @@ -9,4 +9,4 @@ output "mount_path" { output "name_mount_path_map" { description = "Mount path name and its path map" value = { for mount_details in flatten([ibm_is_share_mount_target.share_target_vpc, ibm_is_share_mount_target.share_target_sg]) : split("-", mount_details.name)[length(split("-", mount_details.name)) - 4] => mount_details.mount_path } -} +} \ No newline at end of file diff --git a/modules/inventory/main.tf b/modules/inventory/main.tf index 10c6e56e..b63bf85d 100644 --- a/modules/inventory/main.tf +++ b/modules/inventory/main.tf @@ -7,7 +7,6 @@ resource "local_sensitive_file" "itself" { content = <> ~/.ssh/authorized_keys -echo "${compute_public_key_content}" >> ~/.ssh/authorized_keys +echo "${compute_public_key_content}" | base64 --decode >> ~/.ssh/authorized_keys echo "StrictHostKeyChecking no" >> ~/.ssh/config -echo "${compute_private_key_content}" > ~/.ssh/id_rsa +echo "${compute_private_key_content}" | base64 --decode > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa # network setup diff --git a/modules/landing_zone_vsi/templates/management_user_data.tpl b/modules/landing_zone_vsi/templates/management_user_data.tpl index 8488c692..1db5be80 100644 --- a/modules/landing_zone_vsi/templates/management_user_data.tpl +++ b/modules/landing_zone_vsi/templates/management_user_data.tpl @@ -17,9 +17,9 @@ sed -i -e "s/^/no-port-forwarding,no-agent-forwarding,no-X11-forwarding,command= # input parameters echo "${bastion_public_key_content}" >> ~/.ssh/authorized_keys -echo "${management_public_key_content}" >> ~/.ssh/authorized_keys +echo "${management_public_key_content}" | base64 --decode >> ~/.ssh/authorized_keys echo "StrictHostKeyChecking no" >> ~/.ssh/config -echo "${management_private_key_content}" > ~/.ssh/id_rsa +echo "${management_private_key_content}" | base64 --decode > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa # network setup diff --git a/modules/landing_zone_vsi/variables.tf b/modules/landing_zone_vsi/variables.tf index 5339eb2e..bb965cb1 100644 --- a/modules/landing_zone_vsi/variables.tf +++ b/modules/landing_zone_vsi/variables.tf @@ -298,3 +298,23 @@ variable "boot_volume_encryption_key" { ############################################################################## # TODO: Auth Server (LDAP/AD) Variables ############################################################################## + +variable "compute_public_key_content" { + type = string + sensitive = true + default = null + description = "Compute security key content." +} + +variable "compute_private_key_content" { + type = string + sensitive = true + default = null + description = "Compute security key content." +} + +variable "enable_bastion" { + type = bool + default = true + description = "The solution supports multiple ways to connect to your HPC cluster for example, using bastion node, via VPN or direct connection. If connecting to the HPC cluster via VPN or direct connection, set this value to false." +} \ No newline at end of file diff --git a/modules/observability_instance/datasources.tf b/modules/observability_instance/datasources.tf index 6be8d171..241a7090 100644 --- a/modules/observability_instance/datasources.tf +++ b/modules/observability_instance/datasources.tf @@ -8,6 +8,6 @@ data "http" "sysdig_prws_key" { request_headers = { Accept = "application/json" Authorization = data.ibm_iam_auth_token.tokendata.iam_access_token - IBMInstanceID = var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_guid : "" + IBMInstanceID = var.enable_deployer && var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_guid : "" } } diff --git a/modules/observability_instance/main.tf b/modules/observability_instance/main.tf index 43568f7e..586260a5 100644 --- a/modules/observability_instance/main.tf +++ b/modules/observability_instance/main.tf @@ -5,6 +5,7 @@ locals { } module "observability_instance" { + count = var.enable_deployer ? 1 : 0 # Replace "master" with a GIT release version to lock into a specific release source = "terraform-ibm-modules/observability-instances/ibm" version = "3.3.1" diff --git a/modules/observability_instance/outputs.tf b/modules/observability_instance/outputs.tf index 523a668e..43c89b4e 100644 --- a/modules/observability_instance/outputs.tf +++ b/modules/observability_instance/outputs.tf @@ -1,5 +1,5 @@ output "cloud_monitoring_access_key" { - value = var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_access_key : null + value = var.enable_deployer && var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_access_key : null description = "IBM Cloud Monitoring access key for agents to use" sensitive = true } @@ -10,7 +10,7 @@ output "cloud_monitoring_ingestion_url" { } output "cloud_monitoring_prws_key" { - value = var.cloud_monitoring_provision ? jsondecode(data.http.sysdig_prws_key[0].response_body).token.key : null + value = var.enable_deployer && var.cloud_monitoring_provision ? jsondecode(data.http.sysdig_prws_key[0].response_body).token.key : null description = "IBM Cloud Monitoring Prometheus Remote Write ingestion key" sensitive = true } @@ -31,7 +31,7 @@ output "cloud_logs_ingress_private_endpoint" { } output "cloud_monitoring_url" { - value = var.cloud_monitoring_provision ? "https://cloud.ibm.com/observe/embedded-view/monitoring/${module.observability_instance.cloud_monitoring_guid}" : null + value = var.enable_deployer && var.cloud_monitoring_provision ? "https://cloud.ibm.com/observe/embedded-view/monitoring/${module.observability_instance.cloud_monitoring_guid}" : null description = "IBM Cloud Monitoring URL" } diff --git a/modules/observability_instance/variables.tf b/modules/observability_instance/variables.tf index 25522547..2a80cbbf 100644 --- a/modules/observability_instance/variables.tf +++ b/modules/observability_instance/variables.tf @@ -85,3 +85,9 @@ variable "cloud_metrics_data_bucket" { description = "Generate and store metrics from your events so you can visualize, track, and alert on log events in real-time." type = any } + +variable "enable_deployer" { + type = bool + default = false + description = "Deployer should be only used for better deployment performance" +} \ No newline at end of file diff --git a/modules/playbook/main.tf b/modules/playbook/main.tf index 857e4d44..38c8bd68 100644 --- a/modules/playbook/main.tf +++ b/modules/playbook/main.tf @@ -1,15 +1,19 @@ +locals { + proxyjump = var.enable_bastion ? "-o ProxyJump=ubuntu@${var.bastion_fip}" : "" +} + resource "local_file" "create_playbook" { count = var.inventory_path != null ? 1 : 0 content = < - -o ProxyJump=ubuntu@${var.bastion_fip} + ${local.proxyjump} -o ControlMaster=auto -o ControlPersist=30m -o UserKnownHostsFile=/dev/null @@ -30,15 +34,19 @@ resource "local_file" "create_playbook" { gather_facts: false vars: ansible_ssh_common_args: > - -o ProxyJump=ubuntu@${var.bastion_fip} + ${local.proxyjump} -o ControlMaster=auto -o ControlPersist=30m -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ansible_user: root ansible_ssh_private_key_file: ${var.private_key_path} + pre_tasks: + - name: Load cluster-specific variables + include_vars: all.json roles: - - prerequisite + - vpc_fileshare_configure + - lsf EOT filename = var.playbook_path } @@ -56,15 +64,21 @@ resource "null_resource" "run_playbook" { depends_on = [local_file.create_playbook] } -resource "ansible_playbook" "playbook" { - playbook = var.playbook_path - name = "localhost" - replayable = true - verbosity = 6 - extra_vars = { - ansible_python_interpreter = "auto" - inventory_file = var.inventory_path +resource "null_resource" "run_lsf_playbooks" { + count = var.inventory_path != null ? 1 : 0 + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + command = <