diff --git a/datasource.tf b/datasource.tf index f6fada1e..4f893a4a 100644 --- a/datasource.tf +++ b/datasource.tf @@ -20,3 +20,33 @@ data "ibm_is_subnet" "itself" { identifier = local.subnets[count.index]["id"] } */ + +data "ibm_resource_group" "resource_group" { + count = var.resource_group == null ? 0 : 1 + name = var.resource_group +} + +data "ibm_is_subnet" "existing_compute_subnets" { + count = var.vpc != null && var.compute_subnets != null ? 1 : 0 + name = var.compute_subnets[count.index] +} + +data "ibm_is_subnet" "existing_storage_subnets" { + count = var.vpc != null && var.storage_subnets != null ? 1 : 0 + name = var.storage_subnets[count.index] +} + +data "ibm_is_subnet" "existing_protocol_subnets" { + count = var.vpc != null && var.protocol_subnets != null ? 1 : 0 + name = var.protocol_subnets[count.index] +} + +data "ibm_is_subnet" "existing_client_subnets" { + count = var.vpc != null && var.client_subnets != null ? 1 : 0 + name = var.client_subnets[count.index] +} + +data "ibm_is_subnet" "existing_bastion_subnets" { + count = var.vpc != null && var.bastion_subnets != null ? 1 : 0 + name = var.bastion_subnets[count.index] +} \ No newline at end of file diff --git a/locals.tf b/locals.tf index 537c3d38..535d80b9 100644 --- a/locals.tf +++ b/locals.tf @@ -15,12 +15,13 @@ locals { # locals needed for deployer locals { # dependency: landing_zone -> deployer - vpc_id = var.vpc == null ? one(module.landing_zone.vpc_id) : var.vpc + vpc_id = var.vpc == null ? one(module.landing_zone.vpc_id) : var.vpc_id + vpc = var.vpc == null ? one(module.landing_zone.vpc_name) : var.vpc bastion_subnets = module.landing_zone.bastion_subnets kms_encryption_enabled = var.key_management != null ? true : false boot_volume_encryption_key = var.key_management != null ? one(module.landing_zone.boot_volume_encryption_key)["crn"] : null existing_kms_instance_guid = var.key_management != null ? module.landing_zone.key_management_guid : null - cos_data = module.landing_zone.cos_buckets_data + cos_data = var.enable_bastion ? [] : module.landing_zone.cos_buckets_data # Future use # skip_iam_authorization_policy = true } @@ -29,14 +30,78 @@ locals { # locals needed for landing_zone_vsi locals { # dependency: landing_zone -> deployer -> landing_zone_vsi - bastion_security_group_id = module.deployer.bastion_security_group_id - bastion_public_key_content = module.deployer.bastion_public_key_content + bastion_security_group_id = module.deployer.bastion_security_group_id + bastion_public_key_content = module.deployer.bastion_public_key_content + bastion_private_key_content = module.deployer.bastion_private_key_content + + deployer_hostname = var.enable_bastion ? flatten(module.deployer.deployer_vsi_data[*].list)[0].name : "" + deployer_ip = module.deployer.deployer_ip + + compute_public_key_contents = module.deployer.compute_public_key_content + compute_private_key_contents = module.deployer.compute_private_key_content + + # Existing subnets details + existing_compute_subnets = [ + for subnet in data.ibm_is_subnet.existing_compute_subnets : + { + cidr = subnet.ipv4_cidr_block + id = subnet.id + name = subnet.name + zone = subnet.zone + } + ] + + existing_storage_subnets = [ + for subnet in data.ibm_is_subnet.existing_storage_subnets : + { + cidr = subnet.ipv4_cidr_block + id = subnet.id + name = subnet.name + zone = subnet.zone + } + ] + + existing_protocol_subnets = [ + for subnet in data.ibm_is_subnet.existing_protocol_subnets : + { + cidr = subnet.ipv4_cidr_block + id = subnet.id + name = subnet.name + zone = subnet.zone + } + ] + + existing_client_subnets = [ + for subnet in data.ibm_is_subnet.existing_client_subnets : + { + cidr = subnet.ipv4_cidr_block + id = subnet.id + name = subnet.name + zone = subnet.zone + } + ] + + existing_bastion_subnets = [ + for subnet in data.ibm_is_subnet.existing_bastion_subnets : + { + cidr = subnet.ipv4_cidr_block + id = subnet.id + name = subnet.name + zone = subnet.zone + } + ] # dependency: landing_zone -> landing_zone_vsi - client_subnets = module.landing_zone.client_subnets - compute_subnets = module.landing_zone.compute_subnets - storage_subnets = module.landing_zone.storage_subnets - protocol_subnets = module.landing_zone.protocol_subnets + client_subnets = var.vpc != null && var.client_subnets != null ? local.existing_client_subnets : module.landing_zone.client_subnets + compute_subnets = var.vpc != null && var.compute_subnets != null ? local.existing_compute_subnets : module.landing_zone.compute_subnets + storage_subnets = var.vpc != null && var.storage_subnets != null ? local.existing_storage_subnets : module.landing_zone.storage_subnets + protocol_subnets = var.vpc != null && var.protocol_subnets != null ? local.existing_protocol_subnets : module.landing_zone.protocol_subnets + + storage_subnet = [for subnet in local.storage_subnets : subnet.name] + protocol_subnet = [for subnet in local.protocol_subnets : subnet.name] + compute_subnet = [for subnet in local.compute_subnets : subnet.name] + client_subnet = [for subnet in local.client_subnets : subnet.name] + bastion_subnet = [for subnet in local.bastion_subnets : subnet.name] #boot_volume_encryption_key = var.key_management != null ? one(module.landing_zone.boot_volume_encryption_key)["crn"] : null #skip_iam_authorization_policy = true @@ -52,8 +117,8 @@ locals { #boot_volume_encryption_key = var.key_management != null ? one(module.landing_zone.boot_volume_encryption_key)["crn"] : null # dependency: landing_zone_vsi -> file-share - compute_subnet_id = local.compute_subnets[0].id - compute_security_group_id = module.landing_zone_vsi.compute_sg_id + compute_subnet_id = var.vpc == null && var.compute_subnets == null ? local.compute_subnets[0].id : [for subnet in data.ibm_is_subnet.existing_compute_subnets : subnet.id][0] + compute_security_group_id = var.enable_deployer ? [] : module.landing_zone_vsi[0].compute_sg_id management_instance_count = sum(var.management_instances[*]["count"]) default_share = local.management_instance_count > 0 ? [ { @@ -77,39 +142,65 @@ locals { # locals needed for DNS locals { # dependency: landing_zone -> DNS - resource_group_id = one(values(one(module.landing_zone.resource_group_id))) + resource_group = var.resource_group == null ? "workload-rg" : var.resource_group + resource_group_ids = { + # management_rg = var.resource_group == null ? module.landing_zone.resource_group_id[0]["management-rg"] : one(values(one(module.landing_zone.resource_group_id))) + service_rg = var.resource_group == null ? module.landing_zone.resource_group_id[0]["service-rg"] : data.ibm_resource_group.resource_group[0].id + workload_rg = var.resource_group == null ? module.landing_zone.resource_group_id[0]["workload-rg"] : data.ibm_resource_group.resource_group[0].id + } + # resource_group_id = one(values(one(module.landing_zone.resource_group_id))) vpc_crn = var.vpc == null ? one(module.landing_zone.vpc_crn) : one(data.ibm_is_vpc.itself[*].crn) # TODO: Fix existing subnet logic #subnets_crn = var.vpc == null ? module.landing_zone.subnets_crn : ### + existing_compute_subnet_crns = [for subnet in data.ibm_is_subnet.existing_compute_subnets : subnet.crn] + existing_storage_subnet_crns = [for subnet in data.ibm_is_subnet.existing_storage_subnets : subnet.crn] + existing_protocol_subnet_crns = [for subnet in data.ibm_is_subnet.existing_protocol_subnets : subnet.crn] + existing_client_subnet_crns = [for subnet in data.ibm_is_subnet.existing_client_subnets : subnet.crn] + existing_bastion_subnet_crns = [for subnet in data.ibm_is_subnet.existing_bastion_subnets : subnet.crn] + subnets_crn = concat(local.existing_compute_subnet_crns, local.existing_storage_subnet_crns, local.existing_protocol_subnet_crns, local.existing_client_subnet_crns, local.existing_bastion_subnet_crns) + # subnets_crn = var.vpc == null && var.compute_subnets == null ? module.landing_zone.subnets_crn : concat(local.existing_subnet_crns, module.landing_zone.subnets_crn) #subnets = flatten([local.compute_subnets, local.storage_subnets, local.protocol_subnets]) #subnets_crns = data.ibm_is_subnet.itself[*].crn - subnets_crn = module.landing_zone.subnets_crn + # subnets_crn = module.landing_zone.subnets_crn #boot_volume_encryption_key = var.key_management != null ? one(module.landing_zone.boot_volume_encryption_key)["crn"] : null # dependency: landing_zone_vsi -> file-share } +data "external" "get_hostname" { + program = ["sh", "-c", "echo '{\"name\": \"'$(hostname)'\", \"ipv4_address\": \"'$(hostname -I | awk '{print $1}')'\"}'"] +} + + # locals needed for dns-records locals { # dependency: dns -> dns-records - dns_instance_id = module.dns.dns_instance_id + dns_instance_id = var.enable_deployer ? "" : module.dns[0].dns_instance_id + dns_custom_resolver_id = var.enable_deployer ? "" : module.dns[0].dns_custom_resolver_id + dns_zone_map_list = var.enable_deployer ? [] : module.dns[0].dns_zone_maps compute_dns_zone_id = one(flatten([ - for dns_zone in module.dns.dns_zone_maps : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["compute"] + for dns_zone in local.dns_zone_map_list : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["compute"] ])) storage_dns_zone_id = one(flatten([ - for dns_zone in module.dns.dns_zone_maps : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["storage"] + for dns_zone in local.dns_zone_map_list : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["storage"] ])) protocol_dns_zone_id = one(flatten([ - for dns_zone in module.dns.dns_zone_maps : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["protocol"] + for dns_zone in local.dns_zone_map_list : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["protocol"] ])) # dependency: landing_zone_vsi -> dns-records - compute_instances = flatten([module.landing_zone_vsi.management_vsi_data, module.landing_zone_vsi.compute_vsi_data]) - storage_instances = flatten([module.landing_zone_vsi.storage_vsi_data, module.landing_zone_vsi.protocol_vsi_data]) - protocol_instances = flatten([module.landing_zone_vsi.protocol_vsi_data]) + compute_instances = var.enable_deployer ? [] : flatten([module.landing_zone_vsi[0].management_vsi_data, module.landing_zone_vsi[0].compute_vsi_data]) + storage_instances = var.enable_deployer ? [] : flatten([module.landing_zone_vsi[0].storage_vsi_data, module.landing_zone_vsi[0].protocol_vsi_data]) + protocol_instances = var.enable_deployer ? [] : flatten([module.landing_zone_vsi[0].protocol_vsi_data]) + deployer_instances = [ + { + name = var.deployer_hostname #data.external.get_hostname.result["name"] + ipv4_address = var.deployer_ip #data.external.get_hostname.result["ipv4_address"] + } + ] compute_dns_records = [ - for instance in local.compute_instances : + for instance in concat(local.compute_instances, local.deployer_instances): { name = instance["name"] rdata = instance["ipv4_address"] @@ -133,22 +224,63 @@ locals { # locals needed for inventory locals { - compute_hosts = try([for name in local.compute_instances[*]["name"] : "${name}.${var.dns_domain_names["compute"]}"], []) #local.compute_instances[*]["ipv4_address"] - storage_hosts = try([for name in local.storage_instances[*]["name"] : "${name}.${var.dns_domain_names["storage"]}"], []) #local.storage_instances[*]["ipv4_address"] - compute_inventory_path = "./../../modules/ansible-roles/compute.ini" - storage_inventory_path = "./../../modules/ansible-roles/storage.ini" + compute_hosts = try([for name in local.compute_instances[*]["name"] : "${name}.${var.dns_domain_names["compute"]}"], []) #concat(["${data.external.get_hostname.result["name"]}.${var.dns_domain_names["compute"]}"], try([for name in local.compute_instances[*]["name"] : "${name}.${var.dns_domain_names["compute"]}"], [])) + storage_hosts = try([for name in local.storage_instances[*]["name"] : "${name}.${var.dns_domain_names["storage"]}"], []) + compute_inventory_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/compute.ini" : "${path.root}/modules/ansible-roles/compute.ini" + storage_inventory_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/storage.ini" : "${path.root}/modules/ansible-roles/storage.ini" } # locals needed for playbook locals { bastion_fip = module.deployer.bastion_fip - compute_private_key_path = "./../../modules/ansible-roles/compute_id_rsa" #checkov:skip=CKV_SECRET_6 - storage_private_key_path = "./../../modules/ansible-roles/storage_id_rsa" #checkov:skip=CKV_SECRET_6 - compute_playbook_path = "./../../modules/ansible-roles/compute_ssh.yaml" - storage_playbook_path = "./../../modules/ansible-roles/storage_ssh.yaml" + compute_private_key_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/compute_id_rsa" : "${path.root}/modules/ansible-roles/compute_id_rsa" #checkov:skip=CKV_SECRET_6 + storage_private_key_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/storage_id_rsa" : "${path.root}/modules/ansible-roles/storage_id_rsa" #checkov:skip=CKV_SECRET_6 + compute_playbook_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/compute_ssh.yaml" : "${path.root}/modules/ansible-roles/compute_ssh.yaml" + storage_playbook_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/storage_ssh.yaml" : "${path.root}/modules/ansible-roles/storage_ssh.yaml" } # file Share OutPut locals { - fileshare_name_mount_path_map = module.file_storage.name_mount_path_map + fileshare_name_mount_path_map = var.enable_deployer ? {} : module.file_storage[0].name_mount_path_map +} + +# details needed for json file +locals { + json_inventory_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/all.json" : "${path.root}/modules/ansible-roles/all.json" + management_nodes = var.enable_deployer ? [] : (flatten([module.landing_zone_vsi[0].management_vsi_data]))[*]["name"] + compute_nodes = var.enable_deployer ? [] : (flatten([module.landing_zone_vsi[0].compute_vsi_data]))[*]["name"] + client_nodes = var.enable_deployer ? [] : (flatten([module.landing_zone_vsi[0].client_vsi_data]))[*]["name"] + gui_hosts = var.enable_deployer ? [] : [local.management_nodes[0]] # Without Pac HA + db_hosts = var.enable_deployer ? [] : [local.management_nodes[0]] # Without Pac HA + ha_shared_dir = "/mnt/lsf/shared" + nfs_install_dir = "none" + Enable_Monitoring = false + lsf_deployer_hostname = var.deployer_hostname #data.external.get_hostname.result.name #var.enable_bastion ? "" : flatten(module.deployer.deployer_vsi_data[*].list)[0].name } + +locals { + schematics_inputs_path = "/tmp/.schematics/solution_terraform.auto.tfvars.json" + remote_inputs_path = format("%s/terraform.tfvars.json", "/tmp") + deployer_path = "/opt/ibm" + remote_terraform_path = format("%s/terraform-ibm-hpc", local.deployer_path) + remote_ansible_path = format("%s/terraform-ibm-hpc", local.deployer_path) + da_hpc_repo_url = "https://github.com/terraform-ibm-modules/terraform-ibm-hpc.git" + da_hpc_repo_tag = "jay_lsf_dep_pr" ###### change it to main in future + zones = jsonencode(var.zones) + list_compute_ssh_keys = jsonencode(local.compute_ssh_keys) + list_storage_ssh_keys = jsonencode(local.storage_ssh_keys) + list_storage_instances = jsonencode(var.storage_instances) + list_management_instances = jsonencode(var.management_instances) + list_protocol_instances = jsonencode(var.protocol_instances) + list_compute_instances = jsonencode(var.static_compute_instances) + list_client_instances = jsonencode(var.client_instances) + allowed_cidr = jsonencode(var.allowed_cidr) + list_storage_subnets = jsonencode(length(local.storage_subnet) == 0 ? null : local.storage_subnet) + list_protocol_subnets = jsonencode(length(local.protocol_subnet) == 0 ? null : local.protocol_subnet) + list_compute_subnets = jsonencode(length(local.compute_subnet) == 0 ? null : local.compute_subnet) + list_client_subnets = jsonencode(length(local.client_subnet) == 0 ? null : local.client_subnet) + list_bastion_subnets = jsonencode(length(local.bastion_subnet) == 0 ? null : local.bastion_subnet) + dns_domain_names = jsonencode(var.dns_domain_names) + compute_public_key_content = local.compute_public_key_contents != null ? jsonencode(base64encode(local.compute_public_key_contents)) : "" + compute_private_key_content = local.compute_private_key_contents != null ? jsonencode(base64encode(local.compute_private_key_contents)) : "" +} \ No newline at end of file diff --git a/main.tf b/main.tf index 43846e34..c34e1a7a 100644 --- a/main.tf +++ b/main.tf @@ -1,5 +1,6 @@ module "landing_zone" { source = "./modules/landing_zone" + enable_landing_zone = var.enable_landing_zone allowed_cidr = var.allowed_cidr compute_subnets_cidr = var.compute_subnets_cidr clusters = var.clusters @@ -55,16 +56,23 @@ module "deployer" { boot_volume_encryption_key = local.boot_volume_encryption_key existing_kms_instance_guid = local.existing_kms_instance_guid skip_iam_authorization_policy = var.skip_iam_authorization_policy + static_compute_instances = var.static_compute_instances + management_instances = var.management_instances + dns_domain_names = var.dns_domain_names + } module "landing_zone_vsi" { + count = var.enable_deployer == false ? 1 : 0 source = "./modules/landing_zone_vsi" resource_group = var.resource_group prefix = var.prefix zones = var.zones vpc_id = local.vpc_id - bastion_security_group_id = local.bastion_security_group_id + bastion_security_group_id = var.bastion_security_group_id bastion_public_key_content = local.bastion_public_key_content + compute_public_key_content = var.compute_public_key_content + compute_private_key_content= var.compute_private_key_content client_subnets = local.client_subnets client_ssh_keys = local.client_ssh_keys client_instances = var.client_instances @@ -82,12 +90,126 @@ module "landing_zone_vsi" { dns_domain_names = var.dns_domain_names kms_encryption_enabled = local.kms_encryption_enabled boot_volume_encryption_key = local.boot_volume_encryption_key + enable_bastion = var.enable_bastion +} + + +resource "local_sensitive_file" "prepare_tf_input" { + count = var.enable_deployer == true ? 1 : 0 + content = < + ansible-playbook -f 32 -i lsf-inventory lsf-config-test.yml > config-test.log 2>&1 + args: + chdir: "{{ playbook_path }}" + register: lsf_config_test_result + changed_when: false + failed_when: lsf_config_test_result.rc != 0 + ignore_errors: no + delegate_to: "{{ lsf_deployer_hostname }}.comp.com" + async: 3600 + poll: 5 + +- name: Run LSF Predeploy Test + ansible.builtin.shell: > + ansible-playbook -f 32 -i lsf-inventory lsf-predeploy-test.yml >> predeploy.log 2>&1 + args: + chdir: "{{ playbook_path }}" + register: lsf_predeploy_test_result + changed_when: false + failed_when: lsf_predeploy_test_result.rc != 0 + ignore_errors: no + delegate_to: "{{ lsf_deployer_hostname }}.comp.com" + async: 3600 + poll: 5 + +- name: Run LSF Deploy + ansible.builtin.shell: > + ansible-playbook -f 32 -i lsf-inventory lsf-deploy.yml >> deploy.log 2>&1 + args: + chdir: "{{ playbook_path }}" + register: lsf_deploy_test_result + changed_when: false + failed_when: lsf_deploy_test_result.rc != 0 + ignore_errors: no + delegate_to: "{{ lsf_deployer_hostname }}.comp.com" + async: 3600 + poll: 5 \ No newline at end of file diff --git a/modules/ansible-roles/roles/lsf-ansible-command/tasks/lsf-shared-dir.yml b/modules/ansible-roles/roles/lsf-ansible-command/tasks/lsf-shared-dir.yml new file mode 100644 index 00000000..9eeee896 --- /dev/null +++ b/modules/ansible-roles/roles/lsf-ansible-command/tasks/lsf-shared-dir.yml @@ -0,0 +1,11 @@ +--- + +- name: Display the last directory + debug: + msg: "The last directory is: {{ lsf_dir }}" + +- name: Create base directories for lsf configuration + file: + path: "/mnt/lsf/{{ lsf_dir }}" + state: directory + mode: '0777' \ No newline at end of file diff --git a/modules/ansible-roles/roles/lsf-ansible-command/tasks/main.yml b/modules/ansible-roles/roles/lsf-ansible-command/tasks/main.yml new file mode 100644 index 00000000..bccd4173 --- /dev/null +++ b/modules/ansible-roles/roles/lsf-ansible-command/tasks/main.yml @@ -0,0 +1,7 @@ +--- + +# tasks file for lsf shared directory +- import_tasks: lsf-shared-dir.yml + +# # tasks file for lsf commands +# - import_tasks: lsf-ansible-command.yml \ No newline at end of file diff --git a/modules/ansible-roles/roles/lsf-ansible-command/vars/main.yml b/modules/ansible-roles/roles/lsf-ansible-command/vars/main.yml new file mode 100644 index 00000000..f3e0e9e9 --- /dev/null +++ b/modules/ansible-roles/roles/lsf-ansible-command/vars/main.yml @@ -0,0 +1,6 @@ +--- + +# playbook_path: "/Users/jayesh/scale_version/lsf_installer/playbook/" +playbook_path: "/opt/ibm/lsf_installer/playbook/" + +lsf_dir: "{{ HA_shared_dir | basename }}" \ No newline at end of file diff --git a/modules/ansible-roles/roles/lsf_templates/tasks/lsf-all.yml b/modules/ansible-roles/roles/lsf_templates/tasks/lsf-all.yml new file mode 100644 index 00000000..a9c17687 --- /dev/null +++ b/modules/ansible-roles/roles/lsf_templates/tasks/lsf-all.yml @@ -0,0 +1,11 @@ +--- + +# Update deployer_hostname in group_vars/all + +- name: Update deployer_hostname in group_vars/all + ansible.builtin.lineinfile: + path: "{{ inventory_path }}/group_vars/all" + regexp: '^deployer_hostname: .*' + line: 'deployer_hostname: {{ lsf_deployer_hostname }}' + # backup: yes + delegate_to: "{{ lsf_deployer_hostname }}.comp.com" \ No newline at end of file diff --git a/modules/ansible-roles/roles/lsf_templates/tasks/main.yml b/modules/ansible-roles/roles/lsf_templates/tasks/main.yml new file mode 100644 index 00000000..a8283183 --- /dev/null +++ b/modules/ansible-roles/roles/lsf_templates/tasks/main.yml @@ -0,0 +1,10 @@ +--- + +# tasks file for template tasks +- import_tasks: template-tasks.yml + +# tasks file for restarting on the networkmanager on deployer node +- import_tasks: restart-networkmanager.yml + +# update deployer_hostname in group_vars/all +- import_tasks: lsf-all.yml \ No newline at end of file diff --git a/modules/ansible-roles/roles/lsf_templates/tasks/restart-networkmanager.yml b/modules/ansible-roles/roles/lsf_templates/tasks/restart-networkmanager.yml new file mode 100644 index 00000000..4f645590 --- /dev/null +++ b/modules/ansible-roles/roles/lsf_templates/tasks/restart-networkmanager.yml @@ -0,0 +1,9 @@ +--- + +# Restart the NetworkManager on deployer node +- name: Restart NetworkManager + ansible.builtin.systemd: + name: NetworkManager + state: restarted + enabled: yes + delegate_to: "{{ lsf_deployer_hostname }}.comp.com" \ No newline at end of file diff --git a/modules/ansible-roles/roles/lsf_templates/tasks/template-tasks.yml b/modules/ansible-roles/roles/lsf_templates/tasks/template-tasks.yml new file mode 100644 index 00000000..11e74fa6 --- /dev/null +++ b/modules/ansible-roles/roles/lsf_templates/tasks/template-tasks.yml @@ -0,0 +1,24 @@ +--- + +# tasks file for lsf_templates + +- name: Create inventory file from template + template: + src: lsf-inventory.j2 + dest: "{{ inventory_path }}/lsf-inventory" + # delegate_to: localhost + delegate_to: "{{ lsf_deployer_hostname }}.comp.com" + +- name: Create config file from template + template: + src: lsf-config.j2 + dest: "{{ inventory_path }}/lsf-config.yml" + # delegate_to: localhost + delegate_to: "{{ lsf_deployer_hostname }}.comp.com" + +# - name: Create all variable file from template +# template: +# src: lsf-all.j2 +# dest: "{{ inventory_path }}/group_vars/all" +# # delegate_to: localhost +# delegate_to: "{{ lsf_deployer_hostname }}.comp.com" \ No newline at end of file diff --git a/modules/ansible-roles/roles/lsf_templates/templates/lsf-all.j2 b/modules/ansible-roles/roles/lsf_templates/templates/lsf-all.j2 new file mode 100644 index 00000000..14c581a9 --- /dev/null +++ b/modules/ansible-roles/roles/lsf_templates/templates/lsf-all.j2 @@ -0,0 +1,141 @@ +deployer_hostname: {{ lsf_deployer_hostname }} + +PRODUCT_BRAND_NAME: IBM Spectrum +PRODUCT_HELP_URL: 'https://www.ibm.com/support/knowledgecenter/SSZUDP_10.2.0/suite_welcome/lsf_suite_hpc_welcome.html' +LSF_SUITE_EDITION: LSF Suite for HPC +LSF_SUITE_VERSION: 10.2.0.14 + +# The NFS installation can install the LSF_Masters on the local disks +# of the machines, except for work and conf directories which will be +# on NFS. Set this to: true to install the LSF rpms on the LSF_Masters. +# This allows independent upgrading of a LSF master candidates. +# Default: false +LSF_MASTERS_ON_LOCAL: false + +# The JDBC driver jar file to use to connect to +# the data base +JDBC_Driver: /usr/share/java/mysql-connector-java.jar + +CLEANUP_FILES: true +# The max number of backup files per configuration file change +MAX_CONF_BACKUP_FILES: 2 +# Enable floating clients +ENABLE_FLOATING_CLIENTS: false +# Enable static clients +ENABLE_STATIC_CLIENTS: true + +PACKAGES: + GUI: + es: [ ] + kibana: [ ] + gpfsio: [ ] + beats: [ filebeat-for-lsf, metricbeat-for-lsf, lsf-beat ] + execbeat: [ execbeat-for-lsf ] + pm: [ lsf-pm-server ] + framework: [ lsf-perf, lsf-gui ] + explorer: [ lsf-monitor, lsf-explorer-server ] + pac: [ lsf-appcenter ] + LSF_Master: + base: [ lsf-ego-master, lsf-master, lsf-ls-server, lsf-release, lsf-resource-connector ] + dm: [ lsf-data-mgr ] + LSF_Server: + base: [ lsf-ego-server, lsf-server, lsf-integrations, lsf-devel, lsf-man-pages ] + smpi: [] + smpi_misc: [] + pmpi: [] + nfs: [ lsf-nfs-support ] + LSF_Client: + base: [ lsf-conf, lsf-client, lsf-ls-client ] + pm: [ lsf-pm-client ] + dm: [ lsf-data-mgr ] + Ubuntu_support: [ lsf-ubuntu-support ] + DB: + current: [ MariaDB-server, MariaDB-client ] + previous: [ mariadb-server, mariadb ] + +# NOTE: The packages to remove are not the same as what you need to +# install, because they are dependencies of what was installed +UNPACKAGES: + GUI: + explorer: [ lsf-monitor, lsf-explorer-server ] + pac: [ lsf-appcenter ] + base: [ lsf-perf, lsf-gui, logstash-for-lsf, elasticsearch-for-lsf, lsf-gui ,lsf-pm-server ,lsf-pm-client ] + kibana: [ kibana-for-lsf ] + elk: [ elasticsearch-for-lsf, logstash-for-lsf, kibana-for-lsf ] + beats: [ filebeat-for-lsf, metricbeat-for-lsf, lsf-beat, gpfsio-collector-for-lsf, execbeat-for-lsf ] + jre: [ ibm-jre ] + LSF_Master: [ lsf-ego-master, lsf-master, lsf-ls-client, lsf-ls-server, lsf-release, lsf-resource-connector ,lsf-data-mgr ] + LSF_Server: + base: [ lsf-conf, lsf-ego-server, lsf-server, lsf-integrations, lsf-devel, lsf-man-pages ,ibm_smpi_lic_s, ibm_smpi ,lsf-pmpi-hpc, lsf-nfs-support ] + smpi_misc: [ ibm_smpi-devel, ibm_smpi_gpusupport, ibm_smpi-libgpump, ibm_smpi-pami_devel, ibm_spindle, ibm_smpi_mpipsupport ] + LSF_Client: + base: [ lsf-ls-client, lsf-man-pages ] + pm: [ lsf-pm-client ] + dm: [ lsf-data-mgr ] + conf: [ lsf-conf ] + DB: [ MariaDB-server, MariaDB-client, MariaDB-libs, MariaDB-common, MariaDB-shared, mariadb-server, mariadb ] + Ubuntu_support: [ lsf-ubuntu-support ] + +LSF_REPO_DIR: /var/www/html/lsf_suite_pkgs +LSF_REPO_DIR_SLES: /srv/www/htdocs/lsf_suite_pkgs +LSF_INSTALLER_DIR: /opt/ibm/lsf_installer +LSF_SUITE_TOP: {{ LSF_SUITE_TOP }} +LSF_TOP: "{{ LSF_SUITE_TOP }}/{{ LSF_TOP_FOLDER }}" +LSF_CONF_DIR: "{{ LSF_SUITE_TOP }}/{{ LSF_TOP_FOLDER }}/conf" +LSF_WORK_DIR: "{{ LSF_SUITE_TOP }}/{{ LSF_TOP_FOLDER }}/work" +LSF_LOG_DIR: /opt/ibm/lsflogs + +LSF_SWIDTAG: "{{ LSF_SUITE_TOP }}/properties/version/" + +# EXT +EXT_TOP: {{ EXT_TOP }} +# Process Manager +PM_TOP: "{{ EXT_TOP }}/{{ PM_TOP_FOLDER }}" +PM_CONF_DIR: "{{ EXT_TOP }}/{{ PM_TOP_FOLDER }}/conf" +PM_WORK_DIR: "{{ EXT_TOP }}/{{ PM_TOP_FOLDER }}/work" + +# PERF +PERF_TOP: "{{ EXT_TOP }}/{{ PERF_TOP_FOLDER }}" +PERF_CONF_DIR: "{{ EXT_TOP }}/{{ PERF_TOP_FOLDER }}/conf" +PERF_WORK_DIR: "{{ EXT_TOP }}/{{ PERF_TOP_FOLDER }}/work" +# GUI +GUI_TOP: "{{ EXT_TOP }}/{{ GUI_TOP_FOLDER }}" +GUI_CONF_DIR: "{{ EXT_TOP }}/{{ GUI_TOP_FOLDER }}/conf" +GUI_WORK_DIR: "{{ EXT_TOP }}/{{ GUI_TOP_FOLDER }}/work" +# Elasticsearch +ELK_TOP: {{ ELK_TOP }} +ES_TOP: "{{ ELK_TOP }}/elasticsearch" +# Logstash +LOGSTASH_TOP: "{{ ELK_TOP }}/logstash" +# Kibana +KIBANA_TOP: "{{ ELK_TOP }}/kibana" +# Filebeat +FILE_BEAT_TOP: "{{ ELK_TOP }}/filebeat" +# Metricbeat +METRIC_BEAT_TOP: "{{ ELK_TOP }}/metricbeat" +# Lsfbeat +LSF_BEAT_TOP: "{{ ELK_TOP }}/lsfbeat" +# GPFSIO +GPFSIO_TOP: "{{ ELK_TOP }}/gpfsio" +# execbeat +EXEC_BEAT_TOP: "{{ ELK_TOP }}/execbeat" + +# Explorer +EXPLORER_TOP: "{{ EXT_TOP }}/{{ EXPLORER_TOP_FOLDER }}" +EXPLORER_CONF_DIR: "{{ EXT_TOP }}/{{ EXPLORER_TOP_FOLDER }}/server/config" + +# MPI +SMPI_TOP: /opt/ibm/spectrum_mpi + +# MariaDB +mariadb_version: 10.0 +# profile directory path +etc_profile_d: "/etc/profile.d" + +# entitlement files +lsf_entitlement_file : [ lsf.entitlement ] +ls_entitlement_file : [ ] + +# swidtag files +lsf_master_swidtag_file : [ ibm.com_IBM_Spectrum_LSF_Suite_for_HPC-10.2.0.swidtag ] +lsf_server_swidtag_file : ibm.com_IBM_Spectrum_LSF_Suite_Servers_Capacity-10.2.0.swidtag \ No newline at end of file diff --git a/modules/ansible-roles/roles/lsf_templates/templates/lsf-config.j2 b/modules/ansible-roles/roles/lsf_templates/templates/lsf-config.j2 new file mode 100644 index 00000000..1aee163c --- /dev/null +++ b/modules/ansible-roles/roles/lsf_templates/templates/lsf-config.j2 @@ -0,0 +1,162 @@ +--- +#----------------------------------- +# Copyright IBM Corp. 1992, 2017. All rights reserved. +# US Government Users Restricted Rights - Use, duplication or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. +#----------------------------------- + +LSF: + # Set my_cluster_name to the name of the cluster. + my_cluster_name: {{ my_cluster_name }} + + # Enable_Monitoring controls whether + # system monitoring functions are turned on. + # Set to False to disable monitoring or True + # to enable monitoring. + Enable_Monitoring: {{ Enable_Monitoring }} + + # HA_shared_dir is the shared directory for High-Availability (HA). + # If set to a directory, the installation copies + # the configuration files and work directory + # contents to the specified directory, and updates the configuration + # to point to it. + # Set to "none" if an HA shared directory is not used, + # or if you use an NFS shared directory defined with + # NFS_install_dir below. + # HA_shared_dir: /shared/directory/path + HA_shared_dir: {{ HA_shared_dir }} + + # NFS_install_dir is a shared directory that contains + # the LSF master, server and client binary files, man pages, + # and configuration files. When defined, the directory + # is also used for HA, so HA_shared_dir above + # should be set to "none". + # NFS_install_dir: /nfs-or-gpfs-directory + # Optionally the LSF_Masters may be installed locally, \ + # to permit upgrading of LSF Masters independently \ + # of the rest of the cluster. This is controlled \ + # by the LSF_MASTERS_ON_LOCAL parameter in: \ + # /opt/ibm/lsf_installer/playbook/group_vars/all + NFS_install_dir: {{ NFS_install_dir }} + + # JDBC_string is the connection string for the + # optional external database. If a host has + # been given the role of "database" in the + # lsf-inventory file, the specified value is set + # automatically. If you do not specify a host in the DB_Host role + # in the lsf-inventory file, you must define a JDBC_string. + # The external database must be created + # by sourcing the database schema files: + # - Copy the /opt/ibm/lsf_installer/DBschema/MySQL/*.sql files on the deployer machine to the remote database host + # - Create a database that is named pac + # - Create a database user who is named pacuser + # - Grant this user all privileges on the pac database + # - Run the schema files for the database + # For more information, see + # "https://www.ibm.com/support/knowledgecenter/SSZRJV_10.2.0/install_guide/pac_install_config.html". + # Set the following environment variables on the deployer machine: + # - JDBC_USER + # - JDBC_PASSWORD + # + #JDBC_string: jdbc:mariadb://[:]/?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT + # + JDBC_string: none + + # (Optional) Primary_LSF_ADMIN is a primary LSF administrator. + # + # The primary LSF administrator account must exist + # on all hosts in the cluster before you install LSF. + # + # Note that the primary LSF administrator is lsfadmin by default. + # if the lsfadmin user does not exist, the installation creates lsfadmin + # with UID 495. The primary administrator owns the LSF configuration files + # and log files for job events. + # + #Primary_LSF_ADMIN: user_name + + # (Optional) Secondary_LSF_ADMINS is a space delimited + # list of secondary LSF administrators. + # + # LSF administrators have permission to reconfigure LSF + # and to control batch jobs that are submitted by other users. + # Secondary administrators typically do not have permission to start LSF + # daemons. Usually, only root has permission to start LSF daemons. + # + # All secondary LSF administrator accounts must exist + # on all hosts in the cluster before you install LSF. + # + #Secondary_LSF_ADMINS: user_name1 user_name2 ... + + # (Optional) LSF_Ports is a space delimited + # list of LSF port numbers. + # + # LSF has the following default port number values listed below. + # All are used for TCP, except LSF_LIM_PORT which also uses UDP. + #LSF_LIM_PORT: 7869 + #LSF_RES_PORT: 6878 + #LSB_SBD_PORT: 6882 + #LSB_MBD_PORT: 6881 + #LSB_QUERY_PORT: 6891 + #LSF_DATA_PORT: 9998 + #EGO_KD_PORT: 7870 + #EGO_PEM_PORT: 7871 + #ESC_PORT: 7872 + # + # Specify either nine individual port numbers or + # a range with the starting port number followed by '-'. + # A port number must be an integer in the range 1024 - 65535 + # except for any of the following: + # 1966, 1967, 1968, 5000, 6080, 8046, 8047, 8048, 8080, 8081, 8443, 8444. + # For example, + # LSF_Ports: 9991 9992 9993 9994 9995 9996 9997 9998 9999 + # LSF_Ports: 9991 - + # where '-' indicates eight consecutive numbers from the leading number + # + #LSF_Ports: port_number1 [ - | port_number2 ... port_number9 ] + + # (Optional) Private_IPv4_Range allows you to specify a range of private IPv4 + # addresses used by LSF hosts. + # This parameter can be used in scenarios where the LSF master host has both + # public and private IP addresses, but the compute nodes have only private IP + # addresses. + # Specify a range of IPv4 addresses in the form of a Classless Inter-Domain + # Routing (CIDR) block. + # For example, Private_IPv4_Range: 10.10.99.0/8 + #Private_IPv4_Range: none + + # The ES_SERVERS parameter sets the list of the Elasticsearch servers where + # Elasticsearch 7.2, or later, is running. + # + # It contains the URLs to which Elasticsearch is listening and enables + # LSF Explorer to use the Elasticsearch features. + # + # Enter a space-separated list of the Elasticserch server URLs in the following format: + # http:// or https:// IP_address or host_name : port number + # For example, + # ES_SERVERS: "http://hostA:9200 http://hostB:9200" + # + # If the first server does not respond, the next server in the list is contacted. + # + #ES_SERVERS: "http://ES_host_ip:9200" + + # ES_SSL_CERTIFICATE_PATH must be configured when + # the protocol Elasticsearch (scheme of the URL) is https. + # The file path must be accessible on the deployer machine. + # For example, + #ES_SSL_CERTIFICATE_PATH: /path/to/cert_file.crt + + # LOGSTASH_TOP is the top directory of the Logstash installation. + # Use this parameter to enable the energy accounting and the gpfsio-collector service. + # When LOGSTASH_TOP is defined, ES_SERVERS must also be defined in lsf-config.yml. + # + # The directory path must point to the top directory of Logstash on a host in the GUI_Hosts role. + # For example, LOGSTASH_TOP: /path/to/logstash/top/directory + # + # If the Logstash path.settings is not set to LOGSTASH_TOP/config, + # make a symbolic link for LOGSTASH_TOP/config to the Logstash path.settings directory. + # For example, + # ln -s /etc/logstash LOGSTASH_TOP/config + # + #LOGSTASH_TOP: none + +# END OF LSF-CONFIG.YML \ No newline at end of file diff --git a/modules/ansible-roles/roles/lsf_templates/templates/lsf-inventory.j2 b/modules/ansible-roles/roles/lsf_templates/templates/lsf-inventory.j2 new file mode 100644 index 00000000..51039957 --- /dev/null +++ b/modules/ansible-roles/roles/lsf_templates/templates/lsf-inventory.j2 @@ -0,0 +1,56 @@ +# LSF Suites Inventory file +[local] +localhost +# Use this file to define the machines there roles +# A machine can belong to more than one role, but +# should only belong to one LSF_xxxxx role. + +# LSF_Masters are the machines LSF will use to run the +# management processes. For HA there must be 2 or more, +# and the shared filesystem must be available. +[LSF_Masters] +{% for host in lsf_masters %} +{{ host }} +{% endfor %} + +# LSF_Servers are machines that LSF will use to run jobs. +# Expressions can be used to represent a number of +# machines e.g. +# host[1:100] == host1, host2, host3, ... host100 +# host[a:f] == hosta, hostb, hostc, ... hostf +[LSF_Servers] +{% for host in lsf_servers %} +{{ host }} +{% endfor %} + +# LSF_Clients are machines that cannot run work, but can +# submit jobs via the CLI, and query the cluster. +# These are optional. +[LSF_Clients] +{% for host in lsf_clients %} +{{ host }} +{% endfor %} + +# GUI_Hosts are machines that will run the GUI and +# other supporting services. A minimum of 1 machine +# needs to be a GUI host. If there is no HA_shared_dir +# then this must be set to the LSF_Masters host. +# Use public (external) host name if the machine has +# multiple NICs. Make sure the machine can be ping-able +# by using both its public IP address and the host +# name reported by the hostname command and vice versa. +[GUI_Hosts] +{% for host in gui_hosts %} +{{ host }} +{% endfor %} + +# DB_HOST is optional, and is the machine that hosts the database +# used by the Application Center component in LSF Suite. +# However, this database is not configured for High Availability (HA). +# To enable HA for this database, manually create the database using MariaDB +# and configure it to be HA-ready, then set the JDBC_string parameter +# in the /opt/ibm/lsf_installer/playbook/lsf-config.yml file to specify the database connection. +[DB_Host] +{% for host in db_hosts %} +{{ host }} +{% endfor %} \ No newline at end of file diff --git a/modules/ansible-roles/roles/lsf_templates/vars/main.yml b/modules/ansible-roles/roles/lsf_templates/vars/main.yml new file mode 100644 index 00000000..746847f8 --- /dev/null +++ b/modules/ansible-roles/roles/lsf_templates/vars/main.yml @@ -0,0 +1,13 @@ +--- + +# Static Variables + +inventory_path: "/opt/ibm/lsf_installer/playbook/" +LSF_SUITE_TOP: "/opt/ibm/lsfsuite" +LSF_TOP_FOLDER: "lsf" +EXT_TOP: "/opt/ibm/lsfsuite/ext" +PM_TOP_FOLDER: "ppm" +PERF_TOP_FOLDER: "perf" +GUI_TOP_FOLDER: "gui" +ELK_TOP: "/opt/ibm/elastic" +EXPLORER_TOP_FOLDER: "explorer" \ No newline at end of file diff --git a/modules/ansible-roles/roles/prerequisite/tasks/vpc-fileshare-mount.yml b/modules/ansible-roles/roles/prerequisite/tasks/vpc-fileshare-mount.yml index ca1f1638..aa991f49 100644 --- a/modules/ansible-roles/roles/prerequisite/tasks/vpc-fileshare-mount.yml +++ b/modules/ansible-roles/roles/prerequisite/tasks/vpc-fileshare-mount.yml @@ -20,6 +20,9 @@ with_dict: "{{ name_mount_path_map }}" notify: Mount NFS +- name: Flush handlers immediately + meta: flush_handlers + - name: Verify mounted filesystems command: df -h register: df_output diff --git a/modules/deployer/locals.tf b/modules/deployer/locals.tf index 7c83b9b6..2b84e393 100644 --- a/modules/deployer/locals.tf +++ b/modules/deployer/locals.tf @@ -73,3 +73,13 @@ locals { # Subnets bastion_subnets = var.bastion_subnets } + +locals { + vsi_interfaces = ["eth0", "eth1"] + compute_interfaces = local.vsi_interfaces[0] + compute_dns_domain = var.dns_domain_names["compute"] + + management_instance_count = sum(var.management_instances[*]["count"]) + static_compute_instance_count = sum(var.static_compute_instances[*]["count"]) + enable_compute = local.management_instance_count > 0 || local.static_compute_instance_count > 0 +} diff --git a/modules/deployer/main.tf b/modules/deployer/main.tf index 34c2a8d4..48fb77dd 100644 --- a/modules/deployer/main.tf +++ b/modules/deployer/main.tf @@ -1,3 +1,9 @@ +module "compute_key" { + count = local.enable_deployer && local.enable_compute ? 1 : 0 + source = "./../key" + private_key_path = var.enable_bastion ? "${path.root}/../../modules/ansible-roles/compute_id_rsa" : "${path.root}/modules/ansible-roles/compute_id_rsa" #checkov:skip=CKV_SECRET_6 +} + module "ssh_key" { count = local.enable_bastion ? 1 : 0 source = "./../key" diff --git a/modules/deployer/outputs.tf b/modules/deployer/outputs.tf index 1e288f77..6ecf495c 100644 --- a/modules/deployer/outputs.tf +++ b/modules/deployer/outputs.tf @@ -26,3 +26,21 @@ output "deployer_ip" { description = "Deployer IP" value = one(module.deployer_vsi[*]["list"][0]["ipv4_address"]) } + +output "bastion_private_key_content" { + description = "Bastion private key content" + sensitive = true + value = one(module.ssh_key[*].private_key_content) +} + +output "compute_public_key_content" { + description = "Compute public key content" + sensitive = true + value = one(module.compute_key[*].public_key_content) +} + +output "compute_private_key_content" { + description = "Compute private key content" + sensitive = true + value = one(module.compute_key[*].private_key_content) +} \ No newline at end of file diff --git a/modules/deployer/template_files.tf b/modules/deployer/template_files.tf index ecbd67a9..7569a41a 100644 --- a/modules/deployer/template_files.tf +++ b/modules/deployer/template_files.tf @@ -8,6 +8,11 @@ data "template_file" "bastion_user_data" { data "template_file" "deployer_user_data" { template = file("${path.module}/templates/deployer_user_data.tpl") vars = { - bastion_public_key_content = local.enable_bastion ? module.ssh_key[0].public_key_content : "" + bastion_public_key_content = local.enable_bastion ? module.ssh_key[0].public_key_content : "" + compute_dns_domain = local.enable_bastion ? local.compute_dns_domain : "" + compute_interfaces = local.enable_bastion ? local.compute_interfaces : "" + compute_public_key_content = local.enable_bastion ? module.compute_key[0].public_key_content : "" + compute_private_key_content = local.enable_bastion ? module.compute_key[0].private_key_content : "" + } } diff --git a/modules/deployer/templates/deployer_user_data.tpl b/modules/deployer/templates/deployer_user_data.tpl index a5782268..5648dba2 100644 --- a/modules/deployer/templates/deployer_user_data.tpl +++ b/modules/deployer/templates/deployer_user_data.tpl @@ -14,32 +14,41 @@ then USER=ubuntu fi sed -i -e "s/^/no-port-forwarding,no-agent-forwarding,no-X11-forwarding,command=\"echo \'Please login as the user \\\\\"$USER\\\\\" rather than the user \\\\\"root\\\\\".\';echo;sleep 5; exit 142\" /" /root/.ssh/authorized_keys +echo "DOMAIN=${compute_dns_domain}" >> "/etc/sysconfig/network-scripts/ifcfg-${compute_interfaces}" +echo "MTU=9000" >> "/etc/sysconfig/network-scripts/ifcfg-${compute_interfaces}" +chage -I -1 -m 0 -M 99999 -E -1 -W 14 vpcuser +120 +systemctl restart NetworkManager # input parameters echo "${bastion_public_key_content}" >> /home/$USER/.ssh/authorized_keys echo "StrictHostKeyChecking no" >> /home/$USER/.ssh/config +echo "StrictHostKeyChecking no" >> ~/.ssh/config +echo "${compute_public_key_content}" >> ~/.ssh/authorized_keys +echo "${compute_private_key_content}" > ~/.ssh/id_rsa +chmod 600 ~/.ssh/id_rsa -# setup env -# TODO: Conditional installation (python3, terraform & ansible) -if grep -E -q "CentOS|Red Hat" /etc/os-release -then - # TODO: Terraform Repo access - #yum install -y yum-utils - #yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo - #if (which terraform); then echo "Terraform exists, skipping the installation"; else (yum install -y terraform - if (which python3); then echo "Python3 exists, skipping the installation"; else (yum install -y python38); fi - if (which ansible-playbook); then echo "Ansible exists, skipping the installation"; else (yum install -y ansible); fi -elif grep -q "Ubuntu" /etc/os-release -then - apt update - # TODO: Terraform Repo access - #apt-get update && sudo apt-get install -y gnupg software-properties-common - #wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg - #gpg --no-default-keyring --keyring /usr/share/keyrings/hashicorp-archive-keyring.gpg --fingerprint - apt install software-properties-common - apt-add-repository --yes --update ppa:ansible/ansible - if (which python3); then echo "Python3 exists, skipping the installation"; else (apt install python38); fi - if (which ansible-playbook); then echo "Ansible exists, skipping the installation"; else (apt install ansible); fi -fi +# # setup env +# # TODO: Conditional installation (python3, terraform & ansible) +# if grep -E -q "CentOS|Red Hat" /etc/os-release +# then +# # TODO: Terraform Repo access +# #yum install -y yum-utils +# #yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo +# #if (which terraform); then echo "Terraform exists, skipping the installation"; else (yum install -y terraform +# if (which python3); then echo "Python3 exists, skipping the installation"; else (yum install -y python38); fi +# if (which ansible-playbook); then echo "Ansible exists, skipping the installation"; else (yum install -y ansible); fi +# elif grep -q "Ubuntu" /etc/os-release +# then +# apt update +# # TODO: Terraform Repo access +# #apt-get update && sudo apt-get install -y gnupg software-properties-common +# #wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg +# #gpg --no-default-keyring --keyring /usr/share/keyrings/hashicorp-archive-keyring.gpg --fingerprint +# apt install software-properties-common +# apt-add-repository --yes --update ppa:ansible/ansible +# if (which python3); then echo "Python3 exists, skipping the installation"; else (apt install python38); fi +# if (which ansible-playbook); then echo "Ansible exists, skipping the installation"; else (apt install ansible); fi +# fi # TODO: run terraform diff --git a/modules/deployer/variables.tf b/modules/deployer/variables.tf index 1c9ba1e2..1b89ae38 100644 --- a/modules/deployer/variables.tf +++ b/modules/deployer/variables.tf @@ -130,3 +130,34 @@ variable "skip_iam_authorization_policy" { default = false description = "Set to false if authorization policy is required for VPC block storage volumes to access kms. This can be set to true if authorization policy already exists. For more information on how to create authorization policy manually, see [creating authorization policies for block storage volume](https://cloud.ibm.com/docs/vpc?topic=vpc-block-s2s-auth&interface=ui)." } + +variable "management_instances" { + type = list( + object({ + profile = string + count = number + image = string + }) + ) + description = "Number of instances to be launched for management." +} + +variable "static_compute_instances" { + type = list( + object({ + profile = string + count = number + image = string + }) + ) + description = "Total Number of instances to be launched for compute cluster." +} + +variable "dns_domain_names" { + type = object({ + compute = string + storage = string + protocol = string + }) + description = "IBM Cloud HPC DNS domain names." +} diff --git a/modules/file_storage/outputs.tf b/modules/file_storage/outputs.tf index c8c27cc3..7f265e54 100644 --- a/modules/file_storage/outputs.tf +++ b/modules/file_storage/outputs.tf @@ -9,4 +9,4 @@ output "mount_path" { output "name_mount_path_map" { description = "Mount path name and its path map" value = { for mount_details in flatten([ibm_is_share_mount_target.share_target_vpc, ibm_is_share_mount_target.share_target_sg]) : split("-", mount_details.name)[length(split("-", mount_details.name)) - 4] => mount_details.mount_path } -} +} \ No newline at end of file diff --git a/modules/inventory/main.tf b/modules/inventory/main.tf index 10c6e56e..b63bf85d 100644 --- a/modules/inventory/main.tf +++ b/modules/inventory/main.tf @@ -7,7 +7,6 @@ resource "local_sensitive_file" "itself" { content = <> ~/.ssh/authorized_keys -echo "${compute_public_key_content}" >> ~/.ssh/authorized_keys +echo "${compute_public_key_content}" | base64 --decode >> ~/.ssh/authorized_keys echo "StrictHostKeyChecking no" >> ~/.ssh/config -echo "${compute_private_key_content}" > ~/.ssh/id_rsa +echo "${compute_private_key_content}" | base64 --decode > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa # network setup diff --git a/modules/landing_zone_vsi/templates/management_user_data.tpl b/modules/landing_zone_vsi/templates/management_user_data.tpl index 8488c692..1db5be80 100644 --- a/modules/landing_zone_vsi/templates/management_user_data.tpl +++ b/modules/landing_zone_vsi/templates/management_user_data.tpl @@ -17,9 +17,9 @@ sed -i -e "s/^/no-port-forwarding,no-agent-forwarding,no-X11-forwarding,command= # input parameters echo "${bastion_public_key_content}" >> ~/.ssh/authorized_keys -echo "${management_public_key_content}" >> ~/.ssh/authorized_keys +echo "${management_public_key_content}" | base64 --decode >> ~/.ssh/authorized_keys echo "StrictHostKeyChecking no" >> ~/.ssh/config -echo "${management_private_key_content}" > ~/.ssh/id_rsa +echo "${management_private_key_content}" | base64 --decode > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa # network setup diff --git a/modules/landing_zone_vsi/variables.tf b/modules/landing_zone_vsi/variables.tf index 5339eb2e..bb965cb1 100644 --- a/modules/landing_zone_vsi/variables.tf +++ b/modules/landing_zone_vsi/variables.tf @@ -298,3 +298,23 @@ variable "boot_volume_encryption_key" { ############################################################################## # TODO: Auth Server (LDAP/AD) Variables ############################################################################## + +variable "compute_public_key_content" { + type = string + sensitive = true + default = null + description = "Compute security key content." +} + +variable "compute_private_key_content" { + type = string + sensitive = true + default = null + description = "Compute security key content." +} + +variable "enable_bastion" { + type = bool + default = true + description = "The solution supports multiple ways to connect to your HPC cluster for example, using bastion node, via VPN or direct connection. If connecting to the HPC cluster via VPN or direct connection, set this value to false." +} \ No newline at end of file diff --git a/modules/observability_instance/datasources.tf b/modules/observability_instance/datasources.tf index 6be8d171..241a7090 100644 --- a/modules/observability_instance/datasources.tf +++ b/modules/observability_instance/datasources.tf @@ -8,6 +8,6 @@ data "http" "sysdig_prws_key" { request_headers = { Accept = "application/json" Authorization = data.ibm_iam_auth_token.tokendata.iam_access_token - IBMInstanceID = var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_guid : "" + IBMInstanceID = var.enable_deployer && var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_guid : "" } } diff --git a/modules/observability_instance/main.tf b/modules/observability_instance/main.tf index 43568f7e..586260a5 100644 --- a/modules/observability_instance/main.tf +++ b/modules/observability_instance/main.tf @@ -5,6 +5,7 @@ locals { } module "observability_instance" { + count = var.enable_deployer ? 1 : 0 # Replace "master" with a GIT release version to lock into a specific release source = "terraform-ibm-modules/observability-instances/ibm" version = "3.3.1" diff --git a/modules/observability_instance/outputs.tf b/modules/observability_instance/outputs.tf index 523a668e..43c89b4e 100644 --- a/modules/observability_instance/outputs.tf +++ b/modules/observability_instance/outputs.tf @@ -1,5 +1,5 @@ output "cloud_monitoring_access_key" { - value = var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_access_key : null + value = var.enable_deployer && var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_access_key : null description = "IBM Cloud Monitoring access key for agents to use" sensitive = true } @@ -10,7 +10,7 @@ output "cloud_monitoring_ingestion_url" { } output "cloud_monitoring_prws_key" { - value = var.cloud_monitoring_provision ? jsondecode(data.http.sysdig_prws_key[0].response_body).token.key : null + value = var.enable_deployer && var.cloud_monitoring_provision ? jsondecode(data.http.sysdig_prws_key[0].response_body).token.key : null description = "IBM Cloud Monitoring Prometheus Remote Write ingestion key" sensitive = true } @@ -31,7 +31,7 @@ output "cloud_logs_ingress_private_endpoint" { } output "cloud_monitoring_url" { - value = var.cloud_monitoring_provision ? "https://cloud.ibm.com/observe/embedded-view/monitoring/${module.observability_instance.cloud_monitoring_guid}" : null + value = var.enable_deployer && var.cloud_monitoring_provision ? "https://cloud.ibm.com/observe/embedded-view/monitoring/${module.observability_instance.cloud_monitoring_guid}" : null description = "IBM Cloud Monitoring URL" } diff --git a/modules/observability_instance/variables.tf b/modules/observability_instance/variables.tf index 25522547..2a80cbbf 100644 --- a/modules/observability_instance/variables.tf +++ b/modules/observability_instance/variables.tf @@ -85,3 +85,9 @@ variable "cloud_metrics_data_bucket" { description = "Generate and store metrics from your events so you can visualize, track, and alert on log events in real-time." type = any } + +variable "enable_deployer" { + type = bool + default = false + description = "Deployer should be only used for better deployment performance" +} \ No newline at end of file diff --git a/modules/playbook/main.tf b/modules/playbook/main.tf index 857e4d44..6902c3df 100644 --- a/modules/playbook/main.tf +++ b/modules/playbook/main.tf @@ -1,15 +1,19 @@ +locals { + proxyjump = var.enable_bastion ? "-o ProxyJump=ubuntu@${var.bastion_fip}" : "" +} + resource "local_file" "create_playbook" { count = var.inventory_path != null ? 1 : 0 content = < - -o ProxyJump=ubuntu@${var.bastion_fip} + ${local.proxyjump} -o ControlMaster=auto -o ControlPersist=30m -o UserKnownHostsFile=/dev/null @@ -30,15 +34,20 @@ resource "local_file" "create_playbook" { gather_facts: false vars: ansible_ssh_common_args: > - -o ProxyJump=ubuntu@${var.bastion_fip} + ${local.proxyjump} -o ControlMaster=auto -o ControlPersist=30m -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ansible_user: root ansible_ssh_private_key_file: ${var.private_key_path} + pre_tasks: + - name: Load cluster-specific variables + include_vars: all.json roles: - prerequisite + - lsf_templates + - lsf-ansible-command EOT filename = var.playbook_path } @@ -56,15 +65,45 @@ resource "null_resource" "run_playbook" { depends_on = [local_file.create_playbook] } -resource "ansible_playbook" "playbook" { - playbook = var.playbook_path - name = "localhost" - replayable = true - verbosity = 6 - extra_vars = { - ansible_python_interpreter = "auto" - inventory_file = var.inventory_path +# resource "ansible_playbook" "playbook" { +# playbook = var.playbook_path +# name = "localhost" +# replayable = true +# verbosity = 6 +# extra_vars = { +# ansible_python_interpreter = "auto" +# inventory_file = var.inventory_path +# } +# depends_on = [local_file.create_playbook] +# } + +# resource "ansible_playbook" "playbook" { +# playbook = var.playbook_path +# name = "localhost" +# replayable = true +# verbosity = 6 +# extra_vars = { +# ansible_python_interpreter = "auto" +# inventory_file = var.inventory_path +# } +# depends_on = [local_file.create_playbook] +# } + +resource "null_resource" "run_lsf_playbooks" { + count = var.inventory_path != null ? 1 : 0 + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + command = <