diff --git a/.tekton/scripts/common_utils.sh b/.tekton/scripts/common_utils.sh index 4d267e4d..edb11095 100644 --- a/.tekton/scripts/common_utils.sh +++ b/.tekton/scripts/common_utils.sh @@ -33,5 +33,6 @@ go version python3 -m pip install --upgrade pip python3 -m pip install --pre --upgrade requests==2.20.0 python3 -m pip install --pre --upgrade ibm-cos-sdk==2.0.1 +ibmcloud plugin install key-protect -f echo "************************************************" diff --git a/.tekton/scripts/suites.sh b/.tekton/scripts/suites.sh index 84da4d89..6d7be8bc 100644 --- a/.tekton/scripts/suites.sh +++ b/.tekton/scripts/suites.sh @@ -79,7 +79,7 @@ pr_ubuntu_suite() { # commit based suite on rhel-suite-1 rhel_suite_1() { suite=rhel-suite-1 - test_cases="TestRunBasic,TestRunAppCenter,TestRunNoKMSAndHTOff" + test_cases="TestRunBasic,TestRunAppCenter,TestRunNoKMSAndHTOff,TestRunCosAndVpcFlowLogs" new_line="${test_cases//,/$'\n'}" echo "************** Going to run ${suite} ${new_line} **************" common_suite "${test_cases}" "${suite}" "${compute_image_name_rhel:?}" @@ -88,7 +88,7 @@ rhel_suite_1() { # commit based suite on rhel-suite-2 rhel_suite_2() { suite=rhel-suite-2 - test_cases="TestRunLDAP,TestRunLDAPAndPac,TestRunCustomRGAsNonDefault" + test_cases="TestRunLDAP,TestRunLDAPAndPac,TestRunCustomRGAsNonDefault,TestRunUsingExistingKMSInstanceIDAndWithoutKey" new_line="${test_cases//,/$'\n'}" echo "************** Going to run ${suite} ${new_line} **************" common_suite "${test_cases}" "${suite}" "${compute_image_name_rhel:?}" @@ -124,7 +124,7 @@ ubuntu_suite_2() { # commit based suite on ubuntu-suite-3 ubuntu_suite_3() { suite=ubuntu-suite-3 - test_cases="TestRunBasic,TestRunNoKMSAndHTOff" + test_cases="TestRunBasic,TestRunNoKMSAndHTOff,TestRunExistingLDAP,TestRunExistingPACEnvironment" new_line="${test_cases//,/$'\n'}" echo "************** Going to run ${suite} ${new_line} **************" common_suite "${test_cases}" "${suite}" "${compute_image_name_ubuntu:?}" @@ -142,7 +142,7 @@ regions_suite() { # negative based suite on negative-suite negative_suite() { suite=negative-suite - test_cases="TestRunWithoutMandatory,TestRunInvalidReservationIDAndContractID,TestRunInvalidLDAPServerIP,TestRunInvalidLDAPUsernamePassword,TestRunInvalidAPPCenterPassword,TestRunInvalidDomainName,TestRunKMSInstanceNameAndKMSKeyNameWithInvalidValue,TestRunExistSubnetIDVpcNameAsNull" + test_cases="TestRunWithoutMandatory,TestRunInvalidReservationIDAndContractID,TestRunInvalidLDAPServerIP,TestRunInvalidLDAPUsernamePassword,TestRunInvalidAPPCenterPassword,TestRunInvalidDomainName,TestRunKMSInstanceNameAndKMSKeyNameWithInvalidValue,TestRunExistSubnetIDVpcNameAsNull,TestRunInvalidSshKeysAndRemoteAllowedIP,TestRunInvalidSubnetCIDR" new_line="${test_cases//,/$'\n'}" echo "************** Going to run ${suite} ${new_line} **************" common_suite "${test_cases}" "${suite}" "${compute_image_name_rhel:?}" diff --git a/modules/landing_zone_vsi/configuration_steps/compute_user_data_fragment.sh b/modules/landing_zone_vsi/configuration_steps/compute_user_data_fragment.sh index e3998692..fb7ff888 100644 --- a/modules/landing_zone_vsi/configuration_steps/compute_user_data_fragment.sh +++ b/modules/landing_zone_vsi/configuration_steps/compute_user_data_fragment.sh @@ -233,13 +233,13 @@ cat /opt/ibm/lsf/conf/hosts >> /etc/hosts if [ "$enable_ldap" = "true" ]; then # Detect the operating system - if grep -q "NAME=\"Red Hat Enterprise Linux\"" /etc/os-release; then + if grep -q "NAME=\"Red Hat Enterprise Linux\"" /etc/os-release || grep -q "NAME=\"Rocky Linux\"" /etc/os-release; then - # Detect RHEL version - rhel_version=$(grep -oE 'release [0-9]+' /etc/redhat-release | awk '{print $2}') + # Detect RHEL or Rocky version + version=$(grep -oE 'release [0-9]+' /etc/redhat-release | awk '{print $2}') - if [ "$rhel_version" == "8" ]; then - echo "Detected RHEL 8. Proceeding with LDAP client configuration...." >> $logfile + if [ "$version" == "8" ]; then + echo "Detected as RHEL or Rocky 8. Proceeding with LDAP client configuration...." >> $logfile # Allow Password authentication sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config @@ -294,7 +294,7 @@ EOF echo ". ${LSF_CONF}/profile.lsf" >> /etc/bashrc source /etc/bashrc else - echo "This script is designed for RHEL 8. Detected RHEL version: $rhel_version. Exiting." >> $logfile + echo "This script is intended for RHEL 8 or Rocky Linux 8. Detected version: $version. Exiting." >> $logfile exit 1 fi diff --git a/modules/landing_zone_vsi/configuration_steps/configure_management_vsi.sh b/modules/landing_zone_vsi/configuration_steps/configure_management_vsi.sh index bfa7a9a9..df0e4af0 100644 --- a/modules/landing_zone_vsi/configuration_steps/configure_management_vsi.sh +++ b/modules/landing_zone_vsi/configuration_steps/configure_management_vsi.sh @@ -49,6 +49,7 @@ LSF_TOP_VERSION="$LSF_TOP/10.1" LSF_SUITE_TOP="/opt/ibm/lsfsuite" LSF_SUITE_GUI="${LSF_SUITE_TOP}/ext/gui" LSF_SUITE_GUI_CONF="${LSF_SUITE_GUI}/conf" +LSF_SUITE_GUI_WORK="${LSF_SUITE_GUI}/work" LSF_SUITE_PERF="${LSF_SUITE_TOP}/ext/perf" LSF_SUITE_PERF_CONF="${LSF_SUITE_PERF}/conf" LSF_SUITE_PERF_BIN="${LSF_SUITE_PERF}/1.2/bin" @@ -265,7 +266,10 @@ EOT "ca-tor": "https://ca-tor.iaas.cloud.ibm.com/v1", "jp-osa": "https://jp-osa.iaas.cloud.ibm.com/v1", "jp-tok": "https://jp-tok.iaas.cloud.ibm.com/v1", - "br-sao": "https://br-sao.iaas.cloud.ibm.com/v1" + "br-sao": "https://br-sao.iaas.cloud.ibm.com/v1", + "us-south": "https://us-south.iaas.cloud.ibm.com/v1", + "eu-de": "https://eu-de.iaas.cloud.ibm.com/v1", + "us-east": "https://us-east.iaas.cloud.ibm.com/v1" } } EOT @@ -674,6 +678,7 @@ mount_nfs_with_retries() { if [ -n "${nfs_server_with_mount_path}" ]; then echo "File share ${nfs_server_with_mount_path} found" nfs_client_mount_path="/mnt/lsf" + nfs_client_mount_pac_path="${nfs_client_mount_path}/pac" if mount_nfs_with_retries "${nfs_server_with_mount_path}" "${nfs_client_mount_path}"; then # Move stuff to shared fs for dir in conf work das_staging_area; do @@ -688,15 +693,22 @@ if [ -n "${nfs_server_with_mount_path}" ]; then done # Sharing the lsfsuite.conf folder - if [ "$on_primary" == "true" ]; then - rm -rf "${nfs_client_mount_path}/gui-conf" - mv "${LSF_SUITE_GUI_CONF}" "${nfs_client_mount_path}/gui-conf" - chown -R lsfadmin:root "${nfs_client_mount_path}/gui-conf" - else - rm -rf "${LSF_SUITE_GUI_CONF}" + if [ "$on_primary" == "true" ] && [ "$enable_app_center" == "true" ] && [ "$app_center_high_availability" == "true" ]; then + # Create pac folder if it does not exist + [ ! -d "${nfs_client_mount_pac_path}" ] && mkdir -p "${nfs_client_mount_pac_path}" + + # Remove the original folder and create symlink for gui-conf + [ -d "${nfs_client_mount_pac_path}/gui-conf" ] && rm -rf "${nfs_client_mount_pac_path}/gui-conf" + mv "${LSF_SUITE_GUI_CONF}" "${nfs_client_mount_pac_path}/gui-conf" + chown -R lsfadmin:root "${nfs_client_mount_pac_path}/gui-conf" && chown -R lsfadmin:lsfadmin "${LSF_SUITE_GUI_CONF}" + ln -fs "${nfs_client_mount_pac_path}/gui-conf" "${LSF_SUITE_GUI_CONF}" + + # Remove the original folder and create symlink for gui-work + [ -d "${nfs_client_mount_pac_path}/gui-work" ] && rm -rf "${nfs_client_mount_pac_path}/gui-work" + mv "${LSF_SUITE_GUI_WORK}" "${nfs_client_mount_pac_path}/gui-work" + chown -R lsfadmin:root "${nfs_client_mount_pac_path}/gui-work" && chown -R lsfadmin:lsfadmin "${LSF_SUITE_GUI_WORK}" + ln -fs "${nfs_client_mount_pac_path}/gui-work" "${LSF_SUITE_GUI_WORK}" fi - ln -fs "${nfs_client_mount_path}/gui-conf" "${LSF_SUITE_GUI_CONF}" - chown -R lsfadmin:root "${LSF_SUITE_GUI_CONF}" # Create a data directory for sharing HPC workload data if [ "$on_primary" == "true" ]; then @@ -705,14 +717,6 @@ if [ -n "${nfs_server_with_mount_path}" ]; then chown -R lsfadmin:root "$LSF_TOP/work/data" fi - # Sharing the 10.1 folder - if [ "$on_primary" == "true" ]; then - rm -rf "${nfs_client_mount_path}/10.1" - mv "${LSF_TOP_VERSION}" "${nfs_client_mount_path}" - ln -s "${nfs_client_mount_path}/10.1" "${LSF_TOP_VERSION}" - chown -R lsfadmin:root "${LSF_TOP_VERSION}" - fi - # VNC Sessions if [ "$on_primary" == "true" ]; then mkdir -p "${nfs_client_mount_path}/repository-path" diff --git a/modules/landing_zone_vsi/image_map.tf b/modules/landing_zone_vsi/image_map.tf index 3e9590ed..efd7b2a2 100644 --- a/modules/landing_zone_vsi/image_map.tf +++ b/modules/landing_zone_vsi/image_map.tf @@ -1,19 +1,19 @@ locals { image_region_map = { - "hpcaas-lsf10-rhel88-v9" = { - "us-east" = "r014-d2b18006-c0c4-428f-96f3-e033b970c582" - "eu-de" = "r010-3bf3f57e-1985-431d-aefe-e9914ab7919c" - "us-south" = "r006-7b0aa90b-f52c-44b1-bab7-ccbfae9f1816" + "hpcaas-lsf10-rhel88-v10" = { + "us-east" = "r014-c425251e-b7b5-479f-b9cf-ef72a0f51b5a" + "eu-de" = "r010-ce6dac2b-57e4-4a2f-a77d-8b0e549b2cae" + "us-south" = "r006-c8fefef3-645d-4b5a-bad0-2250c6ddb627" }, - "hpcaas-lsf10-rhel88-compute-v5" = { - "us-east" = "r014-deb34fb1-edbf-464c-9af3-7efa2efcff3f" - "eu-de" = "r010-2d04cfff-6f54-45d1-b3b3-7e259083d71f" - "us-south" = "r006-236ee1f4-38de-4845-b7ec-e2ffa7df5d08" + "hpcaas-lsf10-rhel88-compute-v6" = { + "us-east" = "r014-f4c0dd0f-3bd0-4e2f-bbda-bbbc75a8c33b" + "eu-de" = "r010-0217f13b-e6e5-4500-acd4-c170f111d43f" + "us-south" = "r006-15514933-925c-4923-85dd-3165dcaa3180" }, - "hpcaas-lsf10-ubuntu2204-compute-v5" = { - "us-east" = "r014-ecbf4c89-16a3-472e-8bab-1e76d744e264" - "eu-de" = "r010-9811d8bf-a7f8-4ee6-8342-e5af217bc513" - "us-south" = "r006-ed76cb75-f086-48e9-8090-e2dbc411abe7" + "hpcaas-lsf10-ubuntu2204-compute-v6" = { + "us-east" = "r014-ab2e8be8-d75c-4040-a337-7f086f3ce153" + "eu-de" = "r010-027f5d54-9360-4d1f-821b-583329d63855" + "us-south" = "r006-628b6dbe-e0d4-4c25-bc0f-f554f5523f2e" } } } diff --git a/samples/configs/hpc_catalog_values.json b/samples/configs/hpc_catalog_values.json index 210999b6..28cb5309 100644 --- a/samples/configs/hpc_catalog_values.json +++ b/samples/configs/hpc_catalog_values.json @@ -20,9 +20,9 @@ "enable_cos_integration" : "false", "cos_instance_name" : "__NULL__", "enable_fip" : "true", - "management_image_name" : "hpcaas-lsf10-rhel88-v9", - "compute_image_name" : "hpcaas-lsf10-rhel88-compute-v5", - "login_image_name" : "hpcaas-lsf10-rhel88-compute-v5", + "management_image_name" : "hpcaas-lsf10-rhel88-v10", + "compute_image_name" : "hpcaas-lsf10-rhel88-compute-v6", + "login_image_name" : "hpcaas-lsf10-rhel88-compute-v6", "login_node_instance_type" : "bx2-2x8", "management_node_instance_type" : "bx2-16x64", "management_node_count" : "3", diff --git a/samples/configs/hpc_schematics_values.json b/samples/configs/hpc_schematics_values.json index 2362b330..12912d69 100644 --- a/samples/configs/hpc_schematics_values.json +++ b/samples/configs/hpc_schematics_values.json @@ -197,17 +197,17 @@ }, { "name": "management_image_name", - "value": "hpcaas-lsf10-rhel88-v9", + "value": "hpcaas-lsf10-rhel88-v10", "type": "string", "secure": false, "description": "Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster management nodes. By default, the solution uses a RHEL88 base image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering." }, { "name": "compute_image_name", - "value": "hpcaas-lsf10-rhel88-compute-v5", + "value": "hpcaas-lsf10-rhel88-compute-v6", "type": "string", "secure": false, - "description": "Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v5). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering." + "description": "Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-8 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v6). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering." }, { @@ -215,7 +215,7 @@ "value": "hpcaas-lsf10-rhel88-compute-v5", "type": "string", "secure": false, - "description": "Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v5). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering." + "description": "Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-8 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v6). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering." }, { "name": "login_node_instance_type", @@ -324,7 +324,7 @@ "value": "true", "type": "bool", "secure": false, - "description": "Enable Activity tracker service instance connected to Cloud Object Storage (COS). All the events will be stored into COS so that customers can connect to it and read those events or ingest them in their system." + "description": "Enable Activity tracker service instance connected to Cloud Object Storage (COS). All the events will be stored in COS so customers can retrieve or ingest them in their system. While multiple Activity Tracker instances can be created, only one tracker is needed to capture all events. Creating additional trackers is unnecessary if an existing Activity Tracker is already integrated with a COS bucket. In such cases, set the value to false, as all events can be monitored and accessed through the existing Activity Tracker." }, { "name": "enable_ldap", diff --git a/solutions/hpc/variables.tf b/solutions/hpc/variables.tf index 941dd2df..86144a21 100644 --- a/solutions/hpc/variables.tf +++ b/solutions/hpc/variables.tf @@ -176,21 +176,21 @@ variable "login_node_instance_type" { } variable "management_image_name" { type = string - default = "hpcaas-lsf10-rhel88-v9" + default = "hpcaas-lsf10-rhel88-v10" description = "Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster management nodes. By default, the solution uses a RHEL88 base image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering." } variable "compute_image_name" { type = string - default = "hpcaas-lsf10-rhel88-compute-v5" - description = "Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-8 base OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v5). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering." + default = "hpcaas-lsf10-rhel88-compute-v6" + description = "Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-8 base OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v6). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering." } variable "login_image_name" { type = string - default = "hpcaas-lsf10-rhel88-compute-v5" - description = "Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-8 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v5). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering." + default = "hpcaas-lsf10-rhel88-compute-v6" + description = "Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-8 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v6). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering." } variable "management_node_instance_type" { @@ -296,7 +296,7 @@ variable "cos_instance_name" { variable "observability_atracker_on_cos_enable" { type = bool default = true - description = "Enable Activity tracker service instance connected to Cloud Object Storage (COS). All the events will be stored into COS so that customers can connect to it and read those events or ingest them in their system." + description = "Enable Activity tracker service instance connected to Cloud Object Storage (COS). All the events will be stored in COS so customers can retrieve or ingest them in their system. While multiple Activity Tracker instances can be created, only one tracker is needed to capture all events. Creating additional trackers is unnecessary if an existing Activity Tracker is already integrated with a COS bucket. In such cases, set the value to false, as all events can be monitored and accessed through the existing Activity Tracker." } variable "enable_vpc_flow_logs" { diff --git a/tests/README.md b/tests/README.md index ea0a71dd..17eb945c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -102,7 +102,7 @@ go test -v -timeout 900m -parallel 4 -run "TestRunBasic" | tee test_output.log To override default values, pass the necessary parameters in the command. Example: ```sh -SSH_KEY=your_ssh_key ZONE=your_zone RESOURCE_GROUP=your_resource_group RESERVATION_ID=your_reservation_id KMS_INSTANCE_ID=kms_instance_id KMS_KEY_NAME=kms_key_name IMAGE_NAME=image_name CLUSTER=your_cluster_id DEFAULT_RESOURCE_GROUP=default_resource_group NON_DEFAULT_RESOURCE_GROUP=non_default_resource_group LOGIN_NODE_INSTANCE_TYPE=login_node_instance_type MANAGEMENT_IMAGE_NAME=management_image_name COMPUTE_IMAGE_NAME=compute_image_name MANAGEMENT_NODE_INSTANCE_TYPE=management_node_instance_type MANAGEMENT_NODE_COUNT=management_node_count ENABLE_VPC_FLOW_LOGS=enable_vpc_flow_logs KEY_MANAGEMENT=key_management KMS_INSTANCE_NAME=kms_instance_name HYPERTHREADING_ENABLED=hyperthreading_enabled US_EAST_ZONE=us_east_zone US_EAST_RESERVATION_ID=us_east_reservation_id US_EAST_CLUSTER_ID=us_east_cluster_id US_SOUTH_ZONE=us_south_zone US_SOUTH_RESERVATION_ID=us_south_reservation_id US_SOUTH_CLUSTER_ID=us_south_cluster_id EU_DE_ZONE=eu_de_zone EU_DE_RESERVATION_ID=eu_de_reservation_id EU_DE_CLUSTER_ID=eu_de_cluster_id SSH_FILE_PATH=ssh_file_path go test -v -timeout 900m -parallel 4 -run "TestRunBasic" | tee test_output.log +SSH_KEY=your_ssh_key ZONE=your_zone RESOURCE_GROUP=your_resource_group RESERVATION_ID=your_reservation_id KMS_INSTANCE_ID=kms_instance_id KMS_KEY_NAME=kms_key_name IMAGE_NAME=image_name CLUSTER=your_cluster_id DEFAULT_RESOURCE_GROUP=default_resource_group NON_DEFAULT_RESOURCE_GROUP=non_default_resource_group LOGIN_NODE_INSTANCE_TYPE=login_node_instance_type MANAGEMENT_IMAGE_NAME=management_image_name COMPUTE_IMAGE_NAME=compute_image_name MANAGEMENT_NODE_INSTANCE_TYPE=management_node_instance_type MANAGEMENT_NODE_COUNT=management_node_count ENABLE_VPC_FLOW_LOGS=enable_vpc_flow_logs KEY_MANAGEMENT=key_management KMS_INSTANCE_NAME=kms_instance_name HYPERTHREADING_ENABLED=hyperthreading_enabled SSH_FILE_PATH=ssh_file_path go test -v -timeout 900m -parallel 4 -run "TestRunBasic" | tee test_output.log ``` Replace placeholders (e.g., `your_ssh_key`, `your_zone`, etc.) with actual values. @@ -148,7 +148,7 @@ FAIL github.com/terraform-ibm-modules/terraform-ibcloud-hpc 663.323s ### Viewing Test Output Logs - **Console Output**: Check the console for immediate test results. -- **Log Files**: Detailed logs are saved in `test_output.log` and custom logs in the `/tests/test_output` folder. Logs are timestamped for easier tracking (e.g., `log_20XX-MM-DD_HH-MM-SS.log`). +- **Log Files**: Detailed logs are saved in `test_output.log` and custom logs in the `/tests/logs_output` folder. Logs are timestamped for easier tracking (e.g., `log_20XX-MM-DD_HH-MM-SS.log`). ## Troubleshooting @@ -166,26 +166,29 @@ For additional help, contact the project maintainers. ## Project Structure -```plaintext -/root/HPCAAS/HPCaaS/tests +``` +/root/HPCAAS/tests ├── README.md -├── common_utils -│ ├── deploy_utils.go -│ ├── log_utils.go -│ ├── ssh_utils.go -│ └── utils.go -├── constants.go -├── go.mod -├── go.sum +├── utilities +│ ├── deployment.go # Deployment-related utility functions +│ ├── fileops.go # File operations utility functions +│ ├── helpers.go # General helper functions +│ ├── logging.go # Logging utility functions +│ ├── resources.go # Resource management utility functions +│ └── ssh.go # SSH utility functions +├── constants.go # Project-wide constants +├── go.mod # Go module definition +├── go.sum # Go module checksum ├── lsf -│ ├── lsf_cluster_test_utils.go -│ ├── lsf_cluster_test_validation.go -│ ├── lsf_cluster_utils.go -│ └── lsf_constants.go -├── other_test.go -├── pr_test.go -├── test_config.yml -└── test_output +│ ├── cluster_helpers.go # Helper functions for cluster testing +│ ├── cluster_utils.go # General utilities for cluster operations +│ ├── cluster_validation.go # Validation logic for cluster tests +│ └── constants.go # Constants specific to LSF +├── other_tests.go # Additional test cases +├── pr_tests.go # Pull request-related tests +├── config.yml # Configuration file +└── logs # Directory for log files + ``` ## Utilities diff --git a/tests/common_utils/utils.go b/tests/common_utils/utils.go deleted file mode 100644 index 3c268973..00000000 --- a/tests/common_utils/utils.go +++ /dev/null @@ -1,963 +0,0 @@ -package tests - -import ( - "bufio" - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "math/rand" - "os" - "os/exec" - "path/filepath" - "reflect" - "regexp" - "strconv" - "strings" - "testing" - "time" - - "github.com/IBM/go-sdk-core/v5/core" - "github.com/IBM/secrets-manager-go-sdk/v2/secretsmanagerv2" - "github.com/stretchr/testify/assert" - "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testhelper" - "golang.org/x/crypto/ssh" -) - -const ( - // TimeLayout is the layout string for date and time format (DDMonHHMMSS). - TimeLayout = "Jan02" -) - -// GetValueFromIniFile retrieves a value from an INI file based on the provided section and key. -// It reads the specified INI file, extracts the specified section, and returns the value associated with the key. -func GetValueFromIniFile(filePath, sectionName string) ([]string, error) { - // Read the content of the file - absolutePath, err := filepath.Abs(filePath) - if err != nil { - return nil, err - } - data, err := os.ReadFile(absolutePath) - if err != nil { - return nil, err - } - - // Convert the byte slice to a string - content := string(data) - - // Split the input into sections based on empty lines - sections := strings.Split(content, "\n\n") - - // Loop through sections and find the one with the specified sectionName - for _, section := range sections { - - if strings.Contains(section, "["+sectionName+"]") { - // Split the section into lines - lines := strings.Split(section, "\n") - - // Extract values - var sectionValues []string - for i := 1; i < len(lines); i++ { - // Skip the first line, as it contains the section name - sectionValues = append(sectionValues, strings.TrimSpace(lines[i])) - } - - return sectionValues, nil - } - } - - return nil, fmt.Errorf("section [%s] not found in file %s", sectionName, filePath) -} - -// ######### file operations functions ############## - -// ToCreateFile creates a file on the remote server using SSH. -// It takes an SSH client, the path where the file should be created, and the file name. -// Returns a boolean indicating success or failure and an error if any. -func ToCreateFile(t *testing.T, sClient *ssh.Client, filePath, fileName string, logger *AggregatedLogger) (bool, error) { - // Check if the specified path exists on the remote server - isPathExist, err := IsPathExist(t, sClient, filePath, logger) - if err != nil { - // Error occurred while checking path existence - return false, err - } - - if isPathExist { - // Path exists, create the file using SSH command - command := "cd " + filePath + " && touch " + fileName - _, createFileErr := RunCommandInSSHSession(sClient, command) - - if createFileErr == nil { - logger.Info(t, "File created successfully: "+fileName) - // File created successfully - return true, nil - } else { - // Error occurred while creating the file - return false, fmt.Errorf(" %s file not created", fileName) - } - } - - // Path does not exist - return false, fmt.Errorf("directory not exist : %s", filePath) -} - -// IsFileExist checks if a file exists on the remote server using SSH. -// It takes an SSH client, the path where the file should exist, and the file name. -// Returns a boolean indicating whether the file exists and an error if any. -func IsFileExist(t *testing.T, sClient *ssh.Client, filePath, fileName string, logger *AggregatedLogger) (bool, error) { - // Check if the specified path exists on the remote server - isPathExist, err := IsPathExist(t, sClient, filePath, logger) - if err != nil { - // Error occurred while checking path existence - return false, err - } - - if isPathExist { - // Path exists, check if the file exists using an SSH command - command := "[ -f " + filePath + fileName + " ] && echo 'File exist' || echo 'File not exist'" - isFileExist, err := RunCommandInSSHSession(sClient, command) - - if err != nil { - // Error occurred while running the command - return false, err - } - - if strings.Contains(isFileExist, "File exist") { - logger.Info(t, fmt.Sprintf("File exist : %s", fileName)) - // File exists - return true, nil - } else { - logger.Info(t, fmt.Sprintf("File not exist : %s", fileName)) - // File does not exist - return false, nil - } - } - - // Path does not exist - return false, fmt.Errorf("directory not exist : %s", filePath) -} - -// IsPathExist checks if a directory exists on the remote server using SSH. -// It takes an SSH client and the path to check for existence. -// Returns a boolean indicating whether the directory exists and an error if any. -func IsPathExist(t *testing.T, sClient *ssh.Client, filePath string, logger *AggregatedLogger) (bool, error) { - // Run an SSH command to test if the directory exists - command := "test -d " + filePath + " && echo 'Directory Exist' || echo 'Directory Not Exist'" - result, err := RunCommandInSSHSession(sClient, command) - - if err != nil { - // Error occurred while running the command - return false, err - } - - if strings.Contains(result, "Directory Exist") { - logger.Info(t, fmt.Sprintf("Directory exist : %s", filePath)) - // Directory exists - return true, nil - } else { - logger.Info(t, fmt.Sprintf("Directory not exist : %s", filePath)) - // Directory does not exist - return false, nil - } -} - -// GetDirList retrieves the list of files in a directory on a remote server via SSH. -// It takes an SSH client and the path of the directory. -// Returns a string containing the list of files and an error if any. -func GetDirList(t *testing.T, sClient *ssh.Client, filePath string, logger *AggregatedLogger) ([]string, error) { - command := "cd " + filePath + " && ls" - output, err := RunCommandInSSHSession(sClient, command) - if err == nil { - listDir := strings.Split(strings.TrimSpace(output), "\n") - logger.Info(t, fmt.Sprintf("Directory list : %q ", listDir)) - return listDir, nil - } - return nil, err -} - -// GetDirectoryFileList retrieves the list of files in a directory on a remote server via SSH. -// It takes an SSH client and the path of the directory. -// Returns a slice of strings representing the file names and an error if any. -func GetDirectoryFileList(t *testing.T, sClient *ssh.Client, directoryPath string, logger *AggregatedLogger) ([]string, error) { - command := "cd " + directoryPath + " && ls" - output, err := RunCommandInSSHSession(sClient, command) - if err == nil { - fileList := strings.Split(strings.TrimSpace(output), "\n") - logger.Info(t, fmt.Sprintf("Directory file list : %q", fileList)) - return fileList, nil - } - - return nil, err -} - -// ToDeleteFile deletes a file on the remote server using SSH. -// It takes an SSH client, the path of the file's directory, and the file name. -// Returns a boolean indicating whether the file was deleted successfully and an error if any. -func ToDeleteFile(t *testing.T, sClient *ssh.Client, filePath, fileName string, logger *AggregatedLogger) (bool, error) { - isPathExist, err := IsPathExist(t, sClient, filePath, logger) - if isPathExist { - command := "cd " + filePath + " && rm -rf " + fileName - _, deleFileErr := RunCommandInSSHSession(sClient, command) - if deleFileErr == nil { - logger.Info(t, fmt.Sprintf("File deleted successfully: %s", fileName)) - return true, nil - } - return false, fmt.Errorf("files not deleted: %s", fileName) - } - return isPathExist, err -} - -// ToCreateFileWithContent creates a file on the remote server using SSH. -// It takes an SSH client, the path where the file should be created, the file name, content to write to the file, -// a log file for logging, and returns a boolean indicating success or failure and an error if any. -func ToCreateFileWithContent(t *testing.T, sClient *ssh.Client, filePath, fileName, content string, logger *AggregatedLogger) (bool, error) { - // Check if the specified path exists on the remote server - isPathExist, err := IsPathExist(t, sClient, filePath, logger) - if err != nil { - // Error occurred while checking path existence - return false, fmt.Errorf("error checking path existence: %w", err) - } - - if isPathExist { - // Path exists, create the file using SSH command - command := "cd " + filePath + " && echo '" + content + "' > " + fileName - _, createFileErr := RunCommandInSSHSession(sClient, command) - - if createFileErr == nil { - // File created successfully - logger.Info(t, "File created successfully: "+fileName) - return true, nil - } - - // Error occurred while creating the file - return false, fmt.Errorf("error creating file %s: %w", fileName, createFileErr) - } - - // Path does not exist - return false, fmt.Errorf("directory not exist : %s", filePath) -} - -// ReadRemoteFileContents reads the content of a file on the remote server via SSH. -// It checks if the specified file path exists, creates an SSH command to concatenate the file contents, -// and returns the content as a string upon success. In case of errors, an empty string and an error are returned. -func ReadRemoteFileContents(t *testing.T, sClient *ssh.Client, filePath, fileName string, logger *AggregatedLogger) (string, error) { - isPathExist, err := IsPathExist(t, sClient, filePath, logger) - if err != nil { - return "", fmt.Errorf("error checking path existence: %w", err) - } - - if isPathExist { - command := "cd " + filePath + " && cat " + fileName - actualText, outErr := RunCommandInSSHSession(sClient, command) - - if outErr == nil { - logger.Info(t, "content: "+actualText) - return actualText, nil - } - - return "", fmt.Errorf("error reading file %s: %w", fileName, outErr) - } - - return "", fmt.Errorf("directory does not exist: %s", filePath) -} - -// VerifyDataContains is a generic function that checks if a value is present in data (string or string array) -// VerifyDataContains performs a verification operation on the provided data -// to determine if it contains the specified value. It supports string and -// string array types, logging results with the provided AggregatedLogger. -// Returns true if the value is found, false otherwise. -func VerifyDataContains(t *testing.T, data interface{}, val interface{}, logger *AggregatedLogger) bool { - //The data.(type) syntax is used to check the actual type of the data variable. - switch d := data.(type) { - case string: - //check if the val variable is of type string. - substr, ok := val.(string) - if !ok { - logger.Info(t, "Invalid type for val parameter") - return false - } - if substr != "" && strings.Contains(d, substr) { - logger.Info(t, fmt.Sprintf("The string '%s' contains the substring '%s'\n", d, substr)) - return true - } - logger.Info(t, fmt.Sprintf("The string '%s' does not contain the substring '%s'\n", d, substr)) - return false - - case []string: - switch v := val.(type) { - case string: - for _, arrVal := range d { - if arrVal == v { - logger.Info(t, fmt.Sprintf("The array '%q' contains the value: %s\n", d, v)) - return true - } - } - logger.Info(t, fmt.Sprintf("The array '%q' does not contain the value: %s\n", d, v)) - return false - - case []string: - if reflect.DeepEqual(d, v) { - logger.Info(t, fmt.Sprintf("The array '%q' contains the subarray '%q'\n", d, v)) - return true - } - logger.Info(t, fmt.Sprintf("The array '%q' does not contain the subarray '%q'\n", d, v)) - return false - - default: - logger.Info(t, "Invalid type for val parameter") - return false - } - - default: - logger.Info(t, "Unsupported type for data parameter") - return false - } -} - -func CountStringOccurrences(str string, substr string) int { - return strings.Count(str, substr) -} - -func SplitString(strValue string, splitCharacter string, indexValue int) string { - split := strings.Split(strValue, splitCharacter) - return split[indexValue] -} - -// StringToInt converts a string to an integer. -// Returns the converted integer and an error if the conversion fails. -func StringToInt(str string) (int, error) { - num, err := strconv.Atoi(str) - if err != nil { - return 0, err - } - return num, nil -} - -// RemoveNilValues removes nil value keys from the given map. -func RemoveNilValues(data map[string]interface{}) map[string]interface{} { - for key, value := range data { - if value == nil { - delete(data, key) - } - } - return data -} - -// LogVerificationResult logs the result of a verification check. -func LogVerificationResult(t *testing.T, err error, checkName string, logger *AggregatedLogger) { - if err == nil { - logger.Info(t, fmt.Sprintf("%s verification successful", checkName)) - } else { - assert.Nil(t, err, fmt.Sprintf("%s verification failed", checkName)) - logger.Error(t, fmt.Sprintf("%s verification failed: %s", checkName, err.Error())) - } -} - -// ParsePropertyValue parses the content of a string, searching for a property with the specified key. -// It returns the value of the property if found, or an empty string and an error if the property is not found. -func ParsePropertyValue(content, propertyKey string) (string, error) { - lines := strings.Split(content, "\n") - for _, line := range lines { - if strings.HasPrefix(line, propertyKey+"=") { - // Extract the value, removing quotes if present - value := strings.Trim(line[len(propertyKey)+1:], `"`) - return value, nil - } - } - - // Property not found - return "", fmt.Errorf("property '%s' not found in content:\n%s", propertyKey, content) -} - -// ParsePropertyValue parses the content of a string, searching for a property with the specified key. -// It returns the value of the property if found, or an empty string and an error if the property is not found. -func FindImageNamesByCriteria(name string) (string, error) { - // Get the absolute path of the image map file - absPath, err := filepath.Abs("modules/landing_zone_vsi/image_map.tf") - if err != nil { - return "", err - } - - absPath = strings.ReplaceAll(absPath, "tests", "") - readFile, err := os.Open(absPath) - if err != nil { - return "", err - } - defer readFile.Close() - - // Create a scanner to read lines from the file - fileScanner := bufio.NewScanner(readFile) - - var imageName string - - for fileScanner.Scan() { - line := fileScanner.Text() - if strings.Contains(strings.ToLower(line), strings.ToLower(name)) && strings.Contains(line, "compute") { - pattern := "[^a-zA-Z0-9\\s-]" - - // Compile the regular expression. - regex, err := regexp.Compile(pattern) - if err != nil { - return "", errors.New("error on image compiling regex") - } - // Use the regex to replace all matches with an empty string. - imageName = strings.TrimSpace(regex.ReplaceAllString(line, "")) - } - } - - // Check if any image names were found - if len(imageName) == 0 { - return imageName, errors.New("no image found with the specified criteria") - } - - return imageName, nil -} - -//Create IBMCloud Resources - -// LoginIntoIBMCloudUsingCLI logs into IBM Cloud using CLI. -// LoginIntoIBMCloudUsingCLI logs into IBM Cloud using CLI. -func LoginIntoIBMCloudUsingCLI(t *testing.T, apiKey, region, resourceGroup string) error { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - - // Configure IBM Cloud CLI - configCmd := exec.CommandContext(ctx, "ibmcloud", "config", "--check-version=false") - configOutput, err := configCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to configure IBM Cloud CLI: %w. Output: %s", err, string(configOutput)) - } - - // Login to IBM Cloud and set the target resource group - loginCmd := exec.CommandContext(ctx, "ibmcloud", "login", "--apikey", apiKey, "-r", region, "-g", resourceGroup) - loginOutput, err := loginCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to login to IBM Cloud: %w. Output: %s", err, string(loginOutput)) - } - - return nil -} - -// Prerequisite: Ensure that you have logged into IBM Cloud using the LoginIntoIBMCloudUsingCLI function before calling this function -// CreateVPC creates a new Virtual Private Cloud (VPC) in IBM Cloud with the specified VPC name. -func CreateVPC(vpcName string) error { - existingVPC, err := IsVPCExist(vpcName) - if err != nil { - return err - } - if !existingVPC { - cmd := exec.Command("ibmcloud", "is", "vpc-create", vpcName) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("VPC creation failed: %v\nCommand output: %s", err, string(output)) - } - fmt.Printf("VPC %s created successfully\n", vpcName) - } else { - fmt.Println("An existing VPC is available") - } - return nil -} - -// Prerequisite: Ensure that you have logged into IBM Cloud using the LoginIntoIBMCloudUsingCLI function before calling this function -// IsVPCExist checks if a VPC with the given name exists. -func IsVPCExist(vpcName string) (bool, error) { - cmd := exec.Command("ibmcloud", "is", "vpcs", "--output", "json") - output, err := cmd.Output() - if err != nil { - return false, fmt.Errorf("failed to list VPCs: %w", err) - } - - return bytes.Contains(output, []byte(vpcName)), nil -} - -// GetRegion returns the region from a given zone. -func GetRegion(zone string) string { - // Extract region from zone - region := zone[:len(zone)-2] - return region -} - -// SplitAndTrim splits the input string into a list of trimmed strings based on the comma separator. -func SplitAndTrim(str, separator string) []string { - words := strings.Split(str, separator) - var trimmedWords []string - - for _, word := range words { - trimmedWord := strings.TrimSpace(word) - if trimmedWord != "" { - trimmedWords = append(trimmedWords, trimmedWord) - } - } - return trimmedWords -} - -// RemoveKeys removes the key-value pairs with the specified keys from the given map. -func RemoveKeys(m map[string]interface{}, keysToRemove []string) { - for _, key := range keysToRemove { - delete(m, key) - } -} - -// GetBastionServerIP retrieves the IP address from the BastionServer section in the specified INI file. -func GetBastionServerIP(t *testing.T, filePath string, logger *AggregatedLogger) (string, error) { - value, err := GetValueFromIniFile(filePath+"/bastion.ini", "BastionServer") - if err != nil { - return "", fmt.Errorf("failed to get value from bastion.ini: %w", err) - } - logger.Info(t, fmt.Sprintf("Bastion Server IP: %s", value[1])) - return value[1], nil -} - -// GetManagementNodeIPs retrieves the IP addresses from the HPCAASCluster section in the specified INI file. -func GetManagementNodeIPs(t *testing.T, filePath string, logger *AggregatedLogger) ([]string, error) { - value, err := GetValueFromIniFile(filePath+"/compute.ini", "HPCAASCluster") - if err != nil { - return nil, fmt.Errorf("failed to get value from compute.ini: %w", err) - } - logger.Info(t, fmt.Sprintf("Management Node IPs List: %q", value[1:])) - return value[1:], nil -} - -// GetLoginNodeIP retrieves the IP address from the LoginServer section in the specified login INI file. -func GetLoginNodeIP(t *testing.T, filePath string, logger *AggregatedLogger) (string, error) { - value, err := GetValueFromIniFile(filePath+"/login.ini", "LoginServer") - if err != nil { - return "", fmt.Errorf("failed to get value from login.ini: %w", err) - } - logger.Info(t, fmt.Sprintf("Login Server IP: %s", value[1])) - return value[1], nil -} - -// GetLdapServerIP retrieves the IP address from the LdapServer section in the specified login INI file. -func GetLdapServerIP(t *testing.T, filePath string, logger *AggregatedLogger) (string, error) { - value, err := GetValueFromIniFile(filePath+"/ldap.ini", "LDAPServer") - if err != nil { - return "", fmt.Errorf("failed to get value from ldap.ini: %w", err) - } - logger.Info(t, fmt.Sprintf("Ldap Server IP: %s", value[1])) - return value[1], nil -} - -// GetServerIPs retrieves the IP addresses of the bastion server, management nodes, and login node -// from the specified file path in the provided test options, using the provided logger for logging. -// It returns the bastion server IP, a list of management node IPs, the login node IP, and any error encountered. -func GetServerIPs(t *testing.T, options *testhelper.TestOptions, logger *AggregatedLogger) (bastionIP string, managementNodeIPList []string, loginNodeIP string, err error) { - - filePath := options.TerraformOptions.TerraformDir - - // Get bastion server IP and handle errors - bastionIP, err = GetBastionServerIP(t, filePath, logger) - if err != nil { - return "", nil, "", fmt.Errorf("error getting bastion server IP: %v", err) - } - - // Get management node IPs and handle errors - managementNodeIPList, err = GetManagementNodeIPs(t, filePath, logger) - if err != nil { - return "", nil, "", fmt.Errorf("error getting management node IPs: %v", err) - } - - // Get login node IP and handle errors - loginNodeIP, err = GetLoginNodeIP(t, filePath, logger) - if err != nil { - return "", nil, "", fmt.Errorf("error getting login node IP: %v", err) - } - - return bastionIP, managementNodeIPList, loginNodeIP, nil -} - -// GetServerIPsWithLDAP retrieves the IP addresses of various servers, including the LDAP server. -// from the specified file path in the provided test options, using the provided logger for logging. -// It returns the bastion server IP, a list of management node IPs, the login node IP, ldap server IP and any error encountered. -func GetServerIPsWithLDAP(t *testing.T, options *testhelper.TestOptions, logger *AggregatedLogger) (bastionIP string, managementNodeIPList []string, loginNodeIP, ldapIP string, err error) { - // Retrieve the Terraform directory from the options. - filePath := options.TerraformOptions.TerraformDir - - // Get the bastion server IP and handle errors. - bastionIP, err = GetBastionServerIP(t, filePath, logger) - if err != nil { - return "", nil, "", "", fmt.Errorf("error getting bastion server IP: %v", err) - } - - // Get the management node IPs and handle errors. - managementNodeIPList, err = GetManagementNodeIPs(t, filePath, logger) - if err != nil { - return "", nil, "", "", fmt.Errorf("error getting management node IPs: %v", err) - } - - // Get the login node IP and handle errors. - loginNodeIP, err = GetLoginNodeIP(t, filePath, logger) - if err != nil { - return "", nil, "", "", fmt.Errorf("error getting login node IP: %v", err) - } - - // Get the LDAP server IP and handle errors. - ldapIP, err = GetLdapServerIP(t, filePath, logger) - if err != nil { - return "", nil, "", "", fmt.Errorf("error getting LDAP server IP: %v", err) - } - - // Return the retrieved IP addresses and any error. - return bastionIP, managementNodeIPList, loginNodeIP, ldapIP, nil -} - -// GenerateTimestampedClusterPrefix generates a cluster prefix by appending a timestamp to the given prefix. -func GenerateTimestampedClusterPrefix(prefix string) string { - //Place current time in the string. - t := time.Now() - return strings.ToLower("cicd" + "-" + t.Format(TimeLayout) + "-" + prefix) - -} - -// GetPublicIP returns the public IP address using ifconfig.io API -func GetPublicIP() (string, error) { - cmd := exec.Command("bash", "-c", "(curl -s ifconfig.io)") - output, err := cmd.CombinedOutput() - if err != nil { - return "", err - } - return strings.TrimSpace(string(output)), nil -} - -// GetOrDefault returns the environment variable value if it's not empty, otherwise returns the default value. -func GetOrDefault(envVar, defaultValue string) string { - if envVar != "" { - return envVar - } - return defaultValue -} - -// GenerateRandomString generates a random string of length 4 using lowercase characters -func GenerateRandomString() string { - // Define the character set containing lowercase letters - charset := "abcdefghijklmnopqrstuvwxyz" - - b := make([]byte, 4) - - // Loop through each index of the byte slice - for i := range b { - // Generate a random index within the length of the character set - randomIndex := rand.Intn(len(charset)) - - b[i] = charset[randomIndex] - } - - // Convert the byte slice to a string and return it - return string(b) -} - -// GetSecretsManagerKey retrieves a secret from IBM Secrets Manager. -func GetSecretsManagerKey(smID, smRegion, smKeyID string) (*string, error) { - secretsManagerService, err := secretsmanagerv2.NewSecretsManagerV2(&secretsmanagerv2.SecretsManagerV2Options{ - URL: fmt.Sprintf("https://%s.%s.secrets-manager.appdomain.cloud", smID, smRegion), - Authenticator: &core.IamAuthenticator{ - ApiKey: os.Getenv("TF_VAR_ibmcloud_api_key"), - }, - }) - if err != nil { - return nil, err - } - - getSecretOptions := secretsManagerService.NewGetSecretOptions(smKeyID) - - secret, _, err := secretsManagerService.GetSecret(getSecretOptions) - if err != nil { - return nil, err - } - - secretPayload, ok := secret.(*secretsmanagerv2.ArbitrarySecret) - if !ok { - return nil, fmt.Errorf("unexpected secret type: %T", secret) - } - - return secretPayload.Payload, nil -} - -// GetValueForKey retrieves the value associated with the specified key from the given map. -func GetValueForKey(inputMap map[string]string, key string) string { - return inputMap[key] -} - -// Getting BastionID and ComputeID from separately created brand new VPC -func GetSubnetIds(outputs map[string]interface{}) (string, string) { - subnetDetailsList := outputs["subnet_detail_list"] - var bastion []string - var compute []string - for _, val := range subnetDetailsList.(map[string]interface{}) { - for key1, val1 := range val.(map[string]interface{}) { - isContains := strings.Contains(key1, "bastion") - if isContains { - for key2, val2 := range val1.(map[string]interface{}) { - if key2 == "id" { - bastion = append(bastion, val2.(string)) - } - } - } else { - for key2, val2 := range val1.(map[string]interface{}) { - if key2 == "id" { - compute = append(compute, val2.(string)) - } - } - } - - } - } - bastionsubnetId := strings.Join(bastion, ",") - computesubnetIds := strings.Join(compute, ",") - return bastionsubnetId, computesubnetIds -} - -// Getting DNSInstanceID and CustomResolverID from separately created brand new VPC -func GetDnsCustomResolverIds(outputs map[string]interface{}) (string, string) { - customResolverHub := outputs["custom_resolver_hub"] - var instanceId string - var customResolverId string - for _, details := range customResolverHub.([]interface{}) { - for key, val := range details.(map[string]interface{}) { - if key == "instance_id" { - instanceId = val.(string) - } - if key == "custom_resolver_id" { - customResolverId = val.(string) - } - } - } - return instanceId, customResolverId -} - -// Configuration struct matches the structure of your JSON data -type Configuration struct { - ClusterID string `json:"ClusterID"` - ReservationID string `json:"ReservationID"` - ClusterPrefixName string `json:"ClusterPrefixName"` - ResourceGroup string `json:"ResourceGroup"` - KeyManagement string `json:"KeyManagement"` - DnsDomainName string `json:"DnsDomainName"` - Zones string `json:"Zones"` - HyperthreadingEnabled bool `json:"HyperthreadingEnabled"` - BastionIP string `json:"bastionIP"` - ManagementNodeIPList []string `json:"managementNodeIPList"` - LoginNodeIP string `json:"loginNodeIP"` - LdapServerIP string `json:"LdapServerIP"` - LdapDomain string `json:"LdapDomain"` - LdapAdminPassword string `json:"LdapAdminPassword"` - LdapUserName string `json:"LdapUserName"` - LdapUserPassword string `json:"LdapUserPassword"` - AppCenterEnabledOrNot string `json:"APPCenterEnabledOrNot"` - SshKeyPath string `json:"ssh_key_path"` -} - -// ParseConfig reads a JSON file from the given file path and parses it into a Configuration struct -func ParseConfig(filePath string) (*Configuration, error) { - // Read the entire content of the file - byteValue, err := os.ReadFile(filePath) - if err != nil { - return nil, fmt.Errorf("error reading file %s: %w", filePath, err) - } - - // Unmarshal the JSON data into the Configuration struct - var config Configuration - err = json.Unmarshal(byteValue, &config) - if err != nil { - return nil, fmt.Errorf("error parsing JSON from file %s: %w", filePath, err) - } - - // Return the configuration struct and nil error on success - return &config, nil -} - -// GetClusterSecurityID retrieves the security group ID for a cluster based on the provided parameters. -// It logs in to IBM Cloud, executes a command to find the security group ID associated with the cluster prefix, -// and returns the security group ID or an error if any step fails. -func GetClusterSecurityID(t *testing.T, apiKey, region, resourceGroup, clusterPrefix string, logger *AggregatedLogger) (securityGroupID string, err error) { - // If the resource group is "null", set a custom resource group based on the cluster prefix. - if strings.Contains(resourceGroup, "null") { - resourceGroup = fmt.Sprintf("%s-workload-rg", clusterPrefix) - } - - // Log in to IBM Cloud using the API key, region, and resource group. - if err := LoginIntoIBMCloudUsingCLI(t, apiKey, region, resourceGroup); err != nil { - return "", fmt.Errorf("failed to log in to IBM Cloud: %w", err) - } - - // Determine the command to get the security group ID based on the cluster prefix. - cmd := fmt.Sprintf("ibmcloud is security-groups | grep %s-cluster-sg | awk '{print $1}'", clusterPrefix) - - // Execute the command to retrieve the security group ID. - output, err := exec.Command("bash", "-c", cmd).CombinedOutput() - if err != nil { - return "", fmt.Errorf("failed to retrieve security group ID: %w", err) - } - - // Trim and check if the result is empty. - securityGroupID = strings.TrimSpace(string(output)) - if securityGroupID == "" { - return "", fmt.Errorf("no security group ID found for cluster prefix %s", clusterPrefix) - } - - logger.Info(t, "securityGroupID: "+securityGroupID) - - return securityGroupID, nil -} - -// UpdateSecurityGroupRules updates the security group with specified port and CIDR based on the provided parameters. -// It logs in to IBM Cloud, determines the appropriate command, and executes it to update the security group. -// Returns an error if any step fails. -func UpdateSecurityGroupRules(t *testing.T, apiKey, region, resourceGroup, clusterPrefix, securityGroupId, cidr, minPort, maxPort string, logger *AggregatedLogger) (err error) { - // If the resource group is "null", set a custom resource group based on the cluster prefix. - if strings.Contains(resourceGroup, "null") { - resourceGroup = fmt.Sprintf("%s-workload-rg", clusterPrefix) - } - - // Log in to IBM Cloud using the API key, region, and resource group. - if err := LoginIntoIBMCloudUsingCLI(t, apiKey, region, resourceGroup); err != nil { - return fmt.Errorf("failed to log in to IBM Cloud: %w", err) - } - - // Determine the command to add a rule to the security group with the specified port and CIDR. - addRuleCmd := fmt.Sprintf("ibmcloud is security-group-rule-add %s inbound tcp --remote %s --port-min %s --port-max %s", securityGroupId, cidr, minPort, maxPort) - - // Execute the command to update the security group. - output, err := exec.Command("bash", "-c", addRuleCmd).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to update security group with port and CIDR: %w", err) - } - - logger.Info(t, "security group updated output: "+strings.TrimSpace(string(output))) - - // Verify if the output contains the expected CIDR. - if !VerifyDataContains(t, strings.TrimSpace(string(output)), cidr, logger) { - return fmt.Errorf("failed to update security group CIDR: %s", string(output)) - } - - // Verify if the output contains the expected minimum port. - if !VerifyDataContains(t, strings.TrimSpace(string(output)), minPort, logger) { - return fmt.Errorf("failed to update security group port: %s", string(output)) - } - - return nil -} - -// GetCustomResolverID retrieves the custom resolver ID for a VPC based on the provided cluster prefix. -// It logs in to IBM Cloud, retrieves the DNS instance ID, and then fetches the custom resolver ID. -// Returns the custom resolver ID and any error encountered. -func GetCustomResolverID(t *testing.T, apiKey, region, resourceGroup, clusterPrefix string, logger *AggregatedLogger) (customResolverID string, err error) { - // If the resource group is "null", set a custom resource group based on the cluster prefix. - if strings.Contains(resourceGroup, "null") { - resourceGroup = fmt.Sprintf("%s-workload-rg", clusterPrefix) - } - - // Log in to IBM Cloud using the API key, region, and resource group. - if err := LoginIntoIBMCloudUsingCLI(t, apiKey, region, resourceGroup); err != nil { - return "", fmt.Errorf("failed to log in to IBM Cloud: %w", err) - } - - // Command to get the DNS instance ID based on the cluster prefix. - dnsInstanceCmd := fmt.Sprintf("ibmcloud dns instances | grep %s | awk '{print $2}'", clusterPrefix) - dnsInstanceIDOutput, err := exec.Command("bash", "-c", dnsInstanceCmd).CombinedOutput() - if err != nil { - return "", fmt.Errorf("failed to retrieve DNS instance ID: %w", err) - } - - // Trim whitespace and check if we received a valid DNS instance ID. - dnsInstanceID := strings.TrimSpace(string(dnsInstanceIDOutput)) - if dnsInstanceID == "" { - return "", fmt.Errorf("no DNS instance ID found for cluster prefix %s", clusterPrefix) - } - - // Command to get custom resolvers for the DNS instance ID. - customResolverCmd := fmt.Sprintf("ibmcloud dns custom-resolvers -i %s | awk 'NR>3 {print $1}'", dnsInstanceID) - customResolverIDOutput, err := exec.Command("bash", "-c", customResolverCmd).CombinedOutput() - if err != nil { - return "", fmt.Errorf("failed to retrieve custom resolver ID: %w", err) - } - - // Trim whitespace and check if we received a valid custom resolver ID. - customResolverID = strings.TrimSpace(string(customResolverIDOutput)) - if customResolverID == "" { - return "", fmt.Errorf("no custom resolver ID found for DNS instance ID %s", dnsInstanceID) - } - logger.Info(t, "customResolverID: "+customResolverID) - - return customResolverID, nil -} - -// RetrieveAndUpdateSecurityGroup retrieves the security group ID based on the provided cluster prefix, -// then updates the security group with the specified port and CIDR. -// It logs in to IBM Cloud, determines the appropriate commands, and executes them. -// Returns an error if any step fails. -func RetrieveAndUpdateSecurityGroup(t *testing.T, apiKey, region, resourceGroup, clusterPrefix, cidr, minPort, maxPort string, logger *AggregatedLogger) error { - // If the resource group is "null", set a custom resource group based on the cluster prefix. - if strings.Contains(resourceGroup, "null") { - resourceGroup = fmt.Sprintf("%s-workload-rg", clusterPrefix) - } - - // Log in to IBM Cloud using the API key, region, and resource group. - if err := LoginIntoIBMCloudUsingCLI(t, apiKey, region, resourceGroup); err != nil { - return fmt.Errorf("failed to log in to IBM Cloud: %w", err) - } - - // Command to get the security group ID based on the cluster prefix. - getSecurityGroupIDCmd := fmt.Sprintf("ibmcloud is security-groups | grep %s-cluster-sg | awk '{print $1}'", clusterPrefix) - securityGroupIDBytes, err := exec.Command("bash", "-c", getSecurityGroupIDCmd).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to retrieve security group ID: %w", err) - } - - securityGroupID := strings.TrimSpace(string(securityGroupIDBytes)) - if securityGroupID == "" { - return fmt.Errorf("no security group ID found for cluster prefix %s", clusterPrefix) - } - - logger.Info(t, "securityGroupID: "+securityGroupID) - - // Command to add a rule to the security group with the specified port and CIDR. - addRuleCmd := fmt.Sprintf("ibmcloud is security-group-rule-add %s inbound tcp --remote %s --port-min %s --port-max %s", securityGroupID, cidr, minPort, maxPort) - outputBytes, err := exec.Command("bash", "-c", addRuleCmd).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to update security group with port and CIDR: %w", err) - } - - output := strings.TrimSpace(string(outputBytes)) - logger.Info(t, "security group updated output: "+output) - - // Combine output verification steps. - if !VerifyDataContains(t, output, cidr, logger) || !VerifyDataContains(t, output, minPort, logger) { - return fmt.Errorf("failed to update security group with CIDR %s and port %s: %s", cidr, minPort, output) - } - - return nil -} - -// GetLdapIP retrieves the IP addresses of various servers, including the LDAP server, -// from the specified file path in the provided test options, using the provided logger for logging. -// It returns the LDAP server IP and any error encountered. -func GetLdapIP(t *testing.T, options *testhelper.TestOptions, logger *AggregatedLogger) (ldapIP string, err error) { - // Retrieve the Terraform directory from the options. - filePath := options.TerraformOptions.TerraformDir - - // Get the LDAP server IP and handle errors. - ldapIP, err = GetLdapServerIP(t, filePath, logger) - if err != nil { - return "", fmt.Errorf("error getting LDAP server IP: %w", err) - } - - // Return the retrieved IP address and any error. - return ldapIP, nil -} - -// GetBastionIP retrieves the bastion server IP address based on the provided test options. -// It returns the bastion IP address and an error if any step fails. -func GetBastionIP(t *testing.T, options *testhelper.TestOptions, logger *AggregatedLogger) (bastionIP string, err error) { - // Retrieve the Terraform directory from the options. - filePath := options.TerraformOptions.TerraformDir - - // Get the bastion server IP and handle errors. - bastionIP, err = GetBastionServerIP(t, filePath, logger) - if err != nil { - return "", fmt.Errorf("error getting bastion server IP: %w", err) - } - - // Return the bastion IP address. - return bastionIP, nil -} diff --git a/tests/test_config.yml b/tests/config.yml similarity index 86% rename from tests/test_config.yml rename to tests/config.yml index d7baf9e3..4e70c289 100644 --- a/tests/test_config.yml +++ b/tests/config.yml @@ -6,9 +6,9 @@ reservation_id: remote_allowed_ips: ssh_key: geretain-hpc login_node_instance_type: bx2-2x8 -login_image_name: hpcaas-lsf10-rhel88-compute-v5 -management_image_name: hpcaas-lsf10-rhel88-v9 -compute_image_name: hpcaas-lsf10-rhel88-compute-v5 +login_image_name: hpcaas-lsf10-rhel88-compute-v6 +management_image_name: hpcaas-lsf10-rhel88-v10 +compute_image_name: hpcaas-lsf10-rhel88-compute-v6 management_node_instance_type: bx2-2x8 management_node_count: 2 enable_vpc_flow_logs: false diff --git a/tests/test_output/output.txt b/tests/logs/output.txt similarity index 100% rename from tests/test_output/output.txt rename to tests/logs/output.txt diff --git a/tests/lsf/lsf_cluster_test_utils.go b/tests/lsf/cluster_helpers.go similarity index 97% rename from tests/lsf/lsf_cluster_test_utils.go rename to tests/lsf/cluster_helpers.go index dc0a084e..067459b5 100644 --- a/tests/lsf/lsf_cluster_test_utils.go +++ b/tests/lsf/cluster_helpers.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/common_utils" + utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/utilities" "golang.org/x/crypto/ssh" ) @@ -19,7 +19,6 @@ func VerifyManagementNodeConfig( expectedClusterID, expectedMasterName, expectedReservationID string, expectedHyperthreadingStatus bool, managementNodeIPList []string, - jobCommand string, lsfVersion string, logger *utils.AggregatedLogger, ) { @@ -56,9 +55,6 @@ func VerifyManagementNodeConfig( fileMountErr := HPCCheckFileMount(t, sshMgmtClient, managementNodeIPList, "management", logger) utils.LogVerificationResult(t, fileMountErr, "File mount check on management node", logger) - //Run job - jobErr := LSFRunJobs(t, sshMgmtClient, jobCommand, logger) - utils.LogVerificationResult(t, jobErr, "check Run job on management node", logger) } // VerifySSHKey verifies SSH keys for both management and compute nodes. @@ -112,21 +108,18 @@ func FailoverAndFailback(t *testing.T, sshMgmtClient *ssh.Client, jobCommand str // RestartLsfDaemon restarts the LSF (Load Sharing Facility) daemons. // It logs verification results using the provided logger. -func RestartLsfDaemon(t *testing.T, sshMgmtClient *ssh.Client, jobCommand string, logger *utils.AggregatedLogger) { +func RestartLsfDaemon(t *testing.T, sshMgmtClient *ssh.Client, logger *utils.AggregatedLogger) { //Restart Daemons restartDaemonErr := LSFRestartDaemons(t, sshMgmtClient, logger) utils.LogVerificationResult(t, restartDaemonErr, "check lsf_daemons restart", logger) - //Run job - jobErr := LSFRunJobs(t, sshMgmtClient, jobCommand, logger) - utils.LogVerificationResult(t, jobErr, "check Run job", logger) } // RebootInstance reboots an instance in a cluster using LSF (Load Sharing Facility). // It performs instance reboot, establishes a new SSH connection to the master node, // checks the bhosts response, and logs verification results using the provided logger. -func RebootInstance(t *testing.T, sshMgmtClient *ssh.Client, publicHostIP, publicHostName, privateHostName, managementNodeIP string, jobCommand string, logger *utils.AggregatedLogger) { +func RebootInstance(t *testing.T, sshMgmtClient *ssh.Client, publicHostIP, publicHostName, privateHostName, managementNodeIP string, logger *utils.AggregatedLogger) { //Reboot the management node one rebootErr := LSFRebootInstance(t, sshMgmtClient, logger) @@ -140,10 +133,6 @@ func RebootInstance(t *testing.T, sshMgmtClient *ssh.Client, publicHostIP, publi bhostRespErr := LSFCheckBhostsResponse(t, sshClient, logger) utils.LogVerificationResult(t, bhostRespErr, "bhosts response non-empty", logger) - //Run job - jobErr := LSFRunJobs(t, sshClient, jobCommand, logger) - utils.LogVerificationResult(t, jobErr, "check Run job", logger) - defer sshClient.Close() } @@ -290,6 +279,8 @@ func VerifyNoVNCConfig( // VerifyJobs verifies LSF job execution, logging any errors. func VerifyJobs(t *testing.T, sshClient *ssh.Client, jobCommand string, logger *utils.AggregatedLogger) { + + //Run job jobErr := LSFRunJobs(t, sshClient, jobCommand, logger) utils.LogVerificationResult(t, jobErr, "check Run job", logger) @@ -340,7 +331,7 @@ func VerifyManagementNodeLDAPConfig( lsfCmdErr := VerifyLSFCommandsAsLDAPUser(t, sshLdapClient, ldapUserName, "management", logger) utils.LogVerificationResult(t, lsfCmdErr, "Check the 'lsf' command as an LDAP user on the management node", logger) - // Run job + // Run job as ldap user jobErr := LSFRunJobsAsLDAPUser(t, sshLdapClient, jobCommand, ldapUserName, logger) utils.LogVerificationResult(t, jobErr, "check Run job as an LDAP user on the management node", logger) @@ -389,7 +380,7 @@ func VerifyLoginNodeLDAPConfig( fileMountErr := HPCCheckFileMountAsLDAPUser(t, sshLdapClient, "login", logger) utils.LogVerificationResult(t, fileMountErr, "check file mount as an LDAP user on the login node", logger) - // Run job + // Run job as ldap user jobErr := LSFRunJobsAsLDAPUser(t, sshLdapClient, LOGIN_NODE_EXECUTION_PATH+jobCommand, ldapUserName, logger) utils.LogVerificationResult(t, jobErr, "check Run job as an LDAP user on the login node", logger) diff --git a/tests/lsf/lsf_cluster_utils.go b/tests/lsf/cluster_utils.go similarity index 99% rename from tests/lsf/lsf_cluster_utils.go rename to tests/lsf/cluster_utils.go index 8e677239..0f5e754c 100644 --- a/tests/lsf/lsf_cluster_utils.go +++ b/tests/lsf/cluster_utils.go @@ -13,7 +13,7 @@ import ( "testing" "time" - utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/common_utils" + utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/utilities" "golang.org/x/crypto/ssh" ) diff --git a/tests/lsf/lsf_cluster_test_validation.go b/tests/lsf/cluster_validation.go similarity index 80% rename from tests/lsf/lsf_cluster_test_validation.go rename to tests/lsf/cluster_validation.go index fd95cf74..8c26bd50 100644 --- a/tests/lsf/lsf_cluster_test_validation.go +++ b/tests/lsf/cluster_validation.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testhelper" - utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/common_utils" + utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/utilities" ) // ValidateClusterConfiguration performs comprehensive validation on the cluster setup. @@ -34,8 +34,8 @@ func ValidateClusterConfiguration(t *testing.T, options *testhelper.TestOptions, expectedHyperthreadingEnabled, err := strconv.ParseBool(options.TerraformVars["hyperthreading_enabled"].(string)) require.NoError(t, err, "Error parsing hyperthreading_enabled: %v", err) - JOB_COMMAND_LOW := GetJobCommand(expectedZone, "low") - JOB_COMMAND_MED := GetJobCommand(expectedZone, "med") + jobCommandLow := GetJobCommand(expectedZone, "low") + jobCommandMed := GetJobCommand(expectedZone, "med") // Run the test consistency check output, err := options.RunTestConsistency() @@ -54,14 +54,14 @@ func ValidateClusterConfiguration(t *testing.T, options *testhelper.TestOptions, // Connect to the master node via SSH and handle connection errors sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the master via SSH") defer sshClient.Close() testLogger.Info(t, "SSH connection to the master successful") t.Log("Validation in progress. Please wait...") // Verify management node configuration - VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, EXPECTED_LSF_VERSION, testLogger) // Verify SSH key on management nodes VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "management", managementNodeIPList, expectedNumOfKeys, testLogger) @@ -70,17 +70,17 @@ func ValidateClusterConfiguration(t *testing.T, options *testhelper.TestOptions, VerifyLSFDNS(t, sshClient, managementNodeIPList, expectedDnsDomainName, testLogger) // Perform failover and failback - FailoverAndFailback(t, sshClient, JOB_COMMAND_MED, testLogger) + FailoverAndFailback(t, sshClient, jobCommandMed, testLogger) // Restart LSF daemon - RestartLsfDaemon(t, sshClient, JOB_COMMAND_LOW, testLogger) + RestartLsfDaemon(t, sshClient, testLogger) // Reboot instance - RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], JOB_COMMAND_MED, testLogger) + RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], testLogger) // Reconnect to the management node after reboot sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH") defer sshClient.Close() // Wait for dynamic node disappearance and handle potential errors @@ -90,9 +90,12 @@ func ValidateClusterConfiguration(t *testing.T, options *testhelper.TestOptions, } }() + // Run job + VerifyJobs(t, sshClient, jobCommandLow, testLogger) + // Get dynamic compute node IPs and handle errors computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) - require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs") // Verify compute node configuration VerifyComputeNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) @@ -105,15 +108,15 @@ func ValidateClusterConfiguration(t *testing.T, options *testhelper.TestOptions, // Verify SSH connectivity from login node and handle connection errors sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) - require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH") defer sshLoginNodeClient.Close() // Verify login node configuration - VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, jobCommandLow, EXPECTED_LSF_VERSION, testLogger) // Get dynamic compute node IPs and handle errors computeNodeIPList, computeIPErr = LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) - require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs") // Verify SSH connectivity from login node VerifySSHConnectivityToNodesFromLogin(t, sshLoginNodeClient, managementNodeIPList, computeNodeIPList, testLogger) @@ -159,8 +162,9 @@ func ValidateClusterConfigurationWithAPPCenter(t *testing.T, options *testhelper expectedHyperthreadingEnabled, err := strconv.ParseBool(options.TerraformVars["hyperthreading_enabled"].(string)) require.NoError(t, err, "Error parsing hyperthreading_enabled: %v", err) - JOB_COMMAND_LOW := GetJobCommand(expectedZone, "low") - JOB_COMMAND_MED := GetJobCommand(expectedZone, "med") + // Retrieve job commands for different levels + jobCommandLow := GetJobCommand(expectedZone, "low") + jobCommandMed := GetJobCommand(expectedZone, "med") // Run the test consistency check output, err := options.RunTestConsistency() @@ -179,14 +183,14 @@ func ValidateClusterConfigurationWithAPPCenter(t *testing.T, options *testhelper // Connect to the master node via SSH and handle connection errors sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the master via SSH") defer sshClient.Close() testLogger.Info(t, "SSH connection to the master successful") t.Log("Validation in progress. Please wait...") // Verify management node configuration - VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, EXPECTED_LSF_VERSION, testLogger) // Verify SSH key on management nodes VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "management", managementNodeIPList, expectedNumOfKeys, testLogger) @@ -195,17 +199,17 @@ func ValidateClusterConfigurationWithAPPCenter(t *testing.T, options *testhelper VerifyLSFDNS(t, sshClient, managementNodeIPList, expectedDnsDomainName, testLogger) // Perform failover and failback - FailoverAndFailback(t, sshClient, JOB_COMMAND_MED, testLogger) + FailoverAndFailback(t, sshClient, jobCommandMed, testLogger) // Restart LSF daemon - RestartLsfDaemon(t, sshClient, JOB_COMMAND_LOW, testLogger) + RestartLsfDaemon(t, sshClient, testLogger) // Reboot instance - RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], JOB_COMMAND_MED, testLogger) + RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], testLogger) // Reconnect to the management node after reboot sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH") defer sshClient.Close() // Wait for dynamic node disappearance and handle potential errors @@ -215,9 +219,12 @@ func ValidateClusterConfigurationWithAPPCenter(t *testing.T, options *testhelper } }() + // Run job + VerifyJobs(t, sshClient, jobCommandLow, testLogger) + // Get dynamic compute node IPs and handle errors computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) - require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs") // Verify compute node configuration VerifyComputeNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) @@ -230,15 +237,15 @@ func ValidateClusterConfigurationWithAPPCenter(t *testing.T, options *testhelper // Verify SSH connectivity from login node and handle connection errors sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) - require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH") defer sshLoginNodeClient.Close() // Verify login node configuration - VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, jobCommandLow, EXPECTED_LSF_VERSION, testLogger) // Get dynamic compute node IPs and handle errors computeNodeIPList, computeIPErr = LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) - require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs") // Verify SSH connectivity from login node VerifySSHConnectivityToNodesFromLogin(t, sshLoginNodeClient, managementNodeIPList, computeNodeIPList, testLogger) @@ -279,7 +286,8 @@ func ValidateBasicClusterConfiguration(t *testing.T, options *testhelper.TestOpt expectedHyperthreadingEnabled, err := strconv.ParseBool(options.TerraformVars["hyperthreading_enabled"].(string)) require.NoError(t, err, "Error parsing hyperthreading_enabled: %v", err) - JOB_COMMAND_LOW := GetJobCommand(expectedZone, "low") + // Retrieve job commands for different levels + jobCommandLow := GetJobCommand(expectedZone, "low") // Run the test consistency check output, err := options.RunTestConsistency() @@ -298,19 +306,14 @@ func ValidateBasicClusterConfiguration(t *testing.T, options *testhelper.TestOpt // Connect to the master node via SSH and handle connection errors sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the master via SSH") defer sshClient.Close() testLogger.Info(t, "SSH connection to the master successful") t.Log("Validation in progress. Please wait...") // Verify management node configuration - VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) - - // Reconnect to the management node after reboot - sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH: %v", connectionErr) - defer sshClient.Close() + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, EXPECTED_LSF_VERSION, testLogger) // Wait for dynamic node disappearance and handle potential errors defer func() { @@ -319,20 +322,23 @@ func ValidateBasicClusterConfiguration(t *testing.T, options *testhelper.TestOpt } }() + // Run job + VerifyJobs(t, sshClient, jobCommandLow, testLogger) + // Get dynamic compute node IPs and handle errors computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) - require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs") // Verify compute node configuration VerifyComputeNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) // Verify SSH connectivity from login node and handle connection errors sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) - require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH") defer sshLoginNodeClient.Close() // Verify login node configuration - VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, jobCommandLow, EXPECTED_LSF_VERSION, testLogger) // Verify file share encryption VerifyFileShareEncryption(t, os.Getenv("TF_VAR_ibmcloud_api_key"), utils.GetRegion(expectedZone), expectedResourceGroup, expectedMasterName, expectedKeyManagement, testLogger) @@ -365,8 +371,10 @@ func ValidateLDAPClusterConfiguration(t *testing.T, options *testhelper.TestOpti expectedHyperthreadingEnabled, err := strconv.ParseBool(options.TerraformVars["hyperthreading_enabled"].(string)) require.NoError(t, err, "Error parsing hyperthreading_enabled: %v", err) - JOB_COMMAND_LOW := GetJobCommand(expectedZone, "low") - JOB_COMMAND_MED := GetJobCommand(expectedZone, "med") + + // Retrieve job commands for different levels + jobCommandLow := GetJobCommand(expectedZone, "low") + jobCommandMed := GetJobCommand(expectedZone, "med") // Run the test consistency check output, err := options.RunTestConsistency() @@ -385,30 +393,30 @@ func ValidateLDAPClusterConfiguration(t *testing.T, options *testhelper.TestOpti // Connect to the master node via SSH and handle connection errors sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the master via SSH") defer sshClient.Close() testLogger.Info(t, "SSH connection to the master successful") t.Log("Validation in progress. Please wait...") // Verify management node configuration - VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, EXPECTED_LSF_VERSION, testLogger) // Verify SSH key on management nodes VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "management", managementNodeIPList, expectedNumOfKeys, testLogger) // Perform failover and failback - FailoverAndFailback(t, sshClient, JOB_COMMAND_MED, testLogger) + FailoverAndFailback(t, sshClient, jobCommandMed, testLogger) // Restart LSF daemon - RestartLsfDaemon(t, sshClient, JOB_COMMAND_LOW, testLogger) + RestartLsfDaemon(t, sshClient, testLogger) // Reboot instance - RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], JOB_COMMAND_MED, testLogger) + RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], testLogger) // Reconnect to the management node after reboot sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH") defer sshClient.Close() // Wait for dynamic node disappearance and handle potential errors @@ -418,9 +426,12 @@ func ValidateLDAPClusterConfiguration(t *testing.T, options *testhelper.TestOpti } }() + // Run job + VerifyJobs(t, sshClient, jobCommandLow, testLogger) + // Get dynamic compute node IPs and handle errors computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) - require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs") // Verify compute node configuration VerifyComputeNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) @@ -430,11 +441,11 @@ func ValidateLDAPClusterConfiguration(t *testing.T, options *testhelper.TestOpti // Verify SSH connectivity from login node and handle connection errors sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) - require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH") defer sshLoginNodeClient.Close() // Verify login node configuration - VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, jobCommandLow, EXPECTED_LSF_VERSION, testLogger) // Verify SSH connectivity from login node VerifySSHConnectivityToNodesFromLogin(t, sshLoginNodeClient, managementNodeIPList, computeNodeIPList, testLogger) @@ -444,23 +455,23 @@ func ValidateLDAPClusterConfiguration(t *testing.T, options *testhelper.TestOpti // Connect to the LDAP server via SSH and handle connection errors sshLdapClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_LDAP_HOST_NAME, ldapServerIP) - require.NoError(t, connectionErr, "Failed to connect to the LDAP server via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the LDAP server via SSH") defer sshLdapClient.Close() // Check LDAP server status CheckLDAPServerStatus(t, sshLdapClient, ldapAdminPassword, expectedLdapDomain, ldapUserName, testLogger) // Verify management node LDAP config - VerifyManagementNodeLDAPConfig(t, sshClient, bastionIP, ldapServerIP, managementNodeIPList, JOB_COMMAND_LOW, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) + VerifyManagementNodeLDAPConfig(t, sshClient, bastionIP, ldapServerIP, managementNodeIPList, jobCommandLow, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) // Verify compute node LDAP config VerifyComputeNodeLDAPConfig(t, bastionIP, ldapServerIP, computeNodeIPList, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) // Verify login node LDAP config - VerifyLoginNodeLDAPConfig(t, sshLoginNodeClient, bastionIP, loginNodeIP, ldapServerIP, JOB_COMMAND_LOW, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) + VerifyLoginNodeLDAPConfig(t, sshLoginNodeClient, bastionIP, loginNodeIP, ldapServerIP, jobCommandLow, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) // Verify ability to create LDAP user and perform LSF actions using new user - VerifyCreateNewLdapUserAndManagementNodeLDAPConfig(t, sshLdapClient, bastionIP, ldapServerIP, managementNodeIPList, JOB_COMMAND_LOW, ldapAdminPassword, expectedLdapDomain, ldapUserName, ldapUserPassword, "user2", testLogger) + VerifyCreateNewLdapUserAndManagementNodeLDAPConfig(t, sshLdapClient, bastionIP, ldapServerIP, managementNodeIPList, jobCommandLow, ldapAdminPassword, expectedLdapDomain, ldapUserName, ldapUserPassword, "tester2", testLogger) // Verify PTR records VerifyPTRRecordsForManagementAndLoginNodes(t, sshClient, LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList, loginNodeIP, expectedDnsDomainName, testLogger) @@ -493,8 +504,10 @@ func ValidatePACANDLDAPClusterConfiguration(t *testing.T, options *testhelper.Te expectedHyperthreadingEnabled, err := strconv.ParseBool(options.TerraformVars["hyperthreading_enabled"].(string)) require.NoError(t, err, "Error parsing hyperthreading_enabled: %v", err) - JOB_COMMAND_LOW := GetJobCommand(expectedZone, "low") - JOB_COMMAND_MED := GetJobCommand(expectedZone, "med") + + // Retrieve job commands for different levels + jobCommandLow := GetJobCommand(expectedZone, "low") + jobCommandMed := GetJobCommand(expectedZone, "med") // Run the test consistency check output, err := options.RunTestConsistency() @@ -512,30 +525,30 @@ func ValidatePACANDLDAPClusterConfiguration(t *testing.T, options *testhelper.Te // Connect to the master node via SSH and handle connection errors sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the master via SSH") defer sshClient.Close() testLogger.Info(t, "SSH connection to the master successful") t.Log("Validation in progress. Please wait...") // Verify management node configuration - VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, EXPECTED_LSF_VERSION, testLogger) // Verify SSH key on management nodes VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "management", managementNodeIPList, expectedNumOfKeys, testLogger) // Perform failover and failback - FailoverAndFailback(t, sshClient, JOB_COMMAND_MED, testLogger) + FailoverAndFailback(t, sshClient, jobCommandMed, testLogger) // Restart LSF daemon - RestartLsfDaemon(t, sshClient, JOB_COMMAND_LOW, testLogger) + RestartLsfDaemon(t, sshClient, testLogger) // Reboot instance - RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], JOB_COMMAND_MED, testLogger) + RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], testLogger) // Reconnect to the management node after reboot sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH") defer sshClient.Close() // Wait for dynamic node disappearance and handle potential errors @@ -545,9 +558,12 @@ func ValidatePACANDLDAPClusterConfiguration(t *testing.T, options *testhelper.Te } }() + // Run job + VerifyJobs(t, sshClient, jobCommandLow, testLogger) + // Get dynamic compute node IPs and handle errors computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) - require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs") // Verify compute node configuration VerifyComputeNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) @@ -557,11 +573,11 @@ func ValidatePACANDLDAPClusterConfiguration(t *testing.T, options *testhelper.Te // Verify SSH connectivity from login node and handle connection errors sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) - require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH") defer sshLoginNodeClient.Close() // Verify login node configuration - VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, jobCommandLow, EXPECTED_LSF_VERSION, testLogger) // Verify SSH connectivity from login node VerifySSHConnectivityToNodesFromLogin(t, sshLoginNodeClient, managementNodeIPList, computeNodeIPList, testLogger) @@ -577,23 +593,23 @@ func ValidatePACANDLDAPClusterConfiguration(t *testing.T, options *testhelper.Te // Connect to the LDAP server via SSH and handle connection errors sshLdapClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_LDAP_HOST_NAME, ldapServerIP) - require.NoError(t, connectionErr, "Failed to connect to the LDAP server via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the LDAP server via SSH") defer sshLdapClient.Close() // Check LDAP server status CheckLDAPServerStatus(t, sshLdapClient, ldapAdminPassword, expectedLdapDomain, ldapUserName, testLogger) // Verify management node LDAP config - VerifyManagementNodeLDAPConfig(t, sshClient, bastionIP, ldapServerIP, managementNodeIPList, JOB_COMMAND_LOW, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) + VerifyManagementNodeLDAPConfig(t, sshClient, bastionIP, ldapServerIP, managementNodeIPList, jobCommandLow, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) // Verify compute node LDAP config VerifyComputeNodeLDAPConfig(t, bastionIP, ldapServerIP, computeNodeIPList, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) // Verify login node LDAP config - VerifyLoginNodeLDAPConfig(t, sshLoginNodeClient, bastionIP, loginNodeIP, ldapServerIP, JOB_COMMAND_LOW, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) + VerifyLoginNodeLDAPConfig(t, sshLoginNodeClient, bastionIP, loginNodeIP, ldapServerIP, jobCommandLow, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) // Verify ability to create LDAP user and perform LSF actions using new user - VerifyCreateNewLdapUserAndManagementNodeLDAPConfig(t, sshLdapClient, bastionIP, ldapServerIP, managementNodeIPList, JOB_COMMAND_LOW, ldapAdminPassword, expectedLdapDomain, ldapUserName, ldapUserPassword, "user2", testLogger) + VerifyCreateNewLdapUserAndManagementNodeLDAPConfig(t, sshLdapClient, bastionIP, ldapServerIP, managementNodeIPList, jobCommandLow, ldapAdminPassword, expectedLdapDomain, ldapUserName, ldapUserPassword, "tester2", testLogger) // Verify PTR records VerifyPTRRecordsForManagementAndLoginNodes(t, sshClient, LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList, loginNodeIP, expectedDnsDomainName, testLogger) @@ -602,103 +618,6 @@ func ValidatePACANDLDAPClusterConfiguration(t *testing.T, options *testhelper.Te testLogger.Info(t, t.Name()+" Validation ended") } -// ValidateClusterConfigurationWithAPPCenterForExistingEnv validates the configuration of an existing cluster with App Center integration. -// It verifies various aspects including management node configuration, SSH keys, failover and failback, LSF daemon restart, dynamic compute node configuration, -// login node configuration, SSH connectivity, application center configuration, noVNC configuration, PTR records, and file share encryption. -// -// testLogger: *utils.AggregatedLogger - The logger for the test. -func ValidateClusterConfigurationWithAPPCenterForExistingEnv( - t *testing.T, - expectedNumOfKeys int, - bastionIP, loginNodeIP, expectedClusterID, expectedReservationID, expectedMasterName, expectedResourceGroup, - expectedKeyManagement, expectedZone, expectedDnsDomainName string, - managementNodeIPList []string, - expectedHyperthreadingEnabled bool, - testLogger *utils.AggregatedLogger, -) { - // Retrieve job commands for different levels - JOB_COMMAND_LOW := GetJobCommand(expectedZone, "low") - JOB_COMMAND_MED := GetJobCommand(expectedZone, "med") - - // Log validation start - testLogger.Info(t, t.Name()+" Validation started ......") - - // Connect to the master node via SSH - sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) - defer sshClient.Close() - - testLogger.Info(t, "SSH connection to the master successful") - t.Log("Validation in progress. Please wait...") - - // Verify management node configuration - VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) - - // Verify SSH key on management nodes - VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "management", managementNodeIPList, expectedNumOfKeys, testLogger) - - // Perform failover and failback - FailoverAndFailback(t, sshClient, JOB_COMMAND_MED, testLogger) - - // Restart LSF daemon - RestartLsfDaemon(t, sshClient, JOB_COMMAND_LOW, testLogger) - - // Reboot instance - RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], JOB_COMMAND_MED, testLogger) - - // Reconnect to the master node via SSH after reboot - sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH: %v", connectionErr) - defer sshClient.Close() - - // Wait for dynamic node disappearance and handle potential errors - defer func() { - if err := LSFWaitForDynamicNodeDisappearance(t, sshClient, testLogger); err != nil { - t.Errorf("Error in LSFWaitForDynamicNodeDisappearance: %v", err) - } - }() - - // Get dynamic compute node IPs - computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) - require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) - - // Verify compute node configuration - VerifyComputeNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) - - // Verify SSH key for compute nodes - VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "compute", computeNodeIPList, expectedNumOfKeys, testLogger) - - // Verify SSH connectivity from login node - sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) - require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) - defer sshLoginNodeClient.Close() - - // Verify login node configuration - VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) - - // Re-fetch dynamic compute node IPs - computeNodeIPList, computeIPErr = LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) - require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) - - // Verify SSH connectivity from login node - VerifySSHConnectivityToNodesFromLogin(t, sshLoginNodeClient, managementNodeIPList, computeNodeIPList, testLogger) - - // Verify application center configuration - VerifyAPPCenterConfig(t, sshClient, testLogger) - - // Verify noVNC configuration - VerifyNoVNCConfig(t, sshClient, testLogger) - - // Verify PTR records for management and login nodes - VerifyPTRRecordsForManagementAndLoginNodes(t, sshClient, LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList, loginNodeIP, expectedDnsDomainName, testLogger) - - // Verify file share encryption - VerifyFileShareEncryption(t, os.Getenv("TF_VAR_ibmcloud_api_key"), utils.GetRegion(expectedZone), expectedResourceGroup, expectedMasterName, expectedKeyManagement, testLogger) - - // Log validation end - testLogger.Info(t, t.Name()+" Validation ended") -} - // ValidateBasicClusterConfigurationWithVPCFlowLogsAndCos validates the basic cluster configuration // including VPC flow logs and COS service instance. // It performs validation tasks on essential aspects of the cluster setup, @@ -716,7 +635,8 @@ func ValidateBasicClusterConfigurationWithVPCFlowLogsAndCos(t *testing.T, option expectedHyperthreadingEnabled, _ := strconv.ParseBool(options.TerraformVars["hyperthreading_enabled"].(string)) - JOB_COMMAND_LOW := GetJobCommand(expectedZone, "low") + // Retrieve job commands for different levels + jobCommandLow := GetJobCommand(expectedZone, "low") // Run the test and handle errors output, err := options.RunTest() @@ -735,14 +655,14 @@ func ValidateBasicClusterConfigurationWithVPCFlowLogsAndCos(t *testing.T, option // Connect to the master node via SSH and handle connection errors sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the master via SSH") defer sshClient.Close() testLogger.Info(t, "SSH connection to the master successful") t.Log("Validation in progress. Please wait...") // Verify management node configuration - VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, EXPECTED_LSF_VERSION, testLogger) // Wait for dynamic node disappearance and handle potential errors defer func() { @@ -751,20 +671,23 @@ func ValidateBasicClusterConfigurationWithVPCFlowLogsAndCos(t *testing.T, option } }() + // Run job + VerifyJobs(t, sshClient, jobCommandLow, testLogger) + // Get dynamic compute node IPs computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) - require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs") // Verify compute node configuration VerifyComputeNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) // Verify SSH connectivity from login node sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) - require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH") defer sshLoginNodeClient.Close() // Verify login node configuration - VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, jobCommandLow, EXPECTED_LSF_VERSION, testLogger) // Verify file share encryption VerifyFileShareEncryption(t, os.Getenv("TF_VAR_ibmcloud_api_key"), utils.GetRegion(expectedZone), expectedResourceGroup, expectedMasterName, expectedKeyManagement, testLogger) @@ -794,8 +717,10 @@ func ValidateClusterConfigurationWithMultipleKeys(t *testing.T, options *testhel require.True(t, ok, "Key 'compute' does not exist in dns_domain_name map or dns_domain_name is not of type map[string]string") expectedHyperthreadingEnabled, _ := strconv.ParseBool(options.TerraformVars["hyperthreading_enabled"].(string)) - JOB_COMMAND_LOW := GetJobCommand(expectedZone, "low") - JOB_COMMAND_MED := GetJobCommand(expectedZone, "med") + + // Retrieve job commands for different levels + jobCommandLow := GetJobCommand(expectedZone, "low") + jobCommandMed := GetJobCommand(expectedZone, "med") // Run the test consistency check output, err := options.RunTestConsistency() @@ -814,8 +739,8 @@ func ValidateClusterConfigurationWithMultipleKeys(t *testing.T, options *testhel // Connect to the management node via SSH sshClientOne, sshClientTwo, connectionErrOne, connectionErrTwo := utils.ConnectToHostsWithMultipleUsers(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) - require.NoError(t, connectionErrOne, "Failed to connect to the master via SSH: %v", connectionErrOne) - require.NoError(t, connectionErrTwo, "Failed to connect to the master via SSH: %v", connectionErrTwo) + require.NoError(t, connectionErrOne, "Failed to connect to the master via SSH") + require.NoError(t, connectionErrTwo, "Failed to connect to the master via SSH") defer sshClientOne.Close() defer sshClientTwo.Close() @@ -823,20 +748,20 @@ func ValidateClusterConfigurationWithMultipleKeys(t *testing.T, options *testhel t.Log("Validation in progress. Please wait...") // Verify management node configuration - VerifyManagementNodeConfig(t, sshClientOne, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) - VerifyManagementNodeConfig(t, sshClientTwo, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + VerifyManagementNodeConfig(t, sshClientOne, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, EXPECTED_LSF_VERSION, testLogger) + VerifyManagementNodeConfig(t, sshClientTwo, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, EXPECTED_LSF_VERSION, testLogger) // Verify SSH key on management node VerifySSHKey(t, sshClientOne, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "management", managementNodeIPList, expectedNumOfKeys, testLogger) // Perform failover and failback - FailoverAndFailback(t, sshClientOne, JOB_COMMAND_MED, testLogger) + FailoverAndFailback(t, sshClientOne, jobCommandMed, testLogger) // Restart LSF daemon - RestartLsfDaemon(t, sshClientOne, JOB_COMMAND_LOW, testLogger) + RestartLsfDaemon(t, sshClientOne, testLogger) // Reboot instance - RebootInstance(t, sshClientOne, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], JOB_COMMAND_MED, testLogger) + RebootInstance(t, sshClientOne, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], testLogger) // Reconnect to the management node after reboot sshClientOne, connectionErrOne = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) @@ -850,9 +775,12 @@ func ValidateClusterConfigurationWithMultipleKeys(t *testing.T, options *testhel } }() + // Run job + VerifyJobs(t, sshClientOne, jobCommandLow, testLogger) + // Get dynamic compute node IPs and handle errors computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClientOne, testLogger) - require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs") // Verify compute node configuration VerifyComputeNodeConfig(t, sshClientOne, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) @@ -865,15 +793,15 @@ func ValidateClusterConfigurationWithMultipleKeys(t *testing.T, options *testhel // Verify SSH connectivity from login node sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) - require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH") defer sshLoginNodeClient.Close() // Verify login node configuration - VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, jobCommandLow, EXPECTED_LSF_VERSION, testLogger) // Get dynamic compute node IPs again computeNodeIPList, computeIPErr = LSFGETDynamicComputeNodeIPs(t, sshClientOne, testLogger) - require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs") // Verify SSH connectivity from login node VerifySSHConnectivityToNodesFromLogin(t, sshLoginNodeClient, managementNodeIPList, computeNodeIPList, testLogger) @@ -907,7 +835,6 @@ func ValidateExistingLDAPClusterConfig(t *testing.T, ldapServerBastionIP, ldapSe // Define job commands for different priority levels jobCommandLow := GetJobCommand(expectedZone, "high") - jobCommandMed := GetJobCommand(expectedZone, "med") // Run the test consistency check output, err := options.RunTestConsistency() @@ -926,24 +853,24 @@ func ValidateExistingLDAPClusterConfig(t *testing.T, ldapServerBastionIP, ldapSe // Connect to the master node via SSH sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPs[0]) - require.NoError(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the master via SSH") defer sshClient.Close() testLogger.Info(t, "SSH connection to the master successful") t.Log("Validation in progress. Please wait...") // Verify management node configuration - VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPs, jobCommandLow, EXPECTED_LSF_VERSION, testLogger) + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPs, EXPECTED_LSF_VERSION, testLogger) // Restart LSF daemon - RestartLsfDaemon(t, sshClient, jobCommandLow, testLogger) + RestartLsfDaemon(t, sshClient, testLogger) // Reboot instance - RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPs[0], jobCommandMed, testLogger) + RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPs[0], testLogger) // Reconnect to the management node after reboot sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPs[0]) - require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH") defer sshClient.Close() // Wait for dynamic node disappearance and handle potential errors @@ -953,16 +880,19 @@ func ValidateExistingLDAPClusterConfig(t *testing.T, ldapServerBastionIP, ldapSe } }() + // Run job + VerifyJobs(t, sshClient, jobCommandLow, testLogger) + // Get dynamic compute node IPs computeNodeIPs, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) - require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs") // Verify compute node configuration VerifyComputeNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPs, testLogger) // Verify SSH connectivity from login node sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) - require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH") defer sshLoginNodeClient.Close() // Verify login node configuration @@ -976,7 +906,7 @@ func ValidateExistingLDAPClusterConfig(t *testing.T, ldapServerBastionIP, ldapSe // Connect to the LDAP server via SSH sshLdapClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, ldapServerBastionIP, LSF_LDAP_HOST_NAME, ldapServerIP) - require.NoError(t, connectionErr, "Failed to connect to the LDAP server via SSH: %v", connectionErr) + require.NoError(t, connectionErr, "Failed to connect to the LDAP server via SSH") defer sshLdapClient.Close() // Check LDAP server status @@ -992,7 +922,232 @@ func ValidateExistingLDAPClusterConfig(t *testing.T, ldapServerBastionIP, ldapSe VerifyLoginNodeLDAPConfig(t, sshLoginNodeClient, bastionIP, loginNodeIP, ldapServerIP, jobCommandLow, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) // Verify LDAP user creation and LSF actions using the new user - VerifyCreateNewLdapUserAndManagementNodeLDAPConfig(t, sshLdapClient, bastionIP, ldapServerIP, managementNodeIPs, jobCommandLow, ldapAdminPassword, expectedLdapDomain, ldapUserName, ldapUserPassword, "user2", testLogger) + VerifyCreateNewLdapUserAndManagementNodeLDAPConfig(t, sshLdapClient, bastionIP, ldapServerIP, managementNodeIPs, jobCommandLow, ldapAdminPassword, expectedLdapDomain, ldapUserName, ldapUserPassword, "tester2", testLogger) + + // Log validation end + testLogger.Info(t, t.Name()+" Validation ended") +} + +// ValidateClusterConfigWithAPPCenterOnExistingEnvironment validates the configuration of an existing cluster with App Center integration. +// It verifies management node configuration, SSH keys, failover and failback, LSF daemon restart, dynamic compute node configuration, +// login node configuration, SSH connectivity, application center configuration, noVNC configuration, PTR records, and file share encryption. +// The function connects to various nodes, performs required actions, and logs results using the provided test logger. +// Parameters include expected values, IP addresses, and configuration settings to ensure the cluster operates correctly with the specified integrations. +func ValidateClusterConfigWithAPPCenterOnExistingEnvironment( + t *testing.T, + computeSshKeysList []string, + bastionIP, loginNodeIP, expectedClusterID, expectedReservationID, expectedMasterName, expectedResourceGroup, + expectedKeyManagement, expectedZone, expectedDnsDomainName string, + managementNodeIPList []string, + expectedHyperthreadingEnabled bool, + testLogger *utils.AggregatedLogger, +) { + + expectedNumOfKeys := len(computeSshKeysList) + + // Retrieve job commands for different levels + jobCommandLow := GetJobCommand(expectedZone, "low") + jobCommandMed := GetJobCommand(expectedZone, "med") + + // Log validation start + testLogger.Info(t, t.Name()+" Validation started...") + + // Connect to the master node via SSH + sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.NoError(t, connectionErr, "Failed to connect to the master via SSH") + defer sshClient.Close() + + testLogger.Info(t, "SSH connection to the master successful") + t.Log("Validation in progress. Please wait...") + + // Verify management node configuration + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, EXPECTED_LSF_VERSION, testLogger) + + // Verify SSH key on management nodes + VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "management", managementNodeIPList, expectedNumOfKeys, testLogger) + + // Perform failover and failback + FailoverAndFailback(t, sshClient, jobCommandMed, testLogger) + + // Restart LSF daemon + RestartLsfDaemon(t, sshClient, testLogger) + + // Reboot instance + RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], testLogger) + + // Reconnect to the master node via SSH after reboot + sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH") + defer sshClient.Close() + + // Wait for dynamic node disappearance and handle potential errors + defer func() { + if err := LSFWaitForDynamicNodeDisappearance(t, sshClient, testLogger); err != nil { + t.Errorf("Error in LSFWaitForDynamicNodeDisappearance: %v", err) + } + }() + + // Run job + VerifyJobs(t, sshClient, jobCommandLow, testLogger) + + // Get dynamic compute node IPs + computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) + require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs") + + // Verify compute node configuration + VerifyComputeNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) + + // Verify SSH key for compute nodes + VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "compute", computeNodeIPList, expectedNumOfKeys, testLogger) + + // Verify SSH connectivity from login node + sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH") + defer sshLoginNodeClient.Close() + + // Verify login node configuration + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, jobCommandLow, EXPECTED_LSF_VERSION, testLogger) + + // Re-fetch dynamic compute node IPs + computeNodeIPList, computeIPErr = LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) + require.NoError(t, computeIPErr, "Error getting dynamic compute node IPs") + + // Verify SSH connectivity from login node + VerifySSHConnectivityToNodesFromLogin(t, sshLoginNodeClient, managementNodeIPList, computeNodeIPList, testLogger) + + // Verify application center configuration + VerifyAPPCenterConfig(t, sshClient, testLogger) + + // Verify noVNC configuration + VerifyNoVNCConfig(t, sshClient, testLogger) + + // Verify PTR records for management and login nodes + VerifyPTRRecordsForManagementAndLoginNodes(t, sshClient, LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList, loginNodeIP, expectedDnsDomainName, testLogger) + + // Verify file share encryption + VerifyFileShareEncryption(t, os.Getenv("TF_VAR_ibmcloud_api_key"), utils.GetRegion(expectedZone), expectedResourceGroup, expectedMasterName, expectedKeyManagement, testLogger) + + // Log validation end + testLogger.Info(t, t.Name()+" Validation ended") +} + +// ValidateClusterConfigWithAPPCenterAndLDAPOnExistingEnvironment validates the configuration of an existing cluster with App Center and LDAP integration. +// It verifies management node configuration, SSH keys, failover and failback, LSF daemon restart, dynamic compute node configuration, login node configuration, +// SSH connectivity, application center configuration, noVNC configuration, PTR records, file share encryption, and LDAP server configuration and status. +// The function connects to various nodes, performs required actions, and logs results using the provided test logger. +// Parameters include expected values, IP addresses, credentials for validation, and configuration settings. +// This ensures the cluster operates correctly with the specified configurations and integrations, including LDAP. +func ValidateClusterConfigWithAPPCenterAndLDAPOnExistingEnvironment( + t *testing.T, + computeSshKeysList []string, + bastionIP, loginNodeIP, expectedClusterID, expectedReservationID, expectedMasterName, expectedResourceGroup, + expectedKeyManagement, expectedZone, expectedDnsDomainName string, + managementNodeIPList []string, + expectedHyperthreadingEnabled bool, + ldapServerIP, expectedLdapDomain, ldapAdminPassword, ldapUserName, ldapUserPassword string, + testLogger *utils.AggregatedLogger, +) { + + expectedNumOfKeys := len(computeSshKeysList) + + // Retrieve job commands for different levels + jobCommandLow := GetJobCommand(expectedZone, "low") + jobCommandMed := GetJobCommand(expectedZone, "med") + + // Log validation start + testLogger.Info(t, t.Name()+" Validation started...") + + // Connect to the master node via SSH + sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.NoError(t, connectionErr, "Failed to connect to the master via SSH") + defer sshClient.Close() + + testLogger.Info(t, "SSH connection to the master successful") + t.Log("Validation in progress. Please wait...") + + // Verify management node configuration + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, EXPECTED_LSF_VERSION, testLogger) + + // Verify SSH key on management nodes + VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "management", managementNodeIPList, expectedNumOfKeys, testLogger) + + // Perform failover and failback + FailoverAndFailback(t, sshClient, jobCommandMed, testLogger) + + // Restart LSF daemon + RestartLsfDaemon(t, sshClient, testLogger) + + // Reboot instance + RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], testLogger) + + // Reconnect to the master node via SSH after reboot + sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.NoError(t, connectionErr, "Failed to reconnect to the master via SSH") + defer sshClient.Close() + + // Wait for dynamic node disappearance and handle potential errors + defer func() { + if err := LSFWaitForDynamicNodeDisappearance(t, sshClient, testLogger); err != nil { + t.Errorf("Error in LSFWaitForDynamicNodeDisappearance: %v", err) + } + }() + + // Run job + VerifyJobs(t, sshClient, jobCommandLow, testLogger) + + // Get dynamic compute node IPs + computeNodeIPList, err := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) + require.NoError(t, err, "Error getting dynamic compute node IPs") + + // Verify compute node configuration + VerifyComputeNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) + + // Verify SSH key for compute nodes + VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "compute", computeNodeIPList, expectedNumOfKeys, testLogger) + + // Verify SSH connectivity from login node + sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH") + defer sshLoginNodeClient.Close() + + // Verify login node configuration + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, jobCommandLow, EXPECTED_LSF_VERSION, testLogger) + + // Re-fetch dynamic compute node IPs + computeNodeIPList, connectionErr = LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) + require.NoError(t, connectionErr, "Error getting dynamic compute node IPs") + + // Verify SSH connectivity from login node + VerifySSHConnectivityToNodesFromLogin(t, sshLoginNodeClient, managementNodeIPList, computeNodeIPList, testLogger) + + // Verify application center configuration + VerifyAPPCenterConfig(t, sshClient, testLogger) + + // Verify noVNC configuration + VerifyNoVNCConfig(t, sshClient, testLogger) + + // Verify PTR records for management and login nodes + VerifyPTRRecordsForManagementAndLoginNodes(t, sshClient, LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList, loginNodeIP, expectedDnsDomainName, testLogger) + + // Verify file share encryption + VerifyFileShareEncryption(t, os.Getenv("TF_VAR_ibmcloud_api_key"), utils.GetRegion(expectedZone), expectedResourceGroup, expectedMasterName, expectedKeyManagement, testLogger) + + // Connect to the LDAP server via SSH and handle connection errors + sshLdapClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_LDAP_HOST_NAME, ldapServerIP) + require.NoError(t, connectionErr, "Failed to connect to the LDAP server via SSH") + defer sshLdapClient.Close() + + // Check LDAP server status + CheckLDAPServerStatus(t, sshLdapClient, ldapAdminPassword, expectedLdapDomain, ldapUserName, testLogger) + + // Verify management node LDAP config + VerifyManagementNodeLDAPConfig(t, sshClient, bastionIP, ldapServerIP, managementNodeIPList, jobCommandLow, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) + + // Verify compute node LDAP config + VerifyComputeNodeLDAPConfig(t, bastionIP, ldapServerIP, computeNodeIPList, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) + + // Verify login node LDAP config + VerifyLoginNodeLDAPConfig(t, sshLoginNodeClient, bastionIP, loginNodeIP, ldapServerIP, jobCommandLow, expectedLdapDomain, ldapUserName, ldapUserPassword, testLogger) // Log validation end testLogger.Info(t, t.Name()+" Validation ended") diff --git a/tests/lsf/lsf_constants.go b/tests/lsf/constants.go similarity index 73% rename from tests/lsf/lsf_constants.go rename to tests/lsf/constants.go index 2cfd4059..746fa84c 100644 --- a/tests/lsf/lsf_constants.go +++ b/tests/lsf/constants.go @@ -11,13 +11,13 @@ const ( LSF_CUSTOM_RESOURCE_GROUP_VALUE_AS_NULL = "null" LOGIN_NODE_EXECUTION_PATH = "source /opt/ibm/lsf/conf/profile.lsf;" COMPUTE_NODE_EXECUTION_PATH = "source /opt/ibm/lsf_worker/conf/profile.lsf;" - JOB_COMMAND_LOW_MEM = `bsub -J myjob[1-2] -R "select[family=mx2] rusage[mem=10G]" sleep 90` - JOB_COMMAND_MED_MEM = `bsub -J myjob[1-2] -R "select[family=mx2] rusage[mem=30G]" sleep 90` - JOB_COMMAND_HIGH_MEM = `bsub -J myjob[1-2] -R "select[family=mx2] rusage[mem=90G]" sleep 90` - JOB_COMMAND_LOW_MEM_SOUTH = `bsub -J myjob[1-2] -R "select[family=mx3d] rusage[mem=10G]" sleep 90` - JOB_COMMAND_MED_MEM_SOUTH = `bsub -J myjob[1-2] -R "select[family=mx3d] rusage[mem=30G]" sleep 90` - JOB_COMMAND_HIGH_MEM_SOUTH = `bsub -J myjob[1-2] -R "select[family=mx3d] rusage[mem=90G]" sleep 90` - JOB_COMMAND_LOW_MEM_WITH_MORE_SLEEP = `bsub -J myjob[1-2] -R "select[family=mx2] rusage[mem=30G]" sleep 90` + JOB_COMMAND_LOW_MEM = `bsub -J myjob[1-1] -R "select[family=mx2] rusage[mem=10G]" sleep 90` + JOB_COMMAND_MED_MEM = `bsub -J myjob[1-1] -R "select[family=mx2] rusage[mem=30G]" sleep 90` + JOB_COMMAND_HIGH_MEM = `bsub -J myjob[1-1] -R "select[family=mx2] rusage[mem=90G]" sleep 90` + JOB_COMMAND_LOW_MEM_SOUTH = `bsub -J myjob[1-1] -R "select[family=mx3d] rusage[mem=10G]" sleep 90` + JOB_COMMAND_MED_MEM_SOUTH = `bsub -J myjob[1-1] -R "select[family=mx3d] rusage[mem=30G]" sleep 90` + JOB_COMMAND_HIGH_MEM_SOUTH = `bsub -J myjob[1-1] -R "select[family=mx3d] rusage[mem=90G]" sleep 90` + JOB_COMMAND_LOW_MEM_WITH_MORE_SLEEP = `bsub -J myjob[1-1] -R "select[family=mx2] rusage[mem=30G]" sleep 90` ) var ( diff --git a/tests/other_test.go b/tests/other_test.go index 5a0b2ad8..8ad815c0 100644 --- a/tests/other_test.go +++ b/tests/other_test.go @@ -12,8 +12,8 @@ import ( "github.com/gruntwork-io/terratest/modules/terraform" "github.com/stretchr/testify/assert" - utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/common_utils" lsf "github.com/terraform-ibm-modules/terraform-ibm-hpc/lsf" + utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/utilities" ) // Constants for better organization @@ -761,37 +761,6 @@ func TestRunCIDRsAsNonDefault(t *testing.T) { lsf.ValidateBasicClusterConfiguration(t, options, testLogger) } -// TestRunExistingPACEnvironment tests the validation of an existing PAC environment configuration. -func TestRunExistingPACEnvironment(t *testing.T) { - // Parallelize the test to run concurrently with others - t.Parallel() - - // Setup the test suite environment - setupTestSuite(t) - - // Log the initiation of cluster creation process - testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) - - // Retrieve the environment variable for the JSON file path - val, ok := os.LookupEnv("EXISTING_ENV_JSON_FILE_PATH") - if !ok { - t.Fatal("Environment variable 'EXISTING_ENV_JSON_FILE_PATH' is not set") - } - - // Check if the JSON file exists - if _, err := os.Stat(val); os.IsNotExist(err) { - t.Fatalf("JSON file '%s' does not exist", val) - } - - // Parse the JSON configuration file - config, err := utils.ParseConfig(val) - require.NoError(t, err, "Error parsing JSON configuration: %v", err) - - // Validate the cluster configuration - lsf.ValidateClusterConfigurationWithAPPCenterForExistingEnv(t, 1, config.BastionIP, config.LoginNodeIP, config.ClusterID, config.ReservationID, config.ClusterPrefixName, config.ResourceGroup, - config.KeyManagement, config.Zones, config.DnsDomainName, config.ManagementNodeIPList, config.HyperthreadingEnabled, testLogger) -} - // TestRunInvalidReservationIDAndContractID tests invalid cluster_id and reservation_id values func TestRunInvalidReservationIDAndContractID(t *testing.T) { t.Parallel() @@ -1583,6 +1552,7 @@ func TestRunExistingLDAP(t *testing.T) { require.NoError(t, err, "Error setting up test options for the first cluster: %v", err) // Set Terraform variables for the first cluster + options1.TerraformVars["management_node_count"] = 1 options1.TerraformVars["enable_ldap"] = strings.ToLower(envVars.EnableLdap) options1.TerraformVars["ldap_basedns"] = envVars.LdapBaseDns options1.TerraformVars["ldap_admin_password"] = envVars.LdapAdminPassword // pragma: allowlist secret @@ -1642,3 +1612,72 @@ func TestRunExistingLDAP(t *testing.T) { // Validate LDAP configuration for the second cluster lsf.ValidateExistingLDAPClusterConfig(t, ldapServerBastionIP, ldapIP, envVars.LdapBaseDns, envVars.LdapAdminPassword, envVars.LdapUserName, envVars.LdapUserPassword, options2, testLogger) } + +// TestRunExistingPACEnvironment test the validation of an existing PAC environment configuration. +func TestRunExistingPACEnvironment(t *testing.T) { + // Parallelize the test to run concurrently with others + t.Parallel() + + // Setup the test suite environment + setupTestSuite(t) + + // Log the initiation of cluster creation process + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // Retrieve the environment variable for the JSON file path + val, ok := os.LookupEnv("EXISTING_ENV_JSON_FILE_PATH") + if !ok { + t.Fatal("Environment variable 'EXISTING_ENV_JSON_FILE_PATH' is not set") + } + + // Check if the JSON file exists + if _, err := os.Stat(val); os.IsNotExist(err) { + t.Fatalf("JSON file '%s' does not exist", val) + } + + // Parse the JSON configuration file + config, err := utils.ParseConfig(val) + require.NoError(t, err, "Error parsing JSON configuration: %v", err) + + // Validate the cluster configuration + lsf.ValidateClusterConfigWithAPPCenterOnExistingEnvironment( + t, config.ComputeSshKeysList, config.BastionIP, config.LoginNodeIP, config.ClusterID, config.ReservationID, + config.ClusterPrefixName, config.ResourceGroup, config.KeyManagement, + config.Zones, config.DnsDomainName, config.ManagementNodeIPList, + config.IsHyperthreadingEnabled, testLogger) +} + +// TestRunExistingPACAndLDAPEnvironment test the validation of an existing PAC and LDAP environment configuration. +func TestRunExistingPACAndLDAPEnvironment(t *testing.T) { + // Parallelize the test to run concurrently with others + t.Parallel() + + // Setup the test suite environment + setupTestSuite(t) + + // Log the initiation of cluster creation process + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // Retrieve the environment variable for the JSON file path + val, ok := os.LookupEnv("EXISTING_ENV_JSON_FILE_PATH") + if !ok { + t.Fatal("Environment variable 'EXISTING_ENV_JSON_FILE_PATH' is not set") + } + + // Check if the JSON file exists + if _, err := os.Stat(val); os.IsNotExist(err) { + t.Fatalf("JSON file '%s' does not exist", val) + } + + // Parse the JSON configuration file + config, err := utils.ParseConfig(val) + require.NoError(t, err, "Error parsing JSON configuration: %v", err) + + // Validate the cluster configuration + lsf.ValidateClusterConfigWithAPPCenterAndLDAPOnExistingEnvironment( + t, config.ComputeSshKeysList, config.BastionIP, config.LoginNodeIP, config.ClusterID, config.ReservationID, + config.ClusterPrefixName, config.ResourceGroup, config.KeyManagement, config.Zones, config.DnsDomainName, + config.ManagementNodeIPList, config.IsHyperthreadingEnabled, config.LdapServerIP, config.LdapDomain, + config.LdapAdminPassword, config.LdapUserName, config.LdapUserPassword, testLogger) + +} diff --git a/tests/pr_test.go b/tests/pr_test.go index d0a942af..91f0839c 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testhelper" - utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/common_utils" + utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/utilities" ) // Constants for better organization @@ -248,7 +248,7 @@ func setupOptions(t *testing.T, hpcClusterPrefix, terraformDir, resourceGroup st func TestMain(m *testing.M) { - absPath, err := filepath.Abs("test_config.yml") + absPath, err := filepath.Abs("config.yml") if err != nil { log.Fatalf("error getting absolute path: %v", err) } @@ -263,30 +263,40 @@ func TestMain(m *testing.M) { } -// TestRunDefault create basic cluster of an HPC cluster. +// TestRunDefault creates a basic HPC cluster and verifies its setup. func TestRunDefault(t *testing.T) { - - // Parallelize the test + // Run tests in parallel t.Parallel() - // Setup test suite + // Initialize test suite setupTestSuite(t) - testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + // Log initiation of cluster creation + testLogger.Info(t, "Initiating cluster creation for "+t.Name()) - // HPC cluster prefix + // Generate a unique prefix for the HPC cluster hpcClusterPrefix := utils.GenerateRandomString() - // Retrieve cluster information from environment variables + // Retrieve environment variables for the test envVars := GetEnvVars() - // Create test options + // Prepare test options with necessary parameters options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) - require.NoError(t, err, "Error setting up test options: %v", err) + if err != nil { + testLogger.FAIL(t, fmt.Sprintf("Failed to set up test options: %v", err)) + require.NoError(t, err, "Failed to set up test options: %v", err) + } - // Run the test and handle errors + // Run consistency test and handle potential errors output, err := options.RunTestConsistency() - require.NoError(t, err, "Error running consistency test: %v", err) + if err != nil { + testLogger.FAIL(t, fmt.Sprintf("Error running consistency test: %v", err)) + require.NoError(t, err, "Error running consistency test: %v", err) + } + + // Ensure that output is not nil require.NotNil(t, output, "Expected non-nil output, but got nil") + // Log success if no errors occurred + testLogger.PASS(t, "Test passed successfully") } diff --git a/tests/common_utils/deploy_utils.go b/tests/utilities/deployment.go similarity index 98% rename from tests/common_utils/deploy_utils.go rename to tests/utilities/deployment.go index 17ef8ae3..a237faa9 100644 --- a/tests/common_utils/deploy_utils.go +++ b/tests/utilities/deployment.go @@ -96,7 +96,7 @@ func GetConfigFromYAML(filePath string) (*Config, error) { permanentResources["reservation_id_secret_id"].(string), ) if err != nil { - fmt.Printf("error retrieving reservation id from secrets: %v", err) // pragma: allowlist secret + fmt.Printf("Retrieving reservation id from secrets: %v", err) // pragma: allowlist secret } else if reservationIDEastPtr != nil { reservationIDEast = *reservationIDEastPtr } diff --git a/tests/utilities/fileops.go b/tests/utilities/fileops.go new file mode 100644 index 00000000..b1a535b9 --- /dev/null +++ b/tests/utilities/fileops.go @@ -0,0 +1,199 @@ +package tests + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/crypto/ssh" +) + +// ToCreateFile creates a file on the remote server using SSH. +// It takes an SSH client, the path where the file should be created, and the file name. +// Returns a boolean indicating success or failure and an error if any. +func ToCreateFile(t *testing.T, sClient *ssh.Client, filePath, fileName string, logger *AggregatedLogger) (bool, error) { + // Check if the specified path exists on the remote server + isPathExist, err := IsPathExist(t, sClient, filePath, logger) + if err != nil { + // Error occurred while checking path existence + return false, err + } + + if isPathExist { + // Path exists, create the file using SSH command + command := "cd " + filePath + " && touch " + fileName + _, createFileErr := RunCommandInSSHSession(sClient, command) + + if createFileErr == nil { + logger.Info(t, "File created successfully: "+fileName) + // File created successfully + return true, nil + } else { + // Error occurred while creating the file + return false, fmt.Errorf(" %s file not created", fileName) + } + } + + // Path does not exist + return false, fmt.Errorf("directory not exist : %s", filePath) +} + +// IsFileExist checks if a file exists on the remote server using SSH. +// It takes an SSH client, the path where the file should exist, and the file name. +// Returns a boolean indicating whether the file exists and an error if any. +func IsFileExist(t *testing.T, sClient *ssh.Client, filePath, fileName string, logger *AggregatedLogger) (bool, error) { + // Check if the specified path exists on the remote server + isPathExist, err := IsPathExist(t, sClient, filePath, logger) + if err != nil { + // Error occurred while checking path existence + return false, err + } + + if isPathExist { + // Path exists, check if the file exists using an SSH command + command := "[ -f " + filePath + fileName + " ] && echo 'File exist' || echo 'File not exist'" + isFileExist, err := RunCommandInSSHSession(sClient, command) + + if err != nil { + // Error occurred while running the command + return false, err + } + + if strings.Contains(isFileExist, "File exist") { + logger.Info(t, fmt.Sprintf("File exist : %s", fileName)) + // File exists + return true, nil + } else { + logger.Info(t, fmt.Sprintf("File not exist : %s", fileName)) + // File does not exist + return false, nil + } + } + + // Path does not exist + return false, fmt.Errorf("directory not exist : %s", filePath) +} + +// IsPathExist checks if a directory exists on the remote server using SSH. +// It takes an SSH client and the path to check for existence. +// Returns a boolean indicating whether the directory exists and an error if any. +func IsPathExist(t *testing.T, sClient *ssh.Client, filePath string, logger *AggregatedLogger) (bool, error) { + // Run an SSH command to test if the directory exists + command := "test -d " + filePath + " && echo 'Directory Exist' || echo 'Directory Not Exist'" + result, err := RunCommandInSSHSession(sClient, command) + + if err != nil { + // Error occurred while running the command + return false, err + } + + if strings.Contains(result, "Directory Exist") { + logger.Info(t, fmt.Sprintf("Directory exist : %s", filePath)) + // Directory exists + return true, nil + } else { + logger.Info(t, fmt.Sprintf("Directory not exist : %s", filePath)) + // Directory does not exist + return false, nil + } +} + +// GetDirList retrieves the list of files in a directory on a remote server via SSH. +// It takes an SSH client and the path of the directory. +// Returns a string containing the list of files and an error if any. +func GetDirList(t *testing.T, sClient *ssh.Client, filePath string, logger *AggregatedLogger) ([]string, error) { + command := "cd " + filePath + " && ls" + output, err := RunCommandInSSHSession(sClient, command) + if err == nil { + listDir := strings.Split(strings.TrimSpace(output), "\n") + logger.Info(t, fmt.Sprintf("Directory list : %q ", listDir)) + return listDir, nil + } + return nil, err +} + +// GetDirectoryFileList retrieves the list of files in a directory on a remote server via SSH. +// It takes an SSH client and the path of the directory. +// Returns a slice of strings representing the file names and an error if any. +func GetDirectoryFileList(t *testing.T, sClient *ssh.Client, directoryPath string, logger *AggregatedLogger) ([]string, error) { + command := "cd " + directoryPath + " && ls" + output, err := RunCommandInSSHSession(sClient, command) + if err == nil { + fileList := strings.Split(strings.TrimSpace(output), "\n") + logger.Info(t, fmt.Sprintf("Directory file list : %q", fileList)) + return fileList, nil + } + + return nil, err +} + +// ToDeleteFile deletes a file on the remote server using SSH. +// It takes an SSH client, the path of the file's directory, and the file name. +// Returns a boolean indicating whether the file was deleted successfully and an error if any. +func ToDeleteFile(t *testing.T, sClient *ssh.Client, filePath, fileName string, logger *AggregatedLogger) (bool, error) { + isPathExist, err := IsPathExist(t, sClient, filePath, logger) + if isPathExist { + command := "cd " + filePath + " && rm -rf " + fileName + _, deleFileErr := RunCommandInSSHSession(sClient, command) + if deleFileErr == nil { + logger.Info(t, fmt.Sprintf("File deleted successfully: %s", fileName)) + return true, nil + } + return false, fmt.Errorf("files not deleted: %s", fileName) + } + return isPathExist, err +} + +// ToCreateFileWithContent creates a file on the remote server using SSH. +// It takes an SSH client, the path where the file should be created, the file name, content to write to the file, +// a log file for logging, and returns a boolean indicating success or failure and an error if any. +func ToCreateFileWithContent(t *testing.T, sClient *ssh.Client, filePath, fileName, content string, logger *AggregatedLogger) (bool, error) { + // Check if the specified path exists on the remote server + isPathExist, err := IsPathExist(t, sClient, filePath, logger) + if err != nil { + // Error occurred while checking path existence + return false, fmt.Errorf("error checking path existence: %w", err) + } + + if isPathExist { + // Path exists, create the file using SSH command + command := "cd " + filePath + " && echo '" + content + "' > " + fileName + _, createFileErr := RunCommandInSSHSession(sClient, command) + + if createFileErr == nil { + // File created successfully + logger.Info(t, "File created successfully: "+fileName) + return true, nil + } + + // Error occurred while creating the file + return false, fmt.Errorf("error creating file %s: %w", fileName, createFileErr) + } + + // Path does not exist + return false, fmt.Errorf("directory not exist : %s", filePath) +} + +// ReadRemoteFileContents reads the content of a file on the remote server via SSH. +// It checks if the specified file path exists, creates an SSH command to concatenate the file contents, +// and returns the content as a string upon success. In case of errors, an empty string and an error are returned. +func ReadRemoteFileContents(t *testing.T, sClient *ssh.Client, filePath, fileName string, logger *AggregatedLogger) (string, error) { + isPathExist, err := IsPathExist(t, sClient, filePath, logger) + if err != nil { + return "", fmt.Errorf("error checking path existence: %w", err) + } + + if isPathExist { + command := "cd " + filePath + " && cat " + fileName + actualText, outErr := RunCommandInSSHSession(sClient, command) + + if outErr == nil { + logger.Info(t, "content: "+actualText) + return actualText, nil + } + + return "", fmt.Errorf("error reading file %s: %w", fileName, outErr) + } + + return "", fmt.Errorf("directory does not exist: %s", filePath) +} diff --git a/tests/utilities/helpers.go b/tests/utilities/helpers.go new file mode 100644 index 00000000..f5729545 --- /dev/null +++ b/tests/utilities/helpers.go @@ -0,0 +1,425 @@ +package tests + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "math/rand" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/IBM/go-sdk-core/v5/core" + "github.com/IBM/secrets-manager-go-sdk/v2/secretsmanagerv2" + "github.com/stretchr/testify/assert" + "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testhelper" +) + +const ( + // TimeLayout is the layout string for date and time format (DDMonHHMMSS). + TimeLayout = "Jan02" +) + +// VerifyDataContains is a generic function that checks if a value is present in data (string or string array) +// VerifyDataContains performs a verification operation on the provided data +// to determine if it contains the specified value. It supports string and +// string array types, logging results with the provided AggregatedLogger. +// Returns true if the value is found, false otherwise. +func VerifyDataContains(t *testing.T, data interface{}, val interface{}, logger *AggregatedLogger) bool { + //The data.(type) syntax is used to check the actual type of the data variable. + switch d := data.(type) { + case string: + //check if the val variable is of type string. + substr, ok := val.(string) + if !ok { + logger.Info(t, "Invalid type for val parameter") + return false + } + if substr != "" && strings.Contains(d, substr) { + logger.Info(t, fmt.Sprintf("The string '%s' contains the substring '%s'\n", d, substr)) + return true + } + logger.Info(t, fmt.Sprintf("The string '%s' does not contain the substring '%s'\n", d, substr)) + return false + + case []string: + switch v := val.(type) { + case string: + for _, arrVal := range d { + if arrVal == v { + logger.Info(t, fmt.Sprintf("The array '%q' contains the value: %s\n", d, v)) + return true + } + } + logger.Info(t, fmt.Sprintf("The array '%q' does not contain the value: %s\n", d, v)) + return false + + case []string: + if reflect.DeepEqual(d, v) { + logger.Info(t, fmt.Sprintf("The array '%q' contains the subarray '%q'\n", d, v)) + return true + } + logger.Info(t, fmt.Sprintf("The array '%q' does not contain the subarray '%q'\n", d, v)) + return false + + default: + logger.Info(t, "Invalid type for val parameter") + return false + } + + default: + logger.Info(t, "Unsupported type for data parameter") + return false + } +} + +func CountStringOccurrences(str string, substr string) int { + return strings.Count(str, substr) +} + +func SplitString(strValue string, splitCharacter string, indexValue int) string { + split := strings.Split(strValue, splitCharacter) + return split[indexValue] +} + +// StringToInt converts a string to an integer. +// Returns the converted integer and an error if the conversion fails. +func StringToInt(str string) (int, error) { + num, err := strconv.Atoi(str) + if err != nil { + return 0, err + } + return num, nil +} + +// RemoveNilValues removes nil value keys from the given map. +func RemoveNilValues(data map[string]interface{}) map[string]interface{} { + for key, value := range data { + if value == nil { + delete(data, key) + } + } + return data +} + +// LogVerificationResult logs the result of a verification check. +func LogVerificationResult(t *testing.T, err error, checkName string, logger *AggregatedLogger) { + if err == nil { + logger.Info(t, fmt.Sprintf("%s verification successful", checkName)) + } else { + assert.Nil(t, err, fmt.Sprintf("%s verification failed", checkName)) + logger.Error(t, fmt.Sprintf("%s verification failed: %s", checkName, err.Error())) + } +} + +// ParsePropertyValue parses the content of a string, searching for a property with the specified key. +// It returns the value of the property if found, or an empty string and an error if the property is not found. +func ParsePropertyValue(content, propertyKey string) (string, error) { + lines := strings.Split(content, "\n") + for _, line := range lines { + if strings.HasPrefix(line, propertyKey+"=") { + // Extract the value, removing quotes if present + value := strings.Trim(line[len(propertyKey)+1:], `"`) + return value, nil + } + } + + // Property not found + return "", fmt.Errorf("property '%s' not found in content:\n%s", propertyKey, content) +} + +// ParsePropertyValue parses the content of a string, searching for a property with the specified key. +// It returns the value of the property if found, or an empty string and an error if the property is not found. +func FindImageNamesByCriteria(name string) (string, error) { + // Get the absolute path of the image map file + absPath, err := filepath.Abs("modules/landing_zone_vsi/image_map.tf") + if err != nil { + return "", err + } + + absPath = strings.ReplaceAll(absPath, "tests", "") + readFile, err := os.Open(absPath) + if err != nil { + return "", err + } + defer readFile.Close() + + // Create a scanner to read lines from the file + fileScanner := bufio.NewScanner(readFile) + + var imageName string + + for fileScanner.Scan() { + line := fileScanner.Text() + if strings.Contains(strings.ToLower(line), strings.ToLower(name)) && strings.Contains(line, "compute") { + pattern := "[^a-zA-Z0-9\\s-]" + + // Compile the regular expression. + regex, err := regexp.Compile(pattern) + if err != nil { + return "", errors.New("error on image compiling regex") + } + // Use the regex to replace all matches with an empty string. + imageName = strings.TrimSpace(regex.ReplaceAllString(line, "")) + } + } + + // Check if any image names were found + if len(imageName) == 0 { + return imageName, errors.New("no image found with the specified criteria") + } + + return imageName, nil +} + +//Create IBMCloud Resources + +// LoginIntoIBMCloudUsingCLI logs into IBM Cloud using CLI. +func LoginIntoIBMCloudUsingCLI(t *testing.T, apiKey, region, resourceGroup string) error { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + // Configure IBM Cloud CLI + configCmd := exec.CommandContext(ctx, "ibmcloud", "config", "--check-version=false") + configOutput, err := configCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to configure IBM Cloud CLI: %w. Output: %s", err, string(configOutput)) + } + + // Login to IBM Cloud and set the target resource group + loginCmd := exec.CommandContext(ctx, "ibmcloud", "login", "--apikey", apiKey, "-r", region, "-g", resourceGroup) + loginOutput, err := loginCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to login to IBM Cloud: %w. Output: %s", err, string(loginOutput)) + } + + return nil +} + +// GenerateTimestampedClusterPrefix generates a cluster prefix by appending a timestamp to the given prefix. +func GenerateTimestampedClusterPrefix(prefix string) string { + //Place current time in the string. + t := time.Now() + return strings.ToLower("cicd" + "-" + t.Format(TimeLayout) + "-" + prefix) + +} + +// GetPublicIP returns the public IP address using ifconfig.io API +func GetPublicIP() (string, error) { + cmd := exec.Command("bash", "-c", "(curl -s ifconfig.io)") + output, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + return strings.TrimSpace(string(output)), nil +} + +// GetOrDefault returns the environment variable value if it's not empty, otherwise returns the default value. +func GetOrDefault(envVar, defaultValue string) string { + if envVar != "" { + return envVar + } + return defaultValue +} + +// GenerateRandomString generates a random string of length 4 using lowercase characters +func GenerateRandomString() string { + // Define the character set containing lowercase letters + charset := "abcdefghijklmnopqrstuvwxyz" + + b := make([]byte, 4) + + // Loop through each index of the byte slice + for i := range b { + // Generate a random index within the length of the character set + randomIndex := rand.Intn(len(charset)) + + b[i] = charset[randomIndex] + } + + // Convert the byte slice to a string and return it + return string(b) +} + +// GetSecretsManagerKey retrieves a secret from IBM Secrets Manager. +func GetSecretsManagerKey(smID, smRegion, smKeyID string) (*string, error) { + secretsManagerService, err := secretsmanagerv2.NewSecretsManagerV2(&secretsmanagerv2.SecretsManagerV2Options{ + URL: fmt.Sprintf("https://%s.%s.secrets-manager.appdomain.cloud", smID, smRegion), + Authenticator: &core.IamAuthenticator{ + ApiKey: os.Getenv("TF_VAR_ibmcloud_api_key"), + }, + }) + if err != nil { + return nil, err + } + + getSecretOptions := secretsManagerService.NewGetSecretOptions(smKeyID) + + secret, _, err := secretsManagerService.GetSecret(getSecretOptions) + if err != nil { + return nil, err + } + + secretPayload, ok := secret.(*secretsmanagerv2.ArbitrarySecret) + if !ok { + return nil, fmt.Errorf("unexpected secret type: %T", secret) + } + + return secretPayload.Payload, nil +} + +// GetValueForKey retrieves the value associated with the specified key from the given map. +func GetValueForKey(inputMap map[string]string, key string) string { + return inputMap[key] +} + +// Configuration struct matches the structure of your JSON data +type Configuration struct { + ClusterID string `json:"clusterID"` + ReservationID string `json:"reservationID"` + ClusterPrefixName string `json:"clusterPrefixName"` + ResourceGroup string `json:"resourceGroup"` + KeyManagement string `json:"keyManagement"` + DnsDomainName string `json:"dnsDomainName"` + Zones string `json:"zones"` + IsHyperthreadingEnabled bool `json:"isHyperthreadingEnabled"` + BastionIP string `json:"bastionIP"` + ManagementNodeIPList []string `json:"managementNodeIPList"` + LoginNodeIP string `json:"loginNodeIP"` + LdapServerIP string `json:"ldapServerIP"` + LdapDomain string `json:"ldapDomain"` + LdapAdminPassword string `json:"ldapAdminPassword"` + LdapUserName string `json:"ldapUserName"` + LdapUserPassword string `json:"ldapUserPassword"` + AppCenterEnabled string `json:"appCenterEnabled"` + SshKeyPath string `json:"sshKeyPath"` + ComputeSshKeysList []string `json:"computeSshKeysList"` +} + +// ParseConfig reads a JSON file from the given file path and parses it into a Configuration struct +func ParseConfig(filePath string) (*Configuration, error) { + // Read the entire content of the file + byteValue, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("error reading file %s: %w", filePath, err) + } + + // Unmarshal the JSON data into the Configuration struct + var config Configuration + err = json.Unmarshal(byteValue, &config) + if err != nil { + return nil, fmt.Errorf("error parsing JSON from file %s: %w", filePath, err) + } + + // Return the configuration struct and nil error on success + return &config, nil +} + +// GetLdapIP retrieves the IP addresses of various servers, including the LDAP server, +// from the specified file path in the provided test options, using the provided logger for logging. +// It returns the LDAP server IP and any error encountered. +func GetLdapIP(t *testing.T, options *testhelper.TestOptions, logger *AggregatedLogger) (ldapIP string, err error) { + // Retrieve the Terraform directory from the options. + filePath := options.TerraformOptions.TerraformDir + + // Get the LDAP server IP and handle errors. + ldapIP, err = GetLdapServerIP(t, filePath, logger) + if err != nil { + return "", fmt.Errorf("error getting LDAP server IP: %w", err) + } + + // Return the retrieved IP address and any error. + return ldapIP, nil +} + +// GetBastionIP retrieves the bastion server IP address based on the provided test options. +// It returns the bastion IP address and an error if any step fails. +func GetBastionIP(t *testing.T, options *testhelper.TestOptions, logger *AggregatedLogger) (bastionIP string, err error) { + // Retrieve the Terraform directory from the options. + filePath := options.TerraformOptions.TerraformDir + + // Get the bastion server IP and handle errors. + bastionIP, err = GetBastionServerIP(t, filePath, logger) + if err != nil { + return "", fmt.Errorf("error getting bastion server IP: %w", err) + } + + // Return the bastion IP address. + return bastionIP, nil +} + +// GetValueFromIniFile retrieves a value from an INI file based on the provided section and key. +// It reads the specified INI file, extracts the specified section, and returns the value associated with the key. +func GetValueFromIniFile(filePath, sectionName string) ([]string, error) { + // Read the content of the file + absolutePath, err := filepath.Abs(filePath) + if err != nil { + return nil, err + } + data, err := os.ReadFile(absolutePath) + if err != nil { + return nil, err + } + + // Convert the byte slice to a string + content := string(data) + + // Split the input into sections based on empty lines + sections := strings.Split(content, "\n\n") + + // Loop through sections and find the one with the specified sectionName + for _, section := range sections { + + if strings.Contains(section, "["+sectionName+"]") { + // Split the section into lines + lines := strings.Split(section, "\n") + + // Extract values + var sectionValues []string + for i := 1; i < len(lines); i++ { + // Skip the first line, as it contains the section name + sectionValues = append(sectionValues, strings.TrimSpace(lines[i])) + } + + return sectionValues, nil + } + } + + return nil, fmt.Errorf("section [%s] not found in file %s", sectionName, filePath) +} + +// GetRegion returns the region from a given zone. +func GetRegion(zone string) string { + // Extract region from zone + region := zone[:len(zone)-2] + return region +} + +// SplitAndTrim splits the input string into a list of trimmed strings based on the comma separator. +func SplitAndTrim(str, separator string) []string { + words := strings.Split(str, separator) + var trimmedWords []string + + for _, word := range words { + trimmedWord := strings.TrimSpace(word) + if trimmedWord != "" { + trimmedWords = append(trimmedWords, trimmedWord) + } + } + return trimmedWords +} + +// RemoveKeys removes the key-value pairs with the specified keys from the given map. +func RemoveKeys(m map[string]interface{}, keysToRemove []string) { + for _, key := range keysToRemove { + delete(m, key) + } +} diff --git a/tests/common_utils/log_utils.go b/tests/utilities/logging.go similarity index 97% rename from tests/common_utils/log_utils.go rename to tests/utilities/logging.go index d90c6424..13460355 100644 --- a/tests/common_utils/log_utils.go +++ b/tests/utilities/logging.go @@ -22,7 +22,7 @@ type AggregatedLogger struct { // NewAggregatedLogger creates a new instance of AggregatedLogger. func NewAggregatedLogger(logFileName string) (*AggregatedLogger, error) { - absPath, err := filepath.Abs("test_output") + absPath, err := filepath.Abs("logs") if err != nil { return nil, err } diff --git a/tests/utilities/resources.go b/tests/utilities/resources.go new file mode 100644 index 00000000..ba716319 --- /dev/null +++ b/tests/utilities/resources.go @@ -0,0 +1,355 @@ +package tests + +import ( + "bytes" + "fmt" + "os/exec" + "strings" + "testing" + + "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testhelper" +) + +// Prerequisite: Ensure that you have logged into IBM Cloud using the LoginIntoIBMCloudUsingCLI function before calling this function +// CreateVPC creates a new Virtual Private Cloud (VPC) in IBM Cloud with the specified VPC name. +func CreateVPC(vpcName string) error { + existingVPC, err := IsVPCExist(vpcName) + if err != nil { + return err + } + if !existingVPC { + cmd := exec.Command("ibmcloud", "is", "vpc-create", vpcName) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("VPC creation failed: %v\nCommand output: %s", err, string(output)) + } + fmt.Printf("VPC %s created successfully\n", vpcName) + } else { + fmt.Println("An existing VPC is available") + } + return nil +} + +// Prerequisite: Ensure that you have logged into IBM Cloud using the LoginIntoIBMCloudUsingCLI function before calling this function +// IsVPCExist checks if a VPC with the given name exists. +func IsVPCExist(vpcName string) (bool, error) { + cmd := exec.Command("ibmcloud", "is", "vpcs", "--output", "json") + output, err := cmd.Output() + if err != nil { + return false, fmt.Errorf("failed to list VPCs: %w", err) + } + + return bytes.Contains(output, []byte(vpcName)), nil +} + +// GetBastionServerIP retrieves the IP address from the BastionServer section in the specified INI file. +func GetBastionServerIP(t *testing.T, filePath string, logger *AggregatedLogger) (string, error) { + value, err := GetValueFromIniFile(filePath+"/bastion.ini", "BastionServer") + if err != nil { + return "", fmt.Errorf("failed to get value from bastion.ini: %w", err) + } + logger.Info(t, fmt.Sprintf("Bastion Server IP: %s", value[1])) + return value[1], nil +} + +// GetManagementNodeIPs retrieves the IP addresses from the HPCAASCluster section in the specified INI file. +func GetManagementNodeIPs(t *testing.T, filePath string, logger *AggregatedLogger) ([]string, error) { + value, err := GetValueFromIniFile(filePath+"/compute.ini", "HPCAASCluster") + if err != nil { + return nil, fmt.Errorf("failed to get value from compute.ini: %w", err) + } + logger.Info(t, fmt.Sprintf("Management Node IPs List: %q", value[1:])) + return value[1:], nil +} + +// GetLoginNodeIP retrieves the IP address from the LoginServer section in the specified login INI file. +func GetLoginNodeIP(t *testing.T, filePath string, logger *AggregatedLogger) (string, error) { + value, err := GetValueFromIniFile(filePath+"/login.ini", "LoginServer") + if err != nil { + return "", fmt.Errorf("failed to get value from login.ini: %w", err) + } + logger.Info(t, fmt.Sprintf("Login Server IP: %s", value[1])) + return value[1], nil +} + +// GetLdapServerIP retrieves the IP address from the LdapServer section in the specified login INI file. +func GetLdapServerIP(t *testing.T, filePath string, logger *AggregatedLogger) (string, error) { + value, err := GetValueFromIniFile(filePath+"/ldap.ini", "LDAPServer") + if err != nil { + return "", fmt.Errorf("failed to get value from ldap.ini: %w", err) + } + logger.Info(t, fmt.Sprintf("Ldap Server IP: %s", value[1])) + return value[1], nil +} + +// GetServerIPs retrieves the IP addresses of the bastion server, management nodes, and login node +// from the specified file path in the provided test options, using the provided logger for logging. +// It returns the bastion server IP, a list of management node IPs, the login node IP, and any error encountered. +func GetServerIPs(t *testing.T, options *testhelper.TestOptions, logger *AggregatedLogger) (bastionIP string, managementNodeIPList []string, loginNodeIP string, err error) { + + filePath := options.TerraformOptions.TerraformDir + + // Get bastion server IP and handle errors + bastionIP, err = GetBastionServerIP(t, filePath, logger) + if err != nil { + return "", nil, "", fmt.Errorf("error getting bastion server IP: %v", err) + } + + // Get management node IPs and handle errors + managementNodeIPList, err = GetManagementNodeIPs(t, filePath, logger) + if err != nil { + return "", nil, "", fmt.Errorf("error getting management node IPs: %v", err) + } + + // Get login node IP and handle errors + loginNodeIP, err = GetLoginNodeIP(t, filePath, logger) + if err != nil { + return "", nil, "", fmt.Errorf("error getting login node IP: %v", err) + } + + return bastionIP, managementNodeIPList, loginNodeIP, nil +} + +// GetServerIPsWithLDAP retrieves the IP addresses of various servers, including the LDAP server. +// from the specified file path in the provided test options, using the provided logger for logging. +// It returns the bastion server IP, a list of management node IPs, the login node IP, ldap server IP and any error encountered. +func GetServerIPsWithLDAP(t *testing.T, options *testhelper.TestOptions, logger *AggregatedLogger) (bastionIP string, managementNodeIPList []string, loginNodeIP, ldapIP string, err error) { + // Retrieve the Terraform directory from the options. + filePath := options.TerraformOptions.TerraformDir + + // Get the bastion server IP and handle errors. + bastionIP, err = GetBastionServerIP(t, filePath, logger) + if err != nil { + return "", nil, "", "", fmt.Errorf("error getting bastion server IP: %v", err) + } + + // Get the management node IPs and handle errors. + managementNodeIPList, err = GetManagementNodeIPs(t, filePath, logger) + if err != nil { + return "", nil, "", "", fmt.Errorf("error getting management node IPs: %v", err) + } + + // Get the login node IP and handle errors. + loginNodeIP, err = GetLoginNodeIP(t, filePath, logger) + if err != nil { + return "", nil, "", "", fmt.Errorf("error getting login node IP: %v", err) + } + + // Get the LDAP server IP and handle errors. + ldapIP, err = GetLdapServerIP(t, filePath, logger) + if err != nil { + return "", nil, "", "", fmt.Errorf("error getting LDAP server IP: %v", err) + } + + // Return the retrieved IP addresses and any error. + return bastionIP, managementNodeIPList, loginNodeIP, ldapIP, nil +} + +// Getting BastionID and ComputeID from separately created brand new VPC +func GetSubnetIds(outputs map[string]interface{}) (string, string) { + subnetDetailsList := outputs["subnet_detail_list"] + var bastion []string + var compute []string + for _, val := range subnetDetailsList.(map[string]interface{}) { + for key1, val1 := range val.(map[string]interface{}) { + isContains := strings.Contains(key1, "bastion") + if isContains { + for key2, val2 := range val1.(map[string]interface{}) { + if key2 == "id" { + bastion = append(bastion, val2.(string)) + } + } + } else { + for key2, val2 := range val1.(map[string]interface{}) { + if key2 == "id" { + compute = append(compute, val2.(string)) + } + } + } + + } + } + bastionsubnetId := strings.Join(bastion, ",") + computesubnetIds := strings.Join(compute, ",") + return bastionsubnetId, computesubnetIds +} + +// Getting DNSInstanceID and CustomResolverID from separately created brand new VPC +func GetDnsCustomResolverIds(outputs map[string]interface{}) (string, string) { + customResolverHub := outputs["custom_resolver_hub"] + var instanceId string + var customResolverId string + for _, details := range customResolverHub.([]interface{}) { + for key, val := range details.(map[string]interface{}) { + if key == "instance_id" { + instanceId = val.(string) + } + if key == "custom_resolver_id" { + customResolverId = val.(string) + } + } + } + return instanceId, customResolverId +} + +// GetClusterSecurityID retrieves the security group ID for a cluster based on the provided parameters. +// It logs in to IBM Cloud, executes a command to find the security group ID associated with the cluster prefix, +// and returns the security group ID or an error if any step fails. +func GetClusterSecurityID(t *testing.T, apiKey, region, resourceGroup, clusterPrefix string, logger *AggregatedLogger) (securityGroupID string, err error) { + // If the resource group is "null", set a custom resource group based on the cluster prefix. + if strings.Contains(resourceGroup, "null") { + resourceGroup = fmt.Sprintf("%s-workload-rg", clusterPrefix) + } + + // Log in to IBM Cloud using the API key, region, and resource group. + if err := LoginIntoIBMCloudUsingCLI(t, apiKey, region, resourceGroup); err != nil { + return "", fmt.Errorf("failed to log in to IBM Cloud: %w", err) + } + + // Determine the command to get the security group ID based on the cluster prefix. + cmd := fmt.Sprintf("ibmcloud is security-groups | grep %s-cluster-sg | awk '{print $1}'", clusterPrefix) + + // Execute the command to retrieve the security group ID. + output, err := exec.Command("bash", "-c", cmd).CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to retrieve security group ID: %w", err) + } + + // Trim and check if the result is empty. + securityGroupID = strings.TrimSpace(string(output)) + if securityGroupID == "" { + return "", fmt.Errorf("no security group ID found for cluster prefix %s", clusterPrefix) + } + + logger.Info(t, "securityGroupID: "+securityGroupID) + + return securityGroupID, nil +} + +// UpdateSecurityGroupRules updates the security group with specified port and CIDR based on the provided parameters. +// It logs in to IBM Cloud, determines the appropriate command, and executes it to update the security group. +// Returns an error if any step fails. +func UpdateSecurityGroupRules(t *testing.T, apiKey, region, resourceGroup, clusterPrefix, securityGroupId, cidr, minPort, maxPort string, logger *AggregatedLogger) (err error) { + // If the resource group is "null", set a custom resource group based on the cluster prefix. + if strings.Contains(resourceGroup, "null") { + resourceGroup = fmt.Sprintf("%s-workload-rg", clusterPrefix) + } + + // Log in to IBM Cloud using the API key, region, and resource group. + if err := LoginIntoIBMCloudUsingCLI(t, apiKey, region, resourceGroup); err != nil { + return fmt.Errorf("failed to log in to IBM Cloud: %w", err) + } + + // Determine the command to add a rule to the security group with the specified port and CIDR. + addRuleCmd := fmt.Sprintf("ibmcloud is security-group-rule-add %s inbound tcp --remote %s --port-min %s --port-max %s", securityGroupId, cidr, minPort, maxPort) + + // Execute the command to update the security group. + output, err := exec.Command("bash", "-c", addRuleCmd).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to update security group with port and CIDR: %w", err) + } + + logger.Info(t, "security group updated output: "+strings.TrimSpace(string(output))) + + // Verify if the output contains the expected CIDR. + if !VerifyDataContains(t, strings.TrimSpace(string(output)), cidr, logger) { + return fmt.Errorf("failed to update security group CIDR: %s", string(output)) + } + + // Verify if the output contains the expected minimum port. + if !VerifyDataContains(t, strings.TrimSpace(string(output)), minPort, logger) { + return fmt.Errorf("failed to update security group port: %s", string(output)) + } + + return nil +} + +// GetCustomResolverID retrieves the custom resolver ID for a VPC based on the provided cluster prefix. +// It logs in to IBM Cloud, retrieves the DNS instance ID, and then fetches the custom resolver ID. +// Returns the custom resolver ID and any error encountered. +func GetCustomResolverID(t *testing.T, apiKey, region, resourceGroup, clusterPrefix string, logger *AggregatedLogger) (customResolverID string, err error) { + // If the resource group is "null", set a custom resource group based on the cluster prefix. + if strings.Contains(resourceGroup, "null") { + resourceGroup = fmt.Sprintf("%s-workload-rg", clusterPrefix) + } + + // Log in to IBM Cloud using the API key, region, and resource group. + if err := LoginIntoIBMCloudUsingCLI(t, apiKey, region, resourceGroup); err != nil { + return "", fmt.Errorf("failed to log in to IBM Cloud: %w", err) + } + + // Command to get the DNS instance ID based on the cluster prefix. + dnsInstanceCmd := fmt.Sprintf("ibmcloud dns instances | grep %s | awk '{print $2}'", clusterPrefix) + dnsInstanceIDOutput, err := exec.Command("bash", "-c", dnsInstanceCmd).CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to retrieve DNS instance ID: %w", err) + } + + // Trim whitespace and check if we received a valid DNS instance ID. + dnsInstanceID := strings.TrimSpace(string(dnsInstanceIDOutput)) + if dnsInstanceID == "" { + return "", fmt.Errorf("no DNS instance ID found for cluster prefix %s", clusterPrefix) + } + + // Command to get custom resolvers for the DNS instance ID. + customResolverCmd := fmt.Sprintf("ibmcloud dns custom-resolvers -i %s | awk 'NR>3 {print $1}'", dnsInstanceID) + customResolverIDOutput, err := exec.Command("bash", "-c", customResolverCmd).CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to retrieve custom resolver ID: %w", err) + } + + // Trim whitespace and check if we received a valid custom resolver ID. + customResolverID = strings.TrimSpace(string(customResolverIDOutput)) + if customResolverID == "" { + return "", fmt.Errorf("no custom resolver ID found for DNS instance ID %s", dnsInstanceID) + } + logger.Info(t, "customResolverID: "+customResolverID) + + return customResolverID, nil +} + +// RetrieveAndUpdateSecurityGroup retrieves the security group ID based on the provided cluster prefix, +// then updates the security group with the specified port and CIDR. +// It logs in to IBM Cloud, determines the appropriate commands, and executes them. +// Returns an error if any step fails. +func RetrieveAndUpdateSecurityGroup(t *testing.T, apiKey, region, resourceGroup, clusterPrefix, cidr, minPort, maxPort string, logger *AggregatedLogger) error { + // If the resource group is "null", set a custom resource group based on the cluster prefix. + if strings.Contains(resourceGroup, "null") { + resourceGroup = fmt.Sprintf("%s-workload-rg", clusterPrefix) + } + + // Log in to IBM Cloud using the API key, region, and resource group. + if err := LoginIntoIBMCloudUsingCLI(t, apiKey, region, resourceGroup); err != nil { + return fmt.Errorf("failed to log in to IBM Cloud: %w", err) + } + + // Command to get the security group ID based on the cluster prefix. + getSecurityGroupIDCmd := fmt.Sprintf("ibmcloud is security-groups | grep %s-cluster-sg | awk '{print $1}'", clusterPrefix) + securityGroupIDBytes, err := exec.Command("bash", "-c", getSecurityGroupIDCmd).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to retrieve security group ID: %w", err) + } + + securityGroupID := strings.TrimSpace(string(securityGroupIDBytes)) + if securityGroupID == "" { + return fmt.Errorf("no security group ID found for cluster prefix %s", clusterPrefix) + } + + logger.Info(t, "securityGroupID: "+securityGroupID) + + // Command to add a rule to the security group with the specified port and CIDR. + addRuleCmd := fmt.Sprintf("ibmcloud is security-group-rule-add %s inbound tcp --remote %s --port-min %s --port-max %s", securityGroupID, cidr, minPort, maxPort) + outputBytes, err := exec.Command("bash", "-c", addRuleCmd).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to update security group with port and CIDR: %w", err) + } + + output := strings.TrimSpace(string(outputBytes)) + logger.Info(t, "security group updated output: "+output) + + // Combine output verification steps. + if !VerifyDataContains(t, output, cidr, logger) || !VerifyDataContains(t, output, minPort, logger) { + return fmt.Errorf("failed to update security group with CIDR %s and port %s: %s", cidr, minPort, output) + } + + return nil +} diff --git a/tests/common_utils/ssh_utils.go b/tests/utilities/ssh.go similarity index 100% rename from tests/common_utils/ssh_utils.go rename to tests/utilities/ssh.go