diff --git a/.catalog-onboard-pipeline.yaml b/.catalog-onboard-pipeline.yaml index 4eb02e0a..ebd106ff 100644 --- a/.catalog-onboard-pipeline.yaml +++ b/.catalog-onboard-pipeline.yaml @@ -2,13 +2,13 @@ apiVersion: v1 offerings: # below is an example of a Deployable Architecture (DA) solution -- name: terraform-ibm-modules-terraform-ibm-hpc-ad6e71e # must match the offering name in the ibm_catalog.json +- name: deploy-arch-ibm-hpc # must match the offering name in the ibm_catalog.json kind: solution - catalog_id: fe9d2e76-5ada-44af-821f-b437dcc80f71 - offering_id: 468c5a1c-1f1d-4e5a-b5b3-ec646b1bd298 + catalog_id: 8611e025-10b2-488e-8261-a7f584a5114b + offering_id: bf3c07f8-5a62-4289-8ea0-94dbb2b410e6 # list all of the variations (flavors) you have included in the ibm_catalog.json variations: - - name: advanced + - name: Cluster-with-LSF-v10.1.0.14 mark_ready: false # have pipeline mark as visible if validation passes install_type: fullstack # ensure value matches what is in ibm_catalog.json (fullstack or extension) destroy_resources_on_failure: false # defaults to false if not specified so resources can be inspected to debug failures during validation diff --git a/.gitignore b/.gitignore index a7550ad9..1ae564ec 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ crash.log # to change depending on the environment. # *.tfvars +*.ini # Ignore files for local testing test.tf @@ -51,3 +52,6 @@ terraform.rc # Visual Studio Code .vscode/ + +# tweaks used locally +localtweak__*.tf diff --git a/.secrets.baseline b/.secrets.baseline index fb8b03dc..c7cfd369 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "go.sum|^.secrets.baseline$", "lines": null }, - "generated_at": "2023-12-09T05:22:51Z", + "generated_at": "2024-04-08T15:16:34Z", "plugins_used": [ { "name": "AWSKeyDetector" diff --git a/.tekton/README.md b/.tekton/README.md new file mode 100644 index 00000000..1040f3b3 --- /dev/null +++ b/.tekton/README.md @@ -0,0 +1,42 @@ + +# IBM Cloud HPC - OnePipeline(Tekton) + +## Prerequisites + +Ensure the following are configured on your DevOps: + +- **IBM Cloud Account** +- **Continuous Delivery** +- **Access role with resource_group** + +## Link for setup DevOps +``` +https://cloud.ibm.com/devops/getting-started?env_id=ibm:yp:eu-de +``` + +## Set up Your Go Project + +1. Login to IBMCloud +2. Navigate to Navigation Menu on Left hand side and expand. Then go to DevOps → Toolchains +3. On the Toolchain page select Resource Group and Location where Toolchain has to create +4. Click Create toolchain and it will redirect to Create a Toolchain page +5. After successfull redirect to Toolchain Template collections, select Build your own toolchain +6. On Build your own toolchain page, provide Toolchain name, Resource Group, Region +7. Click Create it will create pipeline. + +## Actions on the OnePipeline + +1. When PR raised to the develop branch from feature branch, pipeline with trigger and it will run PR_TEST related testcases on top of feature branch +2. When Commit/Push happens to develop branch, pipeline will trigger and it will run all PR_TEST and OTHER_TEST testcases + +### Setup required parameters to run pipeline + +1. ibmcloud-api +2. ssh_keys +3. cluster_prefix +4. zones +5. resource_group +6. cluster_id +7. reservation_id + +For additional assistance, contact the project maintainers. diff --git a/.tekton/git-pr-status/listener-git-pr-status.yaml b/.tekton/git-pr-status/listener-git-pr-status.yaml new file mode 100644 index 00000000..83681ec2 --- /dev/null +++ b/.tekton/git-pr-status/listener-git-pr-status.yaml @@ -0,0 +1,170 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: TriggerTemplate +args: [--allow-multiple-documents] +metadata: + name: triggertemplate-git-pr-status +spec: + params: + - name: git-access-token + description: the token to access the git repository for the clone operations + - name: repository + description: The git repo + default: " " + - name: branch + description: the branch for the git repo + - name: directory-name + default: "." + - name: pr-repository + description: The source git repo for the PullRequest + default: " " + - name: pr-branch + description: The source branch for the PullRequest + default: " " + - name: pr-revision + description: the commit id/sha for the PullRequest + default: " " + - name: triggerName + default: "git-pr-process" + - name: pipeline-debug + default: "0" + - name: ssh_keys + default: "" + description: List of names of the SSH keys that is configured in your IBM Cloud account, used to establish a connection to the IBM Cloud HPC bastion and login node. Ensure that the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by according to [SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys). + - name: zones + default: "" + description: IBM Cloud zone names within the selected region where the IBM Cloud HPC cluster should be deployed. Two zone names are required as input value and supported zones for eu-de are eu-de-2, eu-de-3 and for us-east us-east-1, us-east-3. The management nodes and file storage shares will be deployed to the first zone in the list. Compute nodes will be deployed across both first and second zones, where the first zone in the list will be considered as the most preferred zone for compute nodes deployment. [Learn more](https://cloud.ibm.com/docs/vpc?topic=vpc-creating-a-vpc-in-a-different-region#get-zones-using-the-cli). + - name: cluster_prefix + description: Prefix that is used to name the IBM Cloud HPC cluster and IBM Cloud resources that are provisioned to build the IBM Cloud HPC cluster instance. You cannot create more than one instance of the IBM Cloud HPC cluster with the same name. Ensure that the name is unique. + default: cicd-wes + - name: resource_group + description: Resource group name from your IBM Cloud account where the VPC resources should be deployed. Note. If the resource group value is set as null, automation creates two different RG with the name (workload-rg and service-rg). For additional information on resource groups, see [Managing resource groups](https://cloud.ibm.com/docs/account?topic=account-rgs). + default: Default + - name: remote_allowed_ips + default: "" + description: Comma-separated list of IP addresses that can access the IBM Cloud HPC cluster instance through an SSH interface. For security purposes, provide the public IP addresses assigned to the devices that are authorized to establish SSH connections (for example, [\"169.45.117.34\"]). To fetch the IP address of the device, use [https://ipv4.icanhazip.com/](https://ipv4.icanhazip.com/). + - name: compute_image_name_rhel + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: compute_image_name_ubuntu + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: login_image_name + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v2). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: cluster_id + description: Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservation. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment. + default: "" + - name: reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: us_east_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: eu_de_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: us_south_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + resourcetemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: $(params.triggerName)-$(uid)-pvc + spec: + resources: + requests: + storage: 5Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + - apiVersion: tekton.dev/v1beta1 + kind: PipelineRun + metadata: + name: $(params.triggerName)-$(uid) + spec: + pipelineRef: + name: pipeline-git-pr-status + params: + - name: git-access-token + value: $(params.git-access-token) + - name: repository + value: $(params.repository) + - name: branch + value: $(params.branch) + - name: pr-repository + value: $(params.pr-repository) + - name: pr-branch + value: $(params.pr-branch) + - name: pr-revision + value: $(params.pr-revision) + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: directory-name + value: $(params.directory-name) + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: cluster_prefix + value: $(params.cluster_prefix) + - name: resource_group + value: $(params.resource_group) + - name: remote_allowed_ips + value: $(params.remote_allowed_ips) + - name: compute_image_name_rhel + value: $(params.compute_image_name_rhel) + - name: compute_image_name_ubuntu + value: $(params.compute_image_name_ubuntu) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + - name: us_east_reservation_id + value: $(params.us_east_reservation_id) + - name: eu_de_reservation_id + value: $(params.eu_de_reservation_id) + - name: us_south_reservation_id + value: $(params.us_south_reservation_id) + workspaces: + - name: pipeline-ws + persistentVolumeClaim: + claimName: $(params.triggerName)-$(uid)-pvc +... + +--- +apiVersion: tekton.dev/v1beta1 +kind: TriggerBinding +metadata: + name: triggerbinding-git-pr-status-github-pr +spec: + params: + - name: repository + value: "$(event.pull_request.base.repo.clone_url)" + - name: branch + value: "$(event.pull_request.base.ref)" + - name: pr-repository + value: "$(event.pull_request.head.repo.clone_url)" + - name: pr-branch + value: "$(event.pull_request.head.ref)" + - name: pr-revision + value: "$(event.pull_request.head.sha)" + - name: triggerName + value: "github-pullrequest" +... + +--- +apiVersion: tekton.dev/v1beta1 +kind: EventListener +metadata: + name: eventlistener-git-pr-status-github-pr +spec: + triggers: + - binding: + name: triggerbinding-git-pr-status-github-pr + template: + name: triggertemplate-git-pr-status +... diff --git a/.tekton/git-pr-status/pipeline-git-pr-status.yaml b/.tekton/git-pr-status/pipeline-git-pr-status.yaml new file mode 100644 index 00000000..0b2fd80b --- /dev/null +++ b/.tekton/git-pr-status/pipeline-git-pr-status.yaml @@ -0,0 +1,338 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: pipeline-git-pr-status +spec: + params: + - name: repository + description: the git repo + - name: branch + description: the branch for the git repo + - name: pr-repository + description: The source git repo for the PullRequest + default: "" + - name: pr-branch + description: The source branch for the PullRequest + default: "" + - name: pr-revision + description: the commit id/sha for the PullRequest + default: "" + - name: git-access-token + description: the token to access the git repository for the clone operations + default: "" + - name: properties-file + default: "output/thebuild.properties" + - name: git-credentials-json-file + default: "output/secrets/thecredentials.json" + - name: context + default: "commit message check" + - name: description + default: "verify the commit message" + - name: pipeline-debug + default: "0" + - name: directory-name + default: "." + - name: ssh_keys + default: "" + description: List of names of the SSH keys that is configured in your IBM Cloud account, used to establish a connection to the IBM Cloud HPC bastion and login node. Ensure that the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by according to [SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys). + - name: zones + default: "" + description: IBM Cloud zone names within the selected region where the IBM Cloud HPC cluster should be deployed. Two zone names are required as input value and supported zones for eu-de are eu-de-2, eu-de-3 and for us-east us-east-1, us-east-3. The management nodes and file storage shares will be deployed to the first zone in the list. Compute nodes will be deployed across both first and second zones, where the first zone in the list will be considered as the most preferred zone for compute nodes deployment. [Learn more](https://cloud.ibm.com/docs/vpc?topic=vpc-creating-a-vpc-in-a-different-region#get-zones-using-the-cli). + - name: cluster_prefix + description: Prefix that is used to name the IBM Cloud HPC cluster and IBM Cloud resources that are provisioned to build the IBM Cloud HPC cluster instance. You cannot create more than one instance of the IBM Cloud HPC cluster with the same name. Ensure that the name is unique. + default: cicd-wes + - name: resource_group + description: Resource group name from your IBM Cloud account where the VPC resources should be deployed. Note. If the resource group value is set as null, automation creates two different RG with the name (workload-rg and service-rg). For additional information on resource groups, see [Managing resource groups](https://cloud.ibm.com/docs/account?topic=account-rgs). + default: Default + - name: remote_allowed_ips + default: "" + description: Comma-separated list of IP addresses that can access the IBM Cloud HPC cluster instance through an SSH interface. For security purposes, provide the public IP addresses assigned to the devices that are authorized to establish SSH connections (for example, [\"169.45.117.34\"]). To fetch the IP address of the device, use [https://ipv4.icanhazip.com/](https://ipv4.icanhazip.com/). + - name: compute_image_name_rhel + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: compute_image_name_ubuntu + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: login_image_name + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v2). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: cluster_id + description: Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservation. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment. + default: "" + - name: reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + workspaces: + - name: pipeline-ws + tasks: + - name: set-git-pr-pending + taskRef: + name: git-set-commit-status + workspaces: + - name: artifacts + workspace: pipeline-ws + params: + - name: repository + value: $(params.repository) + - name: revision + value: $(params.pr-revision) + - name: context + value: $(params.context) + - name: description + value: $(params.description) + - name: state + value: "pending" + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: git-clone + taskRef: + name: git-clone-repo + runAfter: [set-git-pr-pending] + params: + - name: repository + value: $(params.repository) + - name: branch + value: $(params.branch) + - name: pr-repository + value: $(params.pr-repository) + - name: pr-branch + value: $(params.pr-branch) + - name: pr-revision + value: $(params.pr-revision) + - name: git-access-token + value: $(params.git-access-token) + - name: directory-name + value: $(params.directory-name) + - name: properties-file + value: $(params.properties-file) + - name: git-credentials-json-file + value: $(params.git-credentials-json-file) + - name: pipeline-debug + value: $(params.pipeline-debug) + workspaces: + - name: output + workspace: pipeline-ws + - name: set-git-pr-running + runAfter: [git-clone] + taskRef: + name: git-set-commit-status + workspaces: + - name: artifacts + workspace: pipeline-ws + params: + - name: repository + value: $(params.repository) + - name: revision + value: $(params.pr-revision) + - name: context + value: $(params.context) + - name: description + value: $(params.description) + - name: state + value: "running" + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: pre-requisites-install + runAfter: [git-clone] + taskRef: + name: pre-requisites-install + workspaces: + - name: workspace + workspace: pipeline-ws + params: + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: ssh-key-creation + runAfter: [git-clone, pre-requisites-install] + taskRef: + name: ssh-key-creation + workspaces: + - name: workspace + workspace: pipeline-ws + params: + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: resource_group + value: $(params.resource_group) + - name: wes-hpc-da-rhel-pr + runAfter: [git-clone, pre-requisites-install, ssh-key-creation] + taskRef: + name: wes-hpc-da-rhel-pr + workspaces: + - name: workspace + workspace: pipeline-ws + params: + - name: repository + value: $(params.repository) + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: cluster_prefix + value: $(params.cluster_prefix) + - name: resource_group + value: $(params.resource_group) + - name: remote_allowed_ips + value: $(params.remote_allowed_ips) + - name: compute_image_name_rhel + value: $(params.compute_image_name_rhel) + - name: compute_image_name_ubuntu + value: $(params.compute_image_name_ubuntu) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + - name: wes-hpc-da-ubuntu-pr + runAfter: [git-clone, pre-requisites-install, ssh-key-creation] + taskRef: + name: wes-hpc-da-ubuntu-pr + workspaces: + - name: workspace + workspace: pipeline-ws + params: + - name: repository + value: $(params.repository) + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: cluster_prefix + value: $(params.cluster_prefix) + - name: resource_group + value: $(params.resource_group) + - name: remote_allowed_ips + value: $(params.remote_allowed_ips) + - name: compute_image_name_rhel + value: $(params.compute_image_name_rhel) + - name: compute_image_name_ubuntu + value: $(params.compute_image_name_ubuntu) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + - name: ssh-key-deletion + runAfter: [wes-hpc-da-rhel-pr, wes-hpc-da-ubuntu-pr] + taskRef: + name: ssh-key-deletion + workspaces: + - name: workspace + workspace: pipeline-ws + params: + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: inspect-wes-hpc-infra-log + runAfter: [git-clone, set-git-pr-running, wes-hpc-da-rhel-pr, wes-hpc-da-ubuntu-pr] + workspaces: + - name: workspace + workspace: pipeline-ws + taskSpec: + workspaces: + - name: workspace + description: The git repo will be cloned onto the volume backing this workspace + mountPath: /artifacts + steps: + - name: inspect-infra-error-rhel-pr + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + workingDir: "/artifacts" + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + pwd + LOG_FILE="pipeline-TestRunBasic-rhel*" + DIRECTORY="/artifacts/tests" + if [ -d "$DIRECTORY" ]; then + # Check any error message on the plan/apply log + error_check=$(eval "grep -E -w 'FAIL|Error|ERROR' $DIRECTORY/$LOG_FILE") + if [[ "$error_check" ]]; then + echo "$error_check" + echo "Found Error/FAIL/ERROR in plan/apply log. Please check log." + exit 1 + else + count=`ls -1 $DIRECTORY/test_output/log* 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated and log file not created, check with packages or binaries installation" + exit 1 + else + echo "*******************************************************"" + cat $DIRECTORY/test_output/log* + echo "*******************************************************"" + echo "No Error Found, infra got SUCCESS" + fi + fi + else + echo "$DIRECTORY does not exits" + exit 1 + fi + count=`ls -1 $DIRECTORY/*.cicd 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated, check with packages or binaries installation" + exit 1 + fi + - name: inspect-infra-error-ubuntu-pr + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + workingDir: "/artifacts" + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + pwd + LOG_FILE="pipeline-TestRunBasic-ubuntu*" + DIRECTORY="/artifacts/tests" + if [ -d "$DIRECTORY" ]; then + # Check any error message on the plan/apply log + error_check=$(eval "grep -E -w 'FAIL|Error|ERROR' $DIRECTORY/$LOG_FILE") + if [[ "$error_check" ]]; then + echo "$error_check" + echo "Found Error/FAIL/ERROR in plan/apply log. Please check log." + exit 1 + else + count=`ls -1 $DIRECTORY/test_output/log* 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated and log file not created, check with packages or binaries installation" + exit 1 + else + echo "*******************************************************"" + cat $DIRECTORY/test_output/log* + echo "*******************************************************"" + echo "No Error Found, infra got SUCCESS" + fi + fi + else + echo "$DIRECTORY does not exits" + exit 1 + fi + count=`ls -1 $DIRECTORY/*.cicd 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated, check with packages or binaries installation" + exit 1 + fi + finally: + - name: set-git-commit-status + taskRef: + name: git-set-commit-status + workspaces: + - name: artifacts + workspace: pipeline-ws + params: + - name: repository + value: $(params.repository) + - name: revision + value: $(params.pr-revision) + - name: context + value: $(params.context) + - name: description + value: $(params.description) + - name: state + value: "$(tasks.inspect-wes-hpc-infra-log.status)" + - name: pipeline-debug + value: $(params.pipeline-debug) diff --git a/.tekton/git-trigger/listener-git-trigger.yaml b/.tekton/git-trigger/listener-git-trigger.yaml new file mode 100644 index 00000000..2d064d56 --- /dev/null +++ b/.tekton/git-trigger/listener-git-trigger.yaml @@ -0,0 +1,265 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: TriggerTemplate +args: [--allow-multiple-documents] +metadata: + name: triggertemplate-git-trigger +spec: + params: + - name: git-access-token + description: the token to access the git repository for the clone operations + - name: repository + description: The git repo + default: " " + - name: branch + description: the branch for the git repo + - name: revision + description: the commit id/sha for the clone action + default: " " + - name: pr-repository + description: The source git repo for the PullRequest + default: " " + - name: pr-branch + description: The source branch for the PullRequest + default: " " + - name: pr-revision + description: the commit id/sha for the PullRequest + default: " " + - name: directory-name + default: "." + - name: triggerName + default: "git-pr-process" + - name: pipeline-debug + default: "0" + - name: state + default: "success" + - name: description + default: "The status of tekton commit" + - name: ssh_keys + default: "" + description: List of names of the SSH keys that is configured in your IBM Cloud account, used to establish a connection to the IBM Cloud HPC bastion and login node. Ensure that the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by according to [SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys). + - name: zones + default: "" + description: IBM Cloud zone names within the selected region where the IBM Cloud HPC cluster should be deployed. Two zone names are required as input value and supported zones for eu-de are eu-de-2, eu-de-3 and for us-east us-east-1, us-east-3. The management nodes and file storage shares will be deployed to the first zone in the list. Compute nodes will be deployed across both first and second zones, where the first zone in the list will be considered as the most preferred zone for compute nodes deployment. [Learn more](https://cloud.ibm.com/docs/vpc?topic=vpc-creating-a-vpc-in-a-different-region#get-zones-using-the-cli). + - name: cluster_prefix + description: Prefix that is used to name the IBM Cloud HPC cluster and IBM Cloud resources that are provisioned to build the IBM Cloud HPC cluster instance. You cannot create more than one instance of the IBM Cloud HPC cluster with the same name. Ensure that the name is unique. + default: cicd-wes + - name: resource_group + description: Resource group name from your IBM Cloud account where the VPC resources should be deployed. Note. If the resource group value is set as null, automation creates two different RG with the name (workload-rg and service-rg). For additional information on resource groups, see [Managing resource groups](https://cloud.ibm.com/docs/account?topic=account-rgs). + default: Default + - name: remote_allowed_ips + default: "" + description: Comma-separated list of IP addresses that can access the IBM Cloud HPC cluster instance through an SSH interface. For security purposes, provide the public IP addresses assigned to the devices that are authorized to establish SSH connections (for example, [\"169.45.117.34\"]). To fetch the IP address of the device, use [https://ipv4.icanhazip.com/](https://ipv4.icanhazip.com/). + - name: compute_image_name_rhel + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: compute_image_name_ubuntu + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: login_image_name + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v2). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: cluster_id + description: Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservation. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment. + default: "" + - name: reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: us_east_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: eu_de_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: us_south_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + resourcetemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: $(params.triggerName)-$(uid)-pvc + spec: + resources: + requests: + storage: 5Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + - apiVersion: tekton.dev/v1beta1 + kind: PipelineRun + metadata: + name: $(params.triggerName)-$(uid) + spec: + pipelineRef: + name: pipeline-git-event-processing + params: + - name: git-access-token + value: $(params.git-access-token) + - name: directory-name + value: $(params.directory-name) + - name: repository + value: $(params.repository) + - name: branch + value: $(params.branch) + - name: revision + value: $(params.revision) + - name: pr-repository + value: $(params.pr-repository) + - name: pr-branch + value: $(params.pr-branch) + - name: pr-revision + value: $(params.pr-revision) + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: state + value: $(params.state) + - name: description + value: $(params.description) + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: cluster_prefix + value: $(params.cluster_prefix) + - name: resource_group + value: $(params.resource_group) + - name: remote_allowed_ips + value: $(params.remote_allowed_ips) + - name: compute_image_name_rhel + value: $(params.compute_image_name_rhel) + - name: compute_image_name_ubuntu + value: $(params.compute_image_name_ubuntu) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + - name: us_east_reservation_id + value: $(params.us_east_reservation_id) + - name: eu_de_reservation_id + value: $(params.eu_de_reservation_id) + - name: us_south_reservation_id + value: $(params.us_south_reservation_id) + workspaces: + - name: pipeline-ws + persistentVolumeClaim: + claimName: $(params.triggerName)-$(uid)-pvc +--- +apiVersion: tekton.dev/v1beta1 +kind: TriggerBinding +metadata: + name: triggerbinding-git-trigger-manual +spec: + params: + - name: repository + value: $(params.repository) + - name: branch + value: $(params.branch) + - name: triggerName + value: manual-trigger + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: cluster_prefix + value: $(params.cluster_prefix) + - name: resource_group + value: $(params.resource_group) + - name: remote_allowed_ips + value: $(params.remote_allowed_ips) + - name: compute_image_name_rhel + value: $(params.compute_image_name_rhel) + - name: compute_image_name_ubuntu + value: $(params.compute_image_name_ubuntu) + - name: login_image_name + value: $(params.login_image_name) + - name: reservation_id + value: $(params.reservation_id) + - name: cluster_id + value: $(params.cluster_id) +--- +apiVersion: tekton.dev/v1beta1 +kind: EventListener +metadata: + name: eventlistener-git-trigger-manual +spec: + triggers: + - binding: + name: triggerbinding-git-trigger-manual + template: + name: triggertemplate-git-trigger +--- +apiVersion: tekton.dev/v1beta1 +kind: TriggerBinding +metadata: + name: triggerbinding-git-trigger-github-pr +spec: + params: + - name: repository + value: "$(event.pull_request.base.repo.clone_url)" + - name: branch + value: "$(event.pull_request.base.ref)" + - name: pr-repository + value: "$(event.pull_request.head.repo.clone_url)" + - name: pr-branch + value: "$(event.pull_request.head.ref)" + - name: pr-revision + value: "$(event.pull_request.head.sha)" + - name: triggerName + value: "github-pullrequest" +--- +apiVersion: tekton.dev/v1beta1 +kind: TriggerBinding +metadata: + name: triggerbinding-git-trigger-github-commit +spec: + params: + - name: triggerName + value: "github-commit" + - name: repository + value: "$(event.repository.url)" + - name: revision + value: "$(event.head_commit.id)" + - name: branch + value: "$(event.ref)" + - name: ssh_keys + value: $(event.ref) + - name: reservation_id + value: $(event.ref) + - name: zone + value: $(event.ref) + - name: resource_group + value: $(event.ref) + - name: cluster_id + value: $(event.ref) + - name: compute_image_name_rhel + value: $(event.ref) + - name: compute_image_name_ubuntu + value: $(event.ref) + - name: login_image_name + value: $(event.ref) + +--- +apiVersion: tekton.dev/v1beta1 +kind: EventListener +metadata: + name: eventlistener-git-trigger-github-pr +spec: + triggers: + - binding: + name: triggerbinding-git-trigger-github-pr + template: + name: triggertemplate-git-trigger +--- +apiVersion: tekton.dev/v1beta1 +kind: EventListener +metadata: + name: eventlistener-git-trigger-github-commit +spec: + triggers: + - binding: + name: triggerbinding-git-trigger-github-commit + template: + name: triggertemplate-git-trigger diff --git a/.tekton/git-trigger/pipeline-git-trigger.yaml b/.tekton/git-trigger/pipeline-git-trigger.yaml new file mode 100644 index 00000000..4b49e396 --- /dev/null +++ b/.tekton/git-trigger/pipeline-git-trigger.yaml @@ -0,0 +1,327 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: pipeline-git-event-processing +spec: + params: + - name: repository + description: the git repo + - name: branch + description: the branch for the git repo + - name: revision + description: the commit id/sha for the clone action + default: "" + - name: state + - name: description + default: "The status of tekton commit" + - name: pr-repository + description: The source git repo for the PullRequest + default: "" + - name: pr-branch + description: The source branch for the PullRequest + default: "" + - name: pr-revision + description: the commit id/sha for the PullRequest + default: "" + - name: git-access-token + description: the token to access the git repository for the clone operations + default: "" + - name: properties-file + default: "output/thebuild.properties" + - name: git-credentials-json-file + default: "output/secrets/thecredentials.json" + - name: directory-name + default: "." + - name: pipeline-debug + default: "0" + - name: ssh_keys + default: "" + description: List of names of the SSH keys that is configured in your IBM Cloud account, used to establish a connection to the IBM Cloud HPC bastion and login node. Ensure that the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by according to [SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys). + - name: zones + default: "" + description: IBM Cloud zone names within the selected region where the IBM Cloud HPC cluster should be deployed. Two zone names are required as input value and supported zones for eu-de are eu-de-2, eu-de-3 and for us-east us-east-1, us-east-3. The management nodes and file storage shares will be deployed to the first zone in the list. Compute nodes will be deployed across both first and second zones, where the first zone in the list will be considered as the most preferred zone for compute nodes deployment. [Learn more](https://cloud.ibm.com/docs/vpc?topic=vpc-creating-a-vpc-in-a-different-region#get-zones-using-the-cli). + - name: cluster_prefix + description: Prefix that is used to name the IBM Cloud HPC cluster and IBM Cloud resources that are provisioned to build the IBM Cloud HPC cluster instance. You cannot create more than one instance of the IBM Cloud HPC cluster with the same name. Ensure that the name is unique. + default: cicd-wes + - name: resource_group + description: Resource group name from your IBM Cloud account where the VPC resources should be deployed. Note. If the resource group value is set as null, automation creates two different RG with the name (workload-rg and service-rg). For additional information on resource groups, see [Managing resource groups](https://cloud.ibm.com/docs/account?topic=account-rgs). + default: Default + - name: remote_allowed_ips + default: "" + description: Comma-separated list of IP addresses that can access the IBM Cloud HPC cluster instance through an SSH interface. For security purposes, provide the public IP addresses assigned to the devices that are authorized to establish SSH connections (for example, [\"169.45.117.34\"]). To fetch the IP address of the device, use [https://ipv4.icanhazip.com/](https://ipv4.icanhazip.com/). + - name: compute_image_name_rhel + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: compute_image_name_ubuntu + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: login_image_name + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v2). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: cluster_id + description: Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservation. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment. + default: "" + - name: reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: us_east_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: eu_de_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: us_south_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + workspaces: + - name: pipeline-ws + tasks: + - name: git-clone + taskRef: + name: git-clone-repo + params: + - name: repository + value: $(params.repository) + - name: branch + value: $(params.branch) + - name: revision + value: $(params.revision) + - name: pr-repository + value: $(params.pr-repository) + - name: pr-branch + value: $(params.pr-branch) + - name: pr-revision + value: $(params.pr-revision) + - name: git-access-token + value: $(params.git-access-token) + - name: directory-name + value: "$(params.directory-name)" + - name: properties-file + value: $(params.properties-file) + - name: git-credentials-json-file + value: $(params.git-credentials-json-file) + - name: pipeline-debug + value: $(params.pipeline-debug) + workspaces: + - name: output + workspace: pipeline-ws + - name: pre-requisites-install + runAfter: [git-clone] + taskRef: + name: pre-requisites-install + workspaces: + - name: workspace + workspace: pipeline-ws + params: + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: ssh-key-creation + runAfter: [git-clone, pre-requisites-install] + taskRef: + name: ssh-key-creation + workspaces: + - name: workspace + workspace: pipeline-ws + params: + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: resource_group + value: $(params.resource_group) + - name: wes-hpc-da-rhel + runAfter: [git-clone, pre-requisites-install, ssh-key-creation] + taskRef: + name: wes-hpc-da-rhel + workspaces: + - name: workspace + workspace: pipeline-ws + params: + - name: repository + value: $(params.repository) + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: cluster_prefix + value: $(params.cluster_prefix) + - name: resource_group + value: $(params.resource_group) + - name: remote_allowed_ips + value: $(params.remote_allowed_ips) + - name: compute_image_name_rhel + value: $(params.compute_image_name_rhel) + - name: compute_image_name_ubuntu + value: $(params.compute_image_name_ubuntu) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + - name: us_east_reservation_id + value: $(params.us_east_reservation_id) + - name: eu_de_reservation_id + value: $(params.eu_de_reservation_id) + - name: us_south_reservation_id + value: $(params.us_south_reservation_id) + - name: wes-hpc-da-ubuntu + runAfter: [git-clone, pre-requisites-install, ssh-key-creation] + taskRef: + name: wes-hpc-da-ubuntu + workspaces: + - name: workspace + workspace: pipeline-ws + params: + - name: repository + value: $(params.repository) + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: cluster_prefix + value: $(params.cluster_prefix) + - name: resource_group + value: $(params.resource_group) + - name: remote_allowed_ips + value: $(params.remote_allowed_ips) + - name: compute_image_name_rhel + value: $(params.compute_image_name_rhel) + - name: compute_image_name_ubuntu + value: $(params.compute_image_name_ubuntu) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + - name: us_east_reservation_id + value: $(params.us_east_reservation_id) + - name: eu_de_reservation_id + value: $(params.eu_de_reservation_id) + - name: us_south_reservation_id + value: $(params.us_south_reservation_id) + - name: ssh-key-deletion + runAfter: [wes-hpc-da-rhel, wes-hpc-da-ubuntu] + taskRef: + name: ssh-key-deletion + workspaces: + - name: workspace + workspace: pipeline-ws + params: + - name: pipeline-debug + value: $(params.pipeline-debug) + - name: git-content-inspect + runAfter: [git-clone, wes-hpc-da-rhel, wes-hpc-da-ubuntu] + taskRef: + name: inspect-git-content + workspaces: + - name: workspace + workspace: pipeline-ws + params: + - name: repository + value: $(tasks.git-clone.results.git-repository) + - name: directory-name + value: $(tasks.git-clone.results.clone-directory) + - name: properties-file + value: $(params.properties-file) + - name: git-credentials-json-file + value: $(params.git-credentials-json-file) + - name: git-branch + value: $(tasks.git-clone.results.git-branch) + - name: git-commit + value: $(tasks.git-clone.results.git-commit) + - name: git-user + value: $(tasks.git-clone.results.git-user) + - name: inspect-wes-hpc-infra-log + runAfter: [git-clone, wes-hpc-da-rhel, wes-hpc-da-ubuntu] + workspaces: + - name: workspace + workspace: pipeline-ws + taskSpec: + workspaces: + - name: workspace + description: The git repo will be cloned onto the volume backing this workspace + mountPath: /artifacts + steps: + - name: inspect-infra-error-rhel-suite + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + workingDir: "/artifacts" + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + pwd + LOG_FILE="pipeline-RHEL-Suite*" + DIRECTORY="/artifacts/tests" + if [ -d "$DIRECTORY" ]; then + # Check any error message on the plan/apply log + error_check=$(eval "grep -E -w 'FAIL|Error|ERROR' $DIRECTORY/$LOG_FILE") + if [[ "$error_check" ]]; then + echo "$error_check" + echo "Found Error/FAIL/ERROR in plan/apply log. Please check log." + exit 1 + else + count=`ls -1 $DIRECTORY/test_output/log* 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated and log file not created, check with packages or binaries installation" + exit 1 + else + echo "*******************************************************"" + cat $DIRECTORY/test_output/log* + echo "*******************************************************"" + echo "No Error Found, infra got SUCCESS" + fi + fi + else + echo "$DIRECTORY does not exits" + exit 1 + fi + + count=`ls -1 $DIRECTORY/*.cicd 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated, check with packages or binaries installations" + exit 1 + fi + - name: inspect-infra-error-ubuntu-suite + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + workingDir: "/artifacts" + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + pwd + LOG_FILE="pipeline-UBUNTU-Suite*" + DIRECTORY="/artifacts/tests" + if [ -d "$DIRECTORY" ]; then + # Check any error message on the plan/apply log + error_check=$(eval "grep -E -w 'FAIL|Error|ERROR' $DIRECTORY/$LOG_FILE") + if [[ "$error_check" ]]; then + echo "$error_check" + echo "Found Error/FAIL/ERROR in plan/apply log. Please check log." + exit 1 + else + count=`ls -1 $DIRECTORY/test_output/log* 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated and log file not created, check with packages or binaries installation" + exit 1 + else + echo "*******************************************************"" + cat $DIRECTORY/test_output/log* + echo "*******************************************************"" + echo "No Error Found, infra got SUCCESS" + fi + fi + else + echo "$DIRECTORY does not exits" + exit 1 + fi + + count=`ls -1 $DIRECTORY/*.cicd 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated, check with packages or binaries installation" + exit 1 + fi diff --git a/.tekton/git-trigger/task-inspect-git-content.yaml b/.tekton/git-trigger/task-inspect-git-content.yaml new file mode 100644 index 00000000..a437becd --- /dev/null +++ b/.tekton/git-trigger/task-inspect-git-content.yaml @@ -0,0 +1,81 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: inspect-git-content +spec: + params: + - name: repository + description: the git repo url + - name: directory-name + default: "." + - name: properties-file + default: build.properties + - name: git-credentials-json-file + default: "" + - name: git-branch + description: The active branch for the repository + - name: git-commit + description: The current commit id that was cloned + - name: git-user + description: The auth user that cloned the repository + workspaces: + - name: workspace + mountPath: /artifacts + steps: + - name: inspect-git-content + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.29 + env: + - name: REPOSITORY + value: $(params.repository) + - name: DIRECTORY_NAME + value: $(params.directory-name) + workingDir: /artifacts + command: ["/bin/sh", "-c"] + args: + - | + cd "$DIRECTORY_NAME" + pwd + # show the git content + echo "Executing 'git show-branch --all'" + git show-branch --all + echo "" + # show the directory content recursively + echo "##############" + ls -l -R + echo "" + # show the README.md content + echo "##############" + echo "Executing 'cat README.md'" + cat README.md + echo "" + echo "##############" + echo "Executing 'cat $(workspaces.workspace.path)/$(params.properties-file)'" + cat $(workspaces.workspace.path)/$(params.properties-file) + echo "" + if [ "$(params.git-credentials-json-file)" ]; then + echo "##############" + echo "Executing 'jq $(workspaces.workspace.path)/$(params.git-credentials-json-file)'" + cat $(workspaces.workspace.path)/$(params.git-credentials-json-file) | jq '. | ."GIT_TOKEN"=""' + fi + if [ -z "$GIT_TOKEN" ]; then + AUTHTYPE=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .parameters.auth_type' \ + /cd-config/toolchain.json) + if [[ "${AUTHTYPE}" == "pat" ]]; then + TOKEN=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .parameters.api_token' \ + /cd-config/toolchain.json) + if [[ "${TOKEN}" ]]; then + echo "Using access token from toolchain" + GIT_TOKEN="${TOKEN}" + fi + fi + fi + echo "##############" + echo "Showing task inputs:" + echo "params.repository: $(params.repository)" + echo "params.git-branch: $(params.git-branch)" + echo "params.git-commit: $(params.git-commit)" + echo "params.git-user: $(params.git-user)" + echo "params.directory-name: $(params.directory-name)" diff --git a/.tekton/task-clone.yaml b/.tekton/task-clone.yaml new file mode 100644 index 00000000..0ae26f1f --- /dev/null +++ b/.tekton/task-clone.yaml @@ -0,0 +1,352 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: git-clone-repo +spec: + params: + - name: ibmcloud-api + description: the ibmcloud api + default: https://cloud.ibm.com + - name: continuous-delivery-context-secret + description: name of the secret containing the continuous delivery pipeline context secrets + default: secure-properties + - name: ibmcloud-apikey-secret-key + description: field in the secret that contains the api key used to login to ibmcloud + default: ibmcloud_api_key + - name: git-access-token + description: | + (optional) token to access the git repository. If this token is provided, there will not be an attempt + to use the git token obtained from the authorization flow when adding the git integration in the toolchain + default: "" + - name: resource_group + description: target resource group (name or id) for the ibmcloud login operation + default: "" + - name: repository + description: the git repo url + - name: branch + description: the git branch + default: develop-da + - name: revision + description: | + the git revision/commit to update the git HEAD to. + Default is to mean only use the branch + default: "" + - name: fetch-gitoken-step-image + description: image to use for the fetch-gitoken step (default to icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.29) + default: "icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.29" + - name: git-client-image + description: The image to use to run git clone commands + default: alpine/git + - name: git-max-retry + description: max retry for the git clone operation + default: "1" + - name: pr-repository + description: | + the originating repository where the PullRequest comes from (in case of a fork) + default to '' means same repository (not a fork) or it can be the same as the repository to clone + default: "" + - name: pr-branch + description: | + the branch that is the source of this PullRequest + default: "" + - name: pr-revision + description: the commit/revision in the source branch of the PullRequest that is to be built + default: "" + - name: directory-name + description: | + name of the new directory to clone into. + Default to . in order to clone at the root of the workspace + It will be set to the "humanish" part of the repository if this param is set to blank + default: "." + - name: properties-file + description: file containing properties out of clone task (can be a filepath name relative to the workspace) + default: build.properties + - name: git-credentials-json-file + description: | + JSON file containing the git credentials as found out of the clone task + (can be a file path relative to the workspace). + Default to '' meaning no output of this information + default: "" + - name: pipeline-debug + description: Pipeline debug mode. Value can be 0 or 1. Default to 0 + default: "0" + results: + - name: git-repository + description: The cloned repository + - name: git-branch + description: The active branch for the repository + - name: git-commit + description: The current commit id that was cloned + - name: git-user + description: The auth user that cloned the repository + - name: clone-directory + description: the directory where the cloned repository content is located + workspaces: + - name: output + description: The git repo will be cloned onto the volume backing this workspace + mountPath: /artifacts + stepTemplate: + env: + - name: API_KEY + valueFrom: + secretKeyRef: + name: $(params.continuous-delivery-context-secret) + key: $(params.ibmcloud-apikey-secret-key) + optional: true + - name: PIPELINE_DEBUG + value: $(params.pipeline-debug) + steps: + - name: fetch-git-token + image: $(params.fetch-gitoken-step-image) + env: + - name: REPOSITORY + value: $(params.repository) + script: | + #!/bin/bash + set -e -o pipefail + + if [ $PIPELINE_DEBUG == 1 ]; then + pwd + env + trap env EXIT + set -x + fi + + TOOLCHAIN_ID=$(jq -r '.toolchain_guid' /cd-config/toolchain.json) + ########################################################################## + # Setting HOME explicitly to have ibmcloud plugins available + # doing the export rather than env definition is a workaround + # until https://github.com/tektoncd/pipeline/issues/1836 is fixed + export HOME="/root" + ########################################################################## + if [[ "$REPOSITORY" != *.git ]]; then + echo "Adding .git suffix to Repository URL" + REPOSITORY="${REPOSITORY}.git" + fi + GIT_SERVICE_INSTANCE_ID=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .instance_id' /cd-config/toolchain.json) + if [ -z "$GIT_SERVICE_INSTANCE_ID" ]; then + echo "No Git integration (repository url: $REPOSITORY) found in the toolchain" + exit 1 + fi + GIT_SERVICE_TYPE=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .service_id' /cd-config/toolchain.json) + + if [ "$GIT_SERVICE_TYPE" == "github" ]; then + GIT_AUTH_USER="x-oauth-basic" + else + GIT_AUTH_USER="x-token-auth" + fi; + + GIT_TOKEN="$(params.git-access-token)" + if [ -z "$GIT_TOKEN" ]; then + AUTHTYPE=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .parameters.auth_type' \ + /cd-config/toolchain.json) + if [[ "${AUTHTYPE}" == "pat" ]]; then + TOKEN=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .parameters.api_token' \ + /cd-config/toolchain.json) + if [[ "${TOKEN}" ]]; then + echo "Using access token from toolchain" + GIT_TOKEN="${TOKEN}" + fi + fi + fi + + if [ -z "$GIT_TOKEN" ]; then + echo "Fetching token for $REPOSITORY" + ibmcloud config --check-version false + ibmcloud login -a $(params.ibmcloud-api) --no-region --apikey $API_KEY + if [ "$(params.resource_group)" ]; then + ibmcloud target -g "$(params.resource_group)" + fi + TOKEN=$(ibmcloud iam oauth-tokens --output JSON | jq -r '.iam_token') + GIT_TOKEN_URL=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .parameters.token_url' \ + /cd-config/toolchain.json) + + # GIT_TOKEN_URL is something like + # https://otc-github-consolidated-broker.us-south.devops.cloud.ibm.com/github/token?git_id=github + # as there is already an url param git_id, just put the additional ones using & + + GIT_BROKER_URL="${GIT_TOKEN_URL}&toolchain_id=${TOOLCHAIN_ID}&service_instance_id=${GIT_SERVICE_INSTANCE_ID}&repo_url=${REPOSITORY}" + echo "Doing cURL to ${GIT_BROKER_URL}" + + curl -s -o /steps/github_token_result.json -X GET -H "Accept: application/json" \ + -H "Authorization: $TOKEN" "$GIT_BROKER_URL" + if jq -e '.access_token' /steps/github_token_result.json > /dev/null 2>&1; then + GIT_TOKEN=$(jq -r '.access_token' /steps/github_token_result.json) + echo "Access token found for the Git integration (repository url: $REPOSITORY)" + else + echo "No access token found for the Git integration (repository url: $REPOSITORY)" + cat /steps/github_token_result.json + exit 1 + fi + else + echo "Using git Access Token provided" + fi + + echo "GIT_REPOSITORY=$REPOSITORY" > /steps/next-step-env.properties + echo "GIT_AUTH_USER=$GIT_AUTH_USER" >> /steps/next-step-env.properties + echo "GIT_TOKEN=$GIT_TOKEN" >> /steps/next-step-env.properties + volumeMounts: + - mountPath: /cd-config + name: cd-config-volume + - mountPath: /steps + name: steps-volume + - name: clone-repo + image: $(params.git-client-image) + env: + - name: REPOSITORY + value: $(params.repository) + - name: BRANCH + value: $(params.branch) + - name: REVISION + value: $(params.revision) + - name: PR_REPOSITORY + value: $(params.pr-repository) + - name: PR_BRANCH + value: $(params.pr-branch) + - name: PR_REVISION + value: $(params.pr-revision) + - name: DIRECTORY_NAME + value: $(params.directory-name) + - name: PROPERTIES_FILE + value: $(params.properties-file) + - name: JSON_FILE_GIT_CREDENTIALS + value: $(params.git-credentials-json-file) + script: | + #!/bin/sh + set -e + + if [ $PIPELINE_DEBUG == 1 ]; then + pwd + env + trap env EXIT + set -x + fi + + ########################## + # Workaround until a null/empty param can be flowing to Task + # REVISION if not set is define with a single blank value + BRANCH=$(echo $BRANCH) + REVISION=$(echo $REVISION) + PR_REPOSITORY=$(echo $PR_REPOSITORY) + PR_BRANCH=$(echo $PR_BRANCH) + PR_REVISION=$(echo $PR_REVISION) + ############################ + source /steps/next-step-env.properties + + # If $BRANCH is a full git ref then only keep the name part + # this is using sh so do this in two steps + BRANCH=$(echo ${BRANCH#"refs/heads/"}) + BRANCH=$(echo ${BRANCH#"refs/tags/"}) + + echo "Cloning $REPOSITORY" + # Add the proper creds to the git repository + GIT_URL=$(echo "$REPOSITORY" | sed -e "s/:\/\//:\/\/$GIT_AUTH_USER:$GIT_TOKEN@/g") + ARTIFACTS_PATH="$(workspaces.output.path)" + cd $ARTIFACTS_PATH + + # The attached storage for running pipelines is changing to a Block type from a Local PV. + # This Block type has a "lost+found" directory present. + # This causes an issue when the Git Clone task destination is set to root ".", which is the pattern that the Tekton templates use. + # When attempting a clone, it detects that the clone directory is not empty and fails. + # Note: The remove command needs to come after the cd $ARTIFACTS_PATH command + rm -rf "lost+found" + + if [ -z "${PR_BRANCH}" ]; then + if [ "$REVISION" ]; then + # check if the branch exists (that may not be the case in case of a pipeline-run re-run) + echo "Fetching specific -${REVISION}- commit" + if [ -z "$BRANCH" ]; then + # No branch provided + _clone_command='git clone -q -n $GIT_URL $DIRECTORY_NAME' + elif git ls-remote --heads --exit-code $GIT_URL $BRANCH > /dev/null 2>&1; then + _clone_command='git clone -q -b "$BRANCH" $GIT_URL $DIRECTORY_NAME' + else + echo "branch $BRANCH does not exists in $REPOSITORY" + _clone_command='git clone -q -n $GIT_URL $DIRECTORY_NAME' + fi + else + if [ -z "$BRANCH" ]; then + # No branch provided + echo "No branch or revision provided." + _clone_command='git clone -q -n $GIT_URL $DIRECTORY_NAME' + else + _clone_command='git clone -q -b "$BRANCH" $GIT_URL $DIRECTORY_NAME' + fi + fi + elif [ "${PR_BRANCH}" ]; then + _clone_command='git clone -q -b "$PR_BRANCH" $GIT_URL $DIRECTORY_NAME' + echo "git clone of $GIT_REPOSITORY (branch $PR_BRANCH - commit $GIT_COMMIT) done in directory $DIRECTORY_NAME" + fi + + _max_retry=$(params.git-max-retry) + set +e + eval "$_clone_command" + _clone_code=$? + _retry_counter=1 + while [ $_retry_counter -le $_max_retry ]; do + if [ $_clone_code != 0 ]; then + echo "Clone was not successful. Code $_clone_code - Retrying shortly..." + sleep 10 + if [ $_retry_counter -eq $_max_retry ]; then + set -e # reset on the last attempt so we fail if all attemps fail + fi + eval "$_clone_command" + _clone_code=$? + let "_retry_counter++" + else + break + fi + done + echo "Repository $REPOSITORY successfully cloned" + + GIT_COMMIT=$(git show-ref --head | head -n1 | awk '{print $1}') + + git submodule update --init + + cd $current_dir + if [ "$PROPERTIES_FILE" ]; then + # Ensure directory is there + mkdir -p $ARTIFACTS_PATH/$(dirname "$PROPERTIES_FILE") + echo "GIT_URL=$REPOSITORY" >> $ARTIFACTS_PATH/$PROPERTIES_FILE + echo "GIT_BRANCH=$BRANCH" >> $ARTIFACTS_PATH/$PROPERTIES_FILE + echo "GIT_COMMIT=$GIT_COMMIT" >> $ARTIFACTS_PATH/$PROPERTIES_FILE + # Change write access permission to allow subsequent task(s) to update if needed + chmod go+rw $ARTIFACTS_PATH/$PROPERTIES_FILE + echo "$PROPERTIES_FILE content:" + cat $ARTIFACTS_PATH/$PROPERTIES_FILE + fi + if [ "$JSON_FILE_GIT_CREDENTIALS" ]; then + # Ensure directory is there + mkdir -p $ARTIFACTS_PATH/$(dirname "$JSON_FILE_GIT_CREDENTIALS") + # Create a JSON file as output of this step to store the git credentials for future use + echo "{" > "${ARTIFACTS_PATH}/${JSON_FILE_GIT_CREDENTIALS}" + echo "\"GIT_REPOSITORY\":\"${REPOSITORY}\"," >> "${ARTIFACTS_PATH}/${JSON_FILE_GIT_CREDENTIALS}" + echo "\"GIT_AUTH_USER\":\"${GIT_AUTH_USER}\"," >> "${ARTIFACTS_PATH}/${JSON_FILE_GIT_CREDENTIALS}" + echo "\"GIT_TOKEN\":\"${GIT_TOKEN}\"" >> "${ARTIFACTS_PATH}/${JSON_FILE_GIT_CREDENTIALS}" + echo "}" >> "${ARTIFACTS_PATH}/${JSON_FILE_GIT_CREDENTIALS}" + fi + + # Record task results + echo -n "${REPOSITORY}" > $(results.git-repository.path) + echo -n "${BRANCH}" > $(results.git-branch.path) + echo -n "${GIT_COMMIT}" > $(results.git-commit.path) + echo -n "${GIT_AUTH_USER}" > $(results.git-user.path) + echo -n "${DIRECTORY_NAME}" > $(results.clone-directory.path) + volumeMounts: + - mountPath: /steps + name: steps-volume + volumes: + - name: steps-volume + emptyDir: {} + - name: cd-config-volume + configMap: + name: toolchain + items: + - key: toolchain.json + path: toolchain.json diff --git a/.tekton/task-infra-rhel.yaml b/.tekton/task-infra-rhel.yaml new file mode 100644 index 00000000..a16bbe05 --- /dev/null +++ b/.tekton/task-infra-rhel.yaml @@ -0,0 +1,368 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: wes-hpc-da-rhel +spec: + params: + - name: ibmcloud-api + description: the ibmcloud api + default: https://cloud.ibm.com + - name: continuous-delivery-context-secret + description: name of the secret containing the continuous delivery pipeline context secrets + default: secure-properties + - name: ibmcloud-apikey-secret-key + description: field in the secret that contains the api key used to login to ibmcloud + default: ibmcloud_api_key + - name: pipeline-debug + description: Pipeline debug mode. Value can be 0 or 1. Default to 0 + default: "0" + - name: pr-branch + description: The source branch for the PullRequest + default: "" + - name: directory-name + default: "." + - name: repository + description: the git repo url + - name: ssh_keys + default: "" + description: List of names of the SSH keys that is configured in your IBM Cloud account, used to establish a connection to the IBM Cloud HPC bastion and login node. Ensure that the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by according to [SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys). + - name: zones + default: "" + description: IBM Cloud zone names within the selected region where the IBM Cloud HPC cluster should be deployed. Two zone names are required as input value and supported zones for eu-de are eu-de-2, eu-de-3 and for us-east us-east-1, us-east-3. The management nodes and file storage shares will be deployed to the first zone in the list. Compute nodes will be deployed across both first and second zones, where the first zone in the list will be considered as the most preferred zone for compute nodes deployment. [Learn more](https://cloud.ibm.com/docs/vpc?topic=vpc-creating-a-vpc-in-a-different-region#get-zones-using-the-cli). + - name: cluster_prefix + description: Prefix that is used to name the IBM Cloud HPC cluster and IBM Cloud resources that are provisioned to build the IBM Cloud HPC cluster instance. You cannot create more than one instance of the IBM Cloud HPC cluster with the same name. Ensure that the name is unique. + default: cicd-wes + - name: resource_group + description: Resource group name from your IBM Cloud account where the VPC resources should be deployed. Note. If the resource group value is set as null, automation creates two different RG with the name (workload-rg and service-rg). For additional information on resource groups, see [Managing resource groups](https://cloud.ibm.com/docs/account?topic=account-rgs). + default: Default + - name: remote_allowed_ips + default: "" + description: Comma-separated list of IP addresses that can access the IBM Cloud HPC cluster instance through an SSH interface. For security purposes, provide the public IP addresses assigned to the devices that are authorized to establish SSH connections (for example, [\"169.45.117.34\"]). To fetch the IP address of the device, use [https://ipv4.icanhazip.com/](https://ipv4.icanhazip.com/). + - name: compute_image_name_rhel + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: compute_image_name_ubuntu + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: login_image_name + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v2). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: cluster_id + description: Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservation. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment. + default: "" + - name: reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: us_east_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: eu_de_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: us_south_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + workspaces: + - name: workspace + mountPath: /artifacts + stepTemplate: + env: + - name: API_KEY + valueFrom: + secretKeyRef: + name: $(params.continuous-delivery-context-secret) + key: $(params.ibmcloud-apikey-secret-key) + optional: true + - name: PIPELINE_DEBUG + value: $(params.pipeline-debug) + steps: + - name: rhel-suite-1 + onError: continue + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + env: + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: resource_group + value: $(params.resource_group) + - name: compute_image_name_rhel + value: $(params.compute_image_name_rhel) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + - name: us_east_reservation_id + value: $(params.us_east_reservation_id) + - name: eu_de_reservation_id + value: $(params.eu_de_reservation_id) + - name: us_south_reservation_id + value: $(params.us_south_reservation_id) + workingDir: "/artifacts" + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + + if [[ "${PIPELINE_DEBUG}" == "true" ]]; then + pwd + env + trap env EXIT + set -x + fi + + unzip -n terraform_1.5.7_linux_amd64.zip + mv terraform /usr/local/bin/terraform + terraform --version + + cd $(pwd)/ && + echo "export PATH=\$PATH:$(pwd)/go/bin:\$HOME/go/bin" >> ~/.bashrc && + echo "export GOROOT=$(pwd)/go" >> ~/.bashrc + source ~/.bashrc + go version + + echo -e "************** Going to run in RHEL-Suite-1 ************** + 1.TestRunBasic + 2.TestRunAppCenter + 3.TestRunInUsEastRegion + 4.TestRunInEuDeRegion + 5.TestRunInUSSouthRegion + 6.TestRunNoKMSAndHTOff" + + export TF_VAR_ibmcloud_api_key=$API_KEY + export SSH_FILE_PATH="/artifacts/.ssh/id_rsa" + CICD_SSH_KEY="cicd-toolchain" + + # Check artifacts/tests folder exists or not + DIRECTORY="/artifacts/tests" + if [ -d "$DIRECTORY" ]; then + cd $DIRECTORY + LOG_FILE=pipeline-RHEL-Suite-1-$(date +%d%m%Y).cicd + echo "**************Validating on RHEL-Suite-1**************" + SSH_KEY=$CICD_SSH_KEY US_EAST_RESERVATION_ID=$us_east_reservation_id US_SOUTH_RESERVATION_ID=$us_south_reservation_id EU_DE_RESERVATION_ID=$eu_de_reservation_id COMPUTE_IMAGE_NAME=$compute_image_name_rhel LOGIN_NODE_IMAGE_NAME=$login_image_name ZONE=$zones RESERVATION_ID=$reservation_id CLUSTER_ID=$cluster_id RESOURCE_GROUP=$resource_group go test -v -timeout 9000m -run "TestRunBasic|TestRunAppCenter|TestRunInUsEastRegion|TestRunInEuDeRegion|TestRunInUSSouthRegion|TestRunNoKMSAndHTOff" | tee -a $LOG_FILE + else + pwd + ls -a + echo "$DIRECTORY does not exists" + exit 1 + fi + + # Check any error message on the plan/apply log + error_check=$(eval "grep -E -w 'FAIL|Error|ERROR' $DIRECTORY/$LOG_FILE") + if [[ "$error_check" ]]; then + echo "Found Error/FAIL/ERROR in plan/apply log. Please check log." + exit 1 + fi + + echo "*******************************************************" + count=`ls -1 $DIRECTORY/test_output/log* 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated and log file not created, check with packages or binaries installation" + exit 1 + else + cat $DIRECTORY/test_output/log* + fi + echo "*******************************************************" + + count=`ls -1 $DIRECTORY/*.cicd 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated, check with packages or binaries installation" + exit 1 + fi + - name: rhel-suite-2 + onError: continue + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + env: + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: resource_group + value: $(params.resource_group) + - name: compute_image_name_rhel + value: $(params.compute_image_name_rhel) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + - name: us_east_reservation_id + value: $(params.us_east_reservation_id) + - name: eu_de_reservation_id + value: $(params.eu_de_reservation_id) + - name: us_south_reservation_id + value: $(params.us_south_reservation_id) + workingDir: "/artifacts" + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + + if [[ "${PIPELINE_DEBUG}" == "true" ]]; then + pwd + env + trap env EXIT + set -x + fi + + unzip -n terraform_1.5.7_linux_amd64.zip + mv terraform /usr/local/bin/terraform + terraform --version + + cd $(pwd)/ && + echo "export PATH=\$PATH:$(pwd)/go/bin:\$HOME/go/bin" >> ~/.bashrc && + echo "export GOROOT=$(pwd)/go" >> ~/.bashrc + source ~/.bashrc + go version + + echo "************** Going to run in RHEL-Suite-2 ************** + 1.TestRunLDAP + 2.TestRunUsingExistingKMS + 3.TestRunLDAPAndPac + 4.TestRunCustomRGAsNull + 5.TestRunCustomRGAsNonDefault" + + export TF_VAR_ibmcloud_api_key=$API_KEY + export SSH_FILE_PATH="/artifacts/.ssh/id_rsa" + CICD_SSH_KEY="cicd-toolchain" + + # Check artifacts/tests folder exists or not + DIRECTORY="/artifacts/tests" + if [ -d "$DIRECTORY" ]; then + cd $DIRECTORY + LOG_FILE=pipeline-RHEL-Suite-2-$(date +%d%m%Y).cicd + echo "**************Validating on RHEL-Suite-2**************" + SSH_KEY=$CICD_SSH_KEY US_EAST_RESERVATION_ID=$us_east_reservation_id US_SOUTH_RESERVATION_ID=$us_south_reservation_id EU_DE_RESERVATION_ID=$eu_de_reservation_id COMPUTE_IMAGE_NAME=$compute_image_name_rhel LOGIN_NODE_IMAGE_NAME=$login_image_name ZONE=$zones RESERVATION_ID=$reservation_id CLUSTER_ID=$cluster_id RESOURCE_GROUP=$resource_group go test -v -timeout 9000m -run "TestRunLDAP|TestRunUsingExistingKMS|TestRunLDAPAndPac|TestRunCustomRGAsNull|TestRunCustomRGAsNonDefault" | tee -a $LOG_FILE + else + pwd + ls -a + echo "$DIRECTORY does not exists" + exit 1 + fi + + # Check any error message on the plan/apply log + error_check=$(eval "grep -E -w 'FAIL|Error|ERROR' $DIRECTORY/$LOG_FILE") + if [[ "$error_check" ]]; then + echo "Found Error/FAIL/ERROR in plan/apply log. Please check log." + exit 1 + fi + + echo "*******************************************************" + count=`ls -1 $DIRECTORY/test_output/log* 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated and log file not created, check with packages or binaries installation" + exit 1 + else + cat $DIRECTORY/test_output/log* + fi + echo "*******************************************************" + + count=`ls -1 $DIRECTORY/*.cicd 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated, check with packages or binaries installation" + exit 1 + fi + - name: rhel-suite-3 + onError: continue + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + env: + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: resource_group + value: $(params.resource_group) + - name: compute_image_name_rhel + value: $(params.compute_image_name_rhel) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + - name: us_east_reservation_id + value: $(params.us_east_reservation_id) + - name: eu_de_reservation_id + value: $(params.eu_de_reservation_id) + - name: us_south_reservation_id + value: $(params.us_south_reservation_id) + workingDir: "/artifacts" + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + + if [[ "${PIPELINE_DEBUG}" == "true" ]]; then + pwd + env + trap env EXIT + set -x + fi + + unzip -n terraform_1.5.7_linux_amd64.zip + mv terraform /usr/local/bin/terraform + terraform --version + + cd $(pwd)/ && + echo "export PATH=\$PATH:$(pwd)/go/bin:\$HOME/go/bin" >> ~/.bashrc && + echo "export GOROOT=$(pwd)/go" >> ~/.bashrc + source ~/.bashrc + go version + + echo "************** Going to run in UBUNTU-Suite-3 ************** + 1.TestRunCreateVpc + 1.1 RunHpcExistingVpcSubnetId + 1.2 RunHpcExistingVpcCidr + 2.TestRunVpcWithCustomDns + 2.1 RunHpcExistingVpcCustomDnsExist + 2.2 RunHpcExistingVpcCustomExistDnsNew + 2.3 RunHpcNewVpcCustomNullExistDns + 2.4 RunHpcNewVpcExistCustomDnsNull" + + export TF_VAR_ibmcloud_api_key=$API_KEY + export SSH_FILE_PATH="/artifacts/.ssh/id_rsa" + CICD_SSH_KEY="cicd-toolchain" + + # Check artifacts/tests folder exists or not + DIRECTORY="/artifacts/tests" + if [ -d "$DIRECTORY" ]; then + cd $DIRECTORY + LOG_FILE=pipeline-RHEL-Suite-3-$(date +%d%m%Y).cicd + echo "**************Validating on RHEL-Suite-3**************" + SSH_KEY=$CICD_SSH_KEY US_EAST_RESERVATION_ID=$us_east_reservation_id US_SOUTH_RESERVATION_ID=$us_south_reservation_id EU_DE_RESERVATION_ID=$eu_de_reservation_id COMPUTE_IMAGE_NAME=$compute_image_name_rhel LOGIN_NODE_IMAGE_NAME=$login_image_name ZONE=$zones RESERVATION_ID=$reservation_id CLUSTER_ID=$cluster_id RESOURCE_GROUP=$resource_group go test -v -timeout 9000m -run "TestRunCreateVpc|TestRunVpcWithCustomDns" | tee -a $LOG_FILE + else + pwd + ls -a + echo "$DIRECTORY does not exists" + exit 1 + fi + + # Check any error message on the plan/apply log + error_check=$(eval "grep -E -w 'FAIL|Error|ERROR' $DIRECTORY/$LOG_FILE") + if [[ "$error_check" ]]; then + echo "Found Error/FAIL/ERROR in plan/apply log. Please check log." + exit 1 + fi + + echo "*******************************************************" + count=`ls -1 $DIRECTORY/test_output/log* 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated and log file not created, check with packages or binaries installation" + exit 1 + else + cat $DIRECTORY/test_output/log* + fi + echo "*******************************************************" + + count=`ls -1 $DIRECTORY/*.cicd 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated, check with packages or binaries installation" + exit 1 + fi diff --git a/.tekton/task-pr-rhel.yaml b/.tekton/task-pr-rhel.yaml new file mode 100644 index 00000000..00464759 --- /dev/null +++ b/.tekton/task-pr-rhel.yaml @@ -0,0 +1,152 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: wes-hpc-da-rhel-pr +spec: + params: + - name: ibmcloud-api + description: the ibmcloud api + default: https://cloud.ibm.com + - name: continuous-delivery-context-secret + description: name of the secret containing the continuous delivery pipeline context secrets + default: secure-properties + - name: ibmcloud-apikey-secret-key + description: field in the secret that contains the api key used to login to ibmcloud + default: ibmcloud_api_key + - name: pipeline-debug + description: Pipeline debug mode. Value can be 0 or 1. Default to 0 + default: "0" + - name: pr-branch + description: The source branch for the PullRequest + default: "" + - name: directory-name + default: "." + - name: repository + description: the git repo url + - name: ssh_keys + default: "" + description: List of names of the SSH keys that is configured in your IBM Cloud account, used to establish a connection to the IBM Cloud HPC bastion and login node. Ensure that the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by according to [SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys). + - name: zones + default: "" + description: IBM Cloud zone names within the selected region where the IBM Cloud HPC cluster should be deployed. Two zone names are required as input value and supported zones for eu-de are eu-de-2, eu-de-3 and for us-east us-east-1, us-east-3. The management nodes and file storage shares will be deployed to the first zone in the list. Compute nodes will be deployed across both first and second zones, where the first zone in the list will be considered as the most preferred zone for compute nodes deployment. [Learn more](https://cloud.ibm.com/docs/vpc?topic=vpc-creating-a-vpc-in-a-different-region#get-zones-using-the-cli). + - name: cluster_prefix + description: Prefix that is used to name the IBM Cloud HPC cluster and IBM Cloud resources that are provisioned to build the IBM Cloud HPC cluster instance. You cannot create more than one instance of the IBM Cloud HPC cluster with the same name. Ensure that the name is unique. + default: cicd-wes + - name: resource_group + description: Resource group name from your IBM Cloud account where the VPC resources should be deployed. Note. If the resource group value is set as null, automation creates two different RG with the name (workload-rg and service-rg). For additional information on resource groups, see [Managing resource groups](https://cloud.ibm.com/docs/account?topic=account-rgs). + default: Default + - name: remote_allowed_ips + default: "" + description: Comma-separated list of IP addresses that can access the IBM Cloud HPC cluster instance through an SSH interface. For security purposes, provide the public IP addresses assigned to the devices that are authorized to establish SSH connections (for example, [\"169.45.117.34\"]). To fetch the IP address of the device, use [https://ipv4.icanhazip.com/](https://ipv4.icanhazip.com/). + - name: compute_image_name_rhel + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: compute_image_name_ubuntu + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: login_image_name + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v2). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: cluster_id + description: Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservation. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment. + default: "" + - name: reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + workspaces: + - name: workspace + mountPath: /artifacts + stepTemplate: + env: + - name: API_KEY + valueFrom: + secretKeyRef: + name: $(params.continuous-delivery-context-secret) + key: $(params.ibmcloud-apikey-secret-key) + optional: true + - name: PIPELINE_DEBUG + value: $(params.pipeline-debug) + steps: + - name: test-run-basic-rhel-pr + onError: continue + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + env: + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: resource_group + value: $(params.resource_group) + - name: compute_image_name_rhel + value: $(params.compute_image_name_rhel) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + workingDir: "/artifacts" + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + + if [[ "${PIPELINE_DEBUG}" == "true" ]]; then + pwd + env + trap env EXIT + set -x + fi + + unzip -n terraform_1.5.7_linux_amd64.zip + mv terraform /usr/local/bin/terraform + terraform --version + + cd $(pwd)/ && + echo "export PATH=\$PATH:$(pwd)/go/bin:\$HOME/go/bin" >> ~/.bashrc && + echo "export GOROOT=$(pwd)/go" >> ~/.bashrc + source ~/.bashrc + go version + + export TF_VAR_ibmcloud_api_key=$API_KEY + export SSH_FILE_PATH="/artifacts/.ssh/id_rsa" + CICD_SSH_KEY="cicd-toolchain" + + # Check artifacts/tests folder exists or not + DIRECTORY="/artifacts/tests" + if [ -d "$DIRECTORY" ]; then + cd $DIRECTORY + LOG_FILE=pipeline-TestRunBasic-rhel-$(date +%d%m%Y).cicd + echo "**************Validating on TestRunBasic**************" + SSH_KEY=$CICD_SSH_KEY COMPUTE_IMAGE_NAME=$compute_image_name_rhel LOGIN_NODE_IMAGE_NAME=$login_image_name ZONE=$zones RESERVATION_ID=$reservation_id CLUSTER_ID=$cluster_id RESOURCE_GROUP=$resource_group go test -v -timeout 9000m -run TestRunBasic | tee -a $LOG_FILE + else + pwd + ls -a + echo "$DIRECTORY does not exists" + exit 1 + fi + + # Check any error message on the plan/apply log + error_check=$(eval "grep -E -w 'FAIL|Error|ERROR' $DIRECTORY/$LOG_FILE") + if [[ "$error_check" ]]; then + echo "Found Error/FAIL/ERROR in plan/apply log. Please check log." + exit 1 + fi + + echo "*******************************************************" + count=`ls -1 $DIRECTORY/test_output/log* 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated and log file not created, check with packages or binaries installation" + exit 1 + else + cat $DIRECTORY/test_output/log* + fi + echo "*******************************************************" + + count=`ls -1 $DIRECTORY/*.cicd 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated, check with packages or binaries installation" + exit 1 + fi diff --git a/.tekton/task-pr-ubuntu.yaml b/.tekton/task-pr-ubuntu.yaml new file mode 100644 index 00000000..612a73e0 --- /dev/null +++ b/.tekton/task-pr-ubuntu.yaml @@ -0,0 +1,152 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: wes-hpc-da-ubuntu-pr +spec: + params: + - name: ibmcloud-api + description: the ibmcloud api + default: https://cloud.ibm.com + - name: continuous-delivery-context-secret + description: name of the secret containing the continuous delivery pipeline context secrets + default: secure-properties + - name: ibmcloud-apikey-secret-key + description: field in the secret that contains the api key used to login to ibmcloud + default: ibmcloud_api_key + - name: pipeline-debug + description: Pipeline debug mode. Value can be 0 or 1. Default to 0 + default: "0" + - name: pr-branch + description: The source branch for the PullRequest + default: "" + - name: directory-name + default: "." + - name: repository + description: the git repo url + - name: ssh_keys + default: "" + description: List of names of the SSH keys that is configured in your IBM Cloud account, used to establish a connection to the IBM Cloud HPC bastion and login node. Ensure that the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by according to [SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys). + - name: zones + default: "" + description: IBM Cloud zone names within the selected region where the IBM Cloud HPC cluster should be deployed. Two zone names are required as input value and supported zones for eu-de are eu-de-2, eu-de-3 and for us-east us-east-1, us-east-3. The management nodes and file storage shares will be deployed to the first zone in the list. Compute nodes will be deployed across both first and second zones, where the first zone in the list will be considered as the most preferred zone for compute nodes deployment. [Learn more](https://cloud.ibm.com/docs/vpc?topic=vpc-creating-a-vpc-in-a-different-region#get-zones-using-the-cli). + - name: cluster_prefix + description: Prefix that is used to name the IBM Cloud HPC cluster and IBM Cloud resources that are provisioned to build the IBM Cloud HPC cluster instance. You cannot create more than one instance of the IBM Cloud HPC cluster with the same name. Ensure that the name is unique. + default: cicd-wes + - name: resource_group + description: Resource group name from your IBM Cloud account where the VPC resources should be deployed. Note. If the resource group value is set as null, automation creates two different RG with the name (workload-rg and service-rg). For additional information on resource groups, see [Managing resource groups](https://cloud.ibm.com/docs/account?topic=account-rgs). + default: Default + - name: remote_allowed_ips + default: "" + description: Comma-separated list of IP addresses that can access the IBM Cloud HPC cluster instance through an SSH interface. For security purposes, provide the public IP addresses assigned to the devices that are authorized to establish SSH connections (for example, [\"169.45.117.34\"]). To fetch the IP address of the device, use [https://ipv4.icanhazip.com/](https://ipv4.icanhazip.com/). + - name: compute_image_name_rhel + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: compute_image_name_ubuntu + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: login_image_name + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v2). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: cluster_id + description: Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservation. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment. + default: "" + - name: reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + workspaces: + - name: workspace + mountPath: /artifacts + stepTemplate: + env: + - name: API_KEY + valueFrom: + secretKeyRef: + name: $(params.continuous-delivery-context-secret) + key: $(params.ibmcloud-apikey-secret-key) + optional: true + - name: PIPELINE_DEBUG + value: $(params.pipeline-debug) + steps: + - name: test-run-basic-ubuntu-pr + onError: continue + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + env: + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: resource_group + value: $(params.resource_group) + - name: compute_image_name_ubuntu + value: $(params.compute_image_name_ubuntu) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + workingDir: "/artifacts" + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + + if [[ "${PIPELINE_DEBUG}" == "true" ]]; then + pwd + env + trap env EXIT + set -x + fi + + unzip -n terraform_1.5.7_linux_amd64.zip + mv terraform /usr/local/bin/terraform + terraform --version + + cd $(pwd)/ && + echo "export PATH=\$PATH:$(pwd)/go/bin:\$HOME/go/bin" >> ~/.bashrc && + echo "export GOROOT=$(pwd)/go" >> ~/.bashrc + source ~/.bashrc + go version + + export TF_VAR_ibmcloud_api_key=$API_KEY + export SSH_FILE_PATH="/artifacts/.ssh/id_rsa" + CICD_SSH_KEY="cicd-toolchain" + + # Check artifacts/tests folder exists or not + DIRECTORY="/artifacts/tests" + if [ -d "$DIRECTORY" ]; then + cd $DIRECTORY + LOG_FILE=pipeline-TestRunBasic-ubuntu-$(date +%d%m%Y).cicd + echo "**************Validating on TestRunBasic**************" + SSH_KEY=$CICD_SSH_KEY COMPUTE_IMAGE_NAME=$compute_image_name_ubuntu LOGIN_NODE_IMAGE_NAME=$login_image_name ZONE=$zones RESERVATION_ID=$reservation_id CLUSTER_ID=$cluster_id RESOURCE_GROUP=$resource_group go test -v -timeout 9000m -run TestRunBasic | tee -a $LOG_FILE + else + pwd + ls -a + echo "$DIRECTORY does not exists" + exit 1 + fi + + # Check any error message on the plan/apply log + error_check=$(eval "grep -E -w 'FAIL|Error|ERROR' $DIRECTORY/$LOG_FILE") + if [[ "$error_check" ]]; then + echo "Found Error/FAIL/ERROR in plan/apply log. Please check log." + exit 1 + fi + + echo "*******************************************************" + count=`ls -1 $DIRECTORY/test_output/log* 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated and log file not created, check with packages or binaries installation" + exit 1 + else + cat $DIRECTORY/test_output/log* + fi + echo "*******************************************************" + + count=`ls -1 $DIRECTORY/*.cicd 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated, check with packages or binaries installation" + exit 1 + fi diff --git a/.tekton/task-pre-requisites-install.yaml b/.tekton/task-pre-requisites-install.yaml new file mode 100644 index 00000000..0750b68e --- /dev/null +++ b/.tekton/task-pre-requisites-install.yaml @@ -0,0 +1,122 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: pre-requisites-install +spec: + params: + - name: pipeline-debug + description: Pipeline debug mode. Value can be 0 or 1. Default to 0 + default: "0" + workspaces: + - name: workspace + mountPath: /artifacts + stepTemplate: + env: + - name: PIPELINE_DEBUG + value: $(params.pipeline-debug) + steps: + - name: pre-requisites-install + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.29 + workingDir: /artifacts + command: ["/bin/sh", "-c"] + args: + - | + #!/bin/bash + if [[ "${PIPELINE_DEBUG}" == "true" ]]; then + pwd + env + trap env EXIT + set -x + fi + + TF_DIR="/tmp" + + ####################################### + # IBM Cloud + ####################################### + echo $'***** Installing IBMCLOUD and Plugins *****\n' + curl -fsSL https://clis.cloud.ibm.com/install/linux | sh + ibmcloud plugin install schematics -f + ibmcloud plugin install vpc-infrastructure -f + ibmcloud plugin install key-protect -f + echo $'***** Installed IBMCLOUD and Plugins *****\n' + echo $'***** Check Repository Private or Public *****\n' + if [[ "$REPOSITORY" == *"github.ibm.com"* ]]; then + git config --global url.ssh://git@github.ibm.com/.insteadOf https://github.ibm.com/ + export GOPRIVATE=github.ibm.com/* + fi + + ####################################### + # Golang + ####################################### + echo $'***** Installing Golang *****\n' + yum install wget -y + [ ! -d "$(pwd)/go" ] && + cd $(pwd) && wget https://go.dev/dl/go1.21.0.linux-amd64.tar.gz && + tar -C $(pwd)/ -xzf go1.21.0.linux-amd64.tar.gz && + cd $(pwd)/ && + echo "export PATH=\$PATH:$(pwd)/go/bin:\$HOME/go/bin" >> ~/.bashrc && + echo "export GOROOT=$(pwd)/go" >> ~/.bashrc + source ~/.bashrc + go version + which go + echo $'***** Golang Installed Successfully *****\n' + + ####################################### + # python + ####################################### + echo $'***** Installing Python *****\n' + yum install python3.9 -y + python3.9 --version + + if python3 --version &> /dev/null; then + PYTHON=python3 + elif python --version &> /dev/null; then + PYTHON=python3 + else + echo "python or python3 not detected. Please install python, ensure it is on your \$PATH, and retry." + exit 1 + fi + echo $'***** Python Installed *****\n' + + ####################################### + # pip + ####################################### + echo $'***** Upgrading/Installing PIP *****\n' + python3 -m pip install --upgrade pip + pip install --root-user-action=ignore requests + if ! ${PYTHON} -m pip &> /dev/null; then + echo "Unable to detect pip after running: ${PYTHON} -m pip. Please ensure pip is installed and try again." + exit 1 + fi + echo $'***** Upgraded/Installed PIP *****\n' + + ####################################### + # ansible + ####################################### + echo $'***** Installing Ansible and Ansible core *****\n' + python3 -m pip install ansible==4.10.0 ansible-core==2.11.12 + ansible --version + echo $'***** Ansible and Ansible core Installed *****\n' + + ####################################### + # terraform-provisioner + ####################################### + echo $'***** Installing Terraform-Provisioner *****\n' + curl -sL \ + https://raw.githubusercontent.com/radekg/terraform-provisioner-ansible/master/bin/deploy-release.sh \ + --output /tmp/deploy-release.sh + chmod +x /tmp/deploy-release.sh + /tmp/deploy-release.sh -v 2.3.3 + rm -rf /tmp/deploy-release.sh + echo $'***** Terraform-Provisioner Installed *****\n' + + ####################################### + # terraform + ####################################### + echo $'***** Installing Terraform *****\n' + wget https://releases.hashicorp.com/terraform/1.5.7/terraform_1.5.7_linux_amd64.zip + unzip terraform_1.5.7_linux_amd64.zip + mv terraform /usr/local/bin/terraform + terraform --version diff --git a/.tekton/task-set-commit-status.yaml b/.tekton/task-set-commit-status.yaml new file mode 100644 index 00000000..8fe1c6c0 --- /dev/null +++ b/.tekton/task-set-commit-status.yaml @@ -0,0 +1,411 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: git-set-commit-status + description: This task will set CI status on the commit +spec: + params: + - name: ibmcloud-api + description: the ibmcloud api + default: https://cloud.ibm.com + - name: continuous-delivery-context-secret + description: Name of the secret containing the continuous delivery pipeline context secrets + default: secure-properties + - name: ibmcloud-apikey-secret-key + description: field in the secret that contains the api key used to login to ibmcloud + default: apikey + - name: git-access-token + description: | + (optional) token to access the git repository. If this token is provided, there will not be an attempt + to use the git token obtained from the authorization flow when adding the git integration in the toolchain + default: "" + - name: resource-group + description: target resource group (name or id) for the ibmcloud login operation + default: "" + - name: repository + description: | + The git repository url + - name: revision + description: | + (optional) Commit SHA to set the status for. + If left empty, will attempt to read GIT_COMMIT from build-properties + default: "" + - name: description + description: | + A short description of the status. + - name: context + description: | + A string label to differentiate this status from + the status of other systems. ie: "continuous-integration/tekton" + default: "continuous-integration/tekton" + - name: state + description: | + The state of the status. Can be one of the following: `pending`, `running`, `success`, `failed`, `canceled` + or the execution status of pipelineTask: `Succeeded`, `Failed`, and `None` - see https://github.com/tektoncd/pipeline/blob/master/docs/pipelines.md#using-execution-status-of-pipelinetask + or a value meaningful for the target git repository + - github/integrated github: `pending`, `success`, `failure`, `error` + default: "" + - name: state-var + description: | + Customized variable stored in build-properties to use as state if state params is empty. + type: string + default: "" + - name: build-properties + description: | + file containing properties out of clone task (can be a filepath name relative to the workspace/volume) + default: build.properties + - name: target-url + description: | + (optional) a url to set as the status detail link for the PR. + If left empty, the status detail link will point to the pipeline run. + default: "" + - name: fetch-git-information-step-image + description: image to use for the fetch-git-information step (default to icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.29) + default: "icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.29" + - name: set-status-step-image + description: image to use for the fetch-git-information step (default to registry.access.redhat.com/ubi8/ubi:8.1) + default: "registry.access.redhat.com/ubi8/ubi:8.1" + - name: pipeline-debug + description: Pipeline debug mode. Value can be 0 or 1. Default to 0 + default: "0" + workspaces: + - name: artifacts + description: | + Workspace that may contain git repository information (ie build.properties). + Should be marked as optional when Tekton will permit it + mountPath: /artifacts + stepTemplate: + env: + - name: PIPELINE_DEBUG + value: $(params.pipeline-debug) + steps: + - name: fetch-git-information + image: $(params.fetch-git-information-step-image) + env: + - name: REPOSITORY + value: $(params.repository) + - name: API_KEY + valueFrom: + secretKeyRef: + name: $(params.continuous-delivery-context-secret) + key: $(params.ibmcloud-apikey-secret-key) + optional: true + command: ["/bin/bash", "-c"] + args: + - | + set -e -o pipefail + + if [ $PIPELINE_DEBUG == 1 ]; then + pwd + env + trap env EXIT + set -x + fi + + TOOLCHAIN_ID=$(jq -r '.toolchain_guid' /cd-config/toolchain.json) + TOOLCHAIN_REGION=$(jq -r '.region_id' /cd-config/toolchain.json | awk -F: '{print $3}') + ########################################################################## + # Setting HOME explicitly to have ibmcloud plugins available + # doing the export rather than env definition is a workaround + # until https://github.com/tektoncd/pipeline/issues/1836 is fixed + export HOME="/root" + ########################################################################## + if [[ "$REPOSITORY" != *.git ]]; then + echo "Adding .git suffix to Repository URL" + REPOSITORY="${REPOSITORY}.git" + fi + GIT_SERVICE_INSTANCE_ID=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .instance_id' /cd-config/toolchain.json) + if [ -z "$GIT_SERVICE_INSTANCE_ID" ]; then + echo "No Git integration (repository url: $REPOSITORY) found in the toolchain" + exit 1 + fi + GIT_SERVICE_TYPE=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .service_id' /cd-config/toolchain.json) + if [ "$GIT_SERVICE_TYPE" == "github" ]; then + GIT_AUTH_USER="x-oauth-basic" + else + GIT_AUTH_USER="x-token-auth" + fi; + GIT_TOKEN="$(params.git-access-token)" + if [ -z "$GIT_TOKEN" ]; then + AUTHTYPE=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .parameters.auth_type' \ + /cd-config/toolchain.json) + if [[ "${AUTHTYPE}" == "pat" ]]; then + TOKEN=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .parameters.api_token' \ + /cd-config/toolchain.json) + if [[ "${TOKEN}" ]]; then + echo "Using access token from toolchain" + GIT_TOKEN="${TOKEN}" + fi + fi + fi + + if [ -z "$GIT_TOKEN" ]; then + echo "Fetching token for $REPOSITORY" + ibmcloud config --check-version false + ibmcloud login -a $(params.ibmcloud-api) --no-region --apikey $API_KEY + if [ "$(params.resource-group)" ]; then + ibmcloud target -g "$(params.resource-group)" + fi + TOKEN=$(ibmcloud iam oauth-tokens --output JSON | jq -r '.iam_token') + GIT_TOKEN_URL=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .parameters.token_url' \ + /cd-config/toolchain.json) + + # GIT_TOKEN_URL is something like + # https://otc-github-consolidated-broker.us-south.devops.cloud.ibm.com/github/token?git_id=github + # as there is already an url param git_id, just put the additional ones using & + + GIT_BROKER_URL="${GIT_TOKEN_URL}&toolchain_id=${TOOLCHAIN_ID}&service_instance_id=${GIT_SERVICE_INSTANCE_ID}&repo_url=${REPOSITORY}" + echo "Doing cURL to ${GIT_BROKER_URL}" + + curl -s -o /steps/github_token_result.json -X GET -H "Accept: application/json" \ + -H "Authorization: $TOKEN" "$GIT_BROKER_URL" + if jq -e '.access_token' /steps/github_token_result.json > /dev/null 2>&1; then + GIT_TOKEN=$(jq -r '.access_token' /steps/github_token_result.json) + echo "Access token found for the Git integration (repository url: $REPOSITORY)" + else + echo "No access token found for the Git integration (repository url: $REPOSITORY)" + cat /steps/github_token_result.json + exit 1 + fi + else + echo "Using git Access Token provided" + fi + + GIT_API_ROOT_URL=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .parameters.api_root_url' \ + /cd-config/toolchain.json) + GIT_OWNER_ID=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .parameters.owner_id' /cd-config/toolchain.json) + GIT_REPO_NAME=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .parameters.repo_name' /cd-config/toolchain.json) + GIT_ID=$(jq -r --arg git_repo "$REPOSITORY" \ + '.services[] | select (.parameters.repo_url==$git_repo) | .parameters.git_id' /cd-config/toolchain.json) + + TOOLCHAIN_REGION=$(jq -r '.region_id' /cd-config/toolchain.json | awk -F: '{print $3}') + + echo "GIT_REPOSITORY=$REPOSITORY" > /steps/next-step-env.properties + echo "GIT_AUTH_USER=$GIT_AUTH_USER" >> /steps/next-step-env.properties + echo "GIT_TOKEN=$GIT_TOKEN" >> /steps/next-step-env.properties + echo "GIT_SERVICE_TYPE=$GIT_SERVICE_TYPE" >> /steps/next-step-env.properties + echo "GIT_ID=$GIT_ID" >> /steps/next-step-env.properties + echo "GIT_API_ROOT_URL=$GIT_API_ROOT_URL" >> /steps/next-step-env.properties + echo "GIT_OWNER_ID=$GIT_OWNER_ID" >> /steps/next-step-env.properties + echo "GIT_REPO_NAME=$GIT_REPO_NAME" >> /steps/next-step-env.properties + echo "TOOLCHAIN_REGION=$TOOLCHAIN_REGION" >> /steps/next-step-env.properties + + if [ $PIPELINE_DEBUG == 1 ]; then + cat /steps/next-step-env.properties + fi + volumeMounts: + - mountPath: /cd-config + name: cd-config-volume + - mountPath: /steps + name: steps-volume + - name: set-status + image: $(params.set-status-step-image) + env: + - name: PIPELINE_ID + valueFrom: + fieldRef: + fieldPath: metadata.annotations['devops.cloud.ibm.com/pipeline-id'] + - name: PIPELINE_RUN_ID + valueFrom: + fieldRef: + fieldPath: metadata.annotations['devops.cloud.ibm.com/tekton-pipeline'] + script: | + #!/usr/libexec/platform-python + import json + import os + import sys + import urllib.request + import urllib.parse + + # extract the previouly properties found in previous step + previous_step={} + if os.environ["PIPELINE_DEBUG"] == "1": + print("previous step properties:") + f = open("/steps/next-step-env.properties", "r") + for x in f: + if os.environ["PIPELINE_DEBUG"] == "1": + print(x) + prop = x.split("=", 1) + previous_step[prop[0]] = prop[1].strip() + f.close() + + # extract the build properties in their own structure + build={} + if os.path.exists("/artifacts/$(params.build-properties)"): + if os.environ["PIPELINE_DEBUG"] == "1": + print("$(params.build-properties):") + f = open("/artifacts/$(params.build-properties)", "r") + for x in f: + if os.environ["PIPELINE_DEBUG"] == "1": + print(x) + prop = x.split("=", 1) + build[prop[0]] = prop[1].strip() + f.close() + + # find the state + state = "$(params.state)" + if "$(params.state-var)" != "": + print("Looking for state in $(params.build-properties)") + state = build["$(params.state-var)"] + + # If state is one of PipelineRun Taks execution status convert it to a generic state one + if state == "Succeeded": + state = "success" + elif state == "Failed": + state = "failed" + elif state == "None": + state = "pending" + + # Make the state value consistent to the git flavor + # Generic state values are: pending, running, success, failed, canceled + # Define mapping from generic to git flavor to put the appropriate value for git target + # Allowed Github state values: pending, success, failure, error + state_mapping_generic_to_github = { + "failed": "failure", + "canceled": "error", + "running": "pending" + } + # Gitlab: pending, running, success, failed, canceled + # no mapping for gitlab as generic state value are identical + # Allowed Bitbucket state values: SUCCESSFUL, FAILED, INPROGRESS, STOPPED + state_mapping_generic_to_bitbucket = { + "pending": "INPROGRESS", + "running": "INPROGRESS", + "success": "SUCCESSFUL", + "failed": "FAILED", + "canceled": "STOPPED" + } + + # find the commit to set status on + revision = "$(params.revision)" + if revision == "": + revision = build["GIT_COMMIT"] + + description="$(params.description)" + + context="$(params.context)" + + if "$(params.target-url)" == "": + # compute the target url of this pipeline run + target_url = ("https://cloud.ibm.com/devops/pipelines/tekton/" + os.environ["PIPELINE_ID"] + + "/runs/" + os.environ["PIPELINE_RUN_ID"] + "?env_id=ibm:yp:" + previous_step["TOOLCHAIN_REGION"]) + else: + target_url = "$(params.target-url)" + + # Create the request object according to the Git Flavor API + if previous_step["GIT_ID"] == "gitlab" or previous_step["GIT_ID"] == "hostedgit" : + status_url = (previous_step["GIT_API_ROOT_URL"] + + "/v4/projects/" + previous_step["GIT_OWNER_ID"] + "%2F" + previous_step["GIT_REPO_NAME"] + + "/statuses/" + revision) + params = { + "state": state, + "description": description, + "context": context, + "target_url": target_url + } + url_query_params = urllib.parse.urlencode(params) + req = urllib.request.Request( + status_url + "?" + url_query_params, + data=None, + headers={ + "Authorization": "Bearer " + previous_step["GIT_TOKEN"] + }, + method="POST" + ) + elif previous_step["GIT_ID"] == "bitbucketgit": + status_url = (previous_step["GIT_API_ROOT_URL"] + + "/2.0/repositories/" + previous_step["GIT_OWNER_ID"] + "/" + previous_step["GIT_REPO_NAME"] + + "/commit/" + revision + "/statuses/build") + + if state in state_mapping_generic_to_bitbucket: + bitbucket_state = state_mapping_generic_to_bitbucket[state] + else: + # No mapping found - use the state value provided + bitbucket_state = state + print("State value '" + state + "' mapped to bitbucket state '" + bitbucket_state + "'") + + data = { + "key": os.environ["PIPELINE_ID"], + "url": target_url, + "state": bitbucket_state, + "name": context, + "description": description + } + req = urllib.request.Request( + status_url, + data=json.dumps(data).encode('utf8'), + headers={ + "content-type": "application/json", + "Authorization": "Bearer " + previous_step["GIT_TOKEN"] + }, + method="POST" + ) + else: + # Default to github + if previous_step["GIT_ID"] == "integrated": + api_prefix="/v3" + else: + api_prefix="" + + if state in state_mapping_generic_to_github: + github_state = state_mapping_generic_to_github[state] + else: + # No mapping found - use the state value provided + github_state = state + + print("State value '" + state + "' mapped to github state '" + github_state + "'") + + status_url = (previous_step["GIT_API_ROOT_URL"] + api_prefix + + "/repos/" + previous_step["GIT_OWNER_ID"] + "/" + previous_step["GIT_REPO_NAME"] + + "/statuses/" + revision) + data = { + "state": github_state, + "target_url": target_url, + "description": description, + "context": context + } + req = urllib.request.Request( + status_url, + data=json.dumps(data).encode('utf8'), + headers={ + "content-type": "application/json", + "Authorization": "Bearer " + previous_step["GIT_TOKEN"] + }, + method="POST" + ) + + req.add_header("User-Agent", "TektonCD, the peaceful cat") + + with urllib.request.urlopen(req) as resp: + if not str(resp.status).startswith("2"): + print("Error: %d" % (resp.status)) + print(resp.read()) + sys.exit(1) + else: + print(previous_step["GIT_ID"] + " commit status '" + state + "' has been set on " + + previous_step["GIT_REPOSITORY"] + "#" + revision) + if os.environ["PIPELINE_DEBUG"] == "1": + print("Status: %d" % (resp.status)) + print(resp.read()) + sys.exit(0) + volumeMounts: + - mountPath: /steps + name: steps-volume + volumes: + - name: steps-volume + emptyDir: {} + - name: cd-config-volume + configMap: + name: toolchain + items: + - key: toolchain.json + path: toolchain.json diff --git a/.tekton/task-ssh-key-creation.yaml b/.tekton/task-ssh-key-creation.yaml new file mode 100644 index 00000000..a92f85b9 --- /dev/null +++ b/.tekton/task-ssh-key-creation.yaml @@ -0,0 +1,113 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: ssh-key-creation +spec: + params: + - name: ibmcloud-api + description: the ibmcloud api + default: https://cloud.ibm.com + - name: continuous-delivery-context-secret + description: name of the secret containing the continuous delivery pipeline context secrets + default: secure-properties + - name: ibmcloud-apikey-secret-key + description: field in the secret that contains the api key used to login to ibmcloud + default: ibmcloud_api_key + - name: pipeline-debug + description: Pipeline debug mode. Value can be 0 or 1. Default to 0 + default: "0" + - name: resource_group + description: Resource group name from your IBM Cloud account where the VPC resources should be deployed. Note. If the resource group value is set as null, automation creates two different RG with the name (workload-rg and service-rg). For additional information on resource groups, see [Managing resource groups](https://cloud.ibm.com/docs/account?topic=account-rgs). + default: Default + workspaces: + - name: workspace + mountPath: /artifacts + stepTemplate: + env: + - name: API_KEY + valueFrom: + secretKeyRef: + name: $(params.continuous-delivery-context-secret) + key: $(params.ibmcloud-apikey-secret-key) + optional: true + - name: PIPELINE_DEBUG + value: $(params.pipeline-debug) + steps: + - name: ssh-key-creation + onError: continue + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + env: + - name: resource_group + value: $(params.resource_group) + workingDir: "/artifacts" + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + + if [[ "${PIPELINE_DEBUG}" == "true" ]]; then + pwd + env + trap env EXIT + set -x + fi + + ibmcloud version + + mkdir /artifacts/.ssh + ssh_key_pair=$(eval "ssh-keygen -t rsa -N '' -f /artifacts/.ssh/id_rsa <<< y") + + ibmcloud_login () { + local regions=("$1") + local API_KEY=("$2") + local for_create_or_delete="$3" + CICD_SSH_KEY="cicd-toolchain" + # Looping all region to create SSH-KEYS + for region in ${regions[@]}; + do + disable_update_check=$(eval "ibmcloud config --check-version=false") + auhtenticate=$(eval "ibmcloud login --apikey $API_KEY -r $region") + echo "$auhtenticate" + if [[ $auhtenticate = *OK* ]]; then + echo "************SSH-KEY create progress in $region ************" + if echo $for_create_or_delete | grep -q "create"; then + check_key=`ibmcloud is keys | grep "cicd-toolchain" | awk '{print $2}'` + if [[ -z "$check_key" ]]; then + echo "$CICD_SSH_KEY creating in $region" + ssh_key_create=$(eval "ibmcloud is key-create $CICD_SSH_KEY @/artifacts/.ssh/id_rsa.pub --resource-group-name $resource_group") + if [[ $ssh_key_create = *Created* ]]; then + echo "$CICD_SSH_KEY created in $region" + else + echo "ssh-key creation failed in $region" + exit 1 + fi + else + echo "$CICD_SSH_KEY already exists in region $region. So, deleting exist key $CICD_SSH_KEY in region $region" + ssh_key_delete=$(eval "ibmcloud is key-delete $CICD_SSH_KEY -f") + if [[ $ssh_key_delete = *deleted* ]]; then + echo "Exist $CICD_SSH_KEY deleted in $region" + echo "New $CICD_SSH_KEY creating in $region" + ssh_key_create=$(eval "ibmcloud is key-create $CICD_SSH_KEY @/artifacts/.ssh/id_rsa.pub --resource-group-name $resource_group") + if [[ $ssh_key_create = *Created* ]]; then + echo "New $CICD_SSH_KEY created in $region" + else + echo "ssh-key creation failed in $region" + exit 1 + fi + else + echo "ssh-key deletion failed in $region" + fi + fi + echo "************SSH-KEY create progress in $region done ************" + fi + else + echo "Issue Login with IBMCLOUD $auhtenticate" + exit 1 + fi + done + } + + regions="us-south eu-de us-east" + ibmcloud_login "${regions}" "${API_KEY}" "create" diff --git a/.tekton/task-ssh-key-deletion.yaml b/.tekton/task-ssh-key-deletion.yaml new file mode 100644 index 00000000..e51e103c --- /dev/null +++ b/.tekton/task-ssh-key-deletion.yaml @@ -0,0 +1,81 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: ssh-key-deletion +spec: + params: + - name: ibmcloud-api + description: the ibmcloud api + default: https://cloud.ibm.com + - name: continuous-delivery-context-secret + description: name of the secret containing the continuous delivery pipeline context secrets + default: secure-properties + - name: ibmcloud-apikey-secret-key + description: field in the secret that contains the api key used to login to ibmcloud + default: ibmcloud_api_key + - name: pipeline-debug + description: Pipeline debug mode. Value can be 0 or 1. Default to 0 + default: "0" + workspaces: + - name: workspace + mountPath: /artifacts + stepTemplate: + env: + - name: API_KEY + valueFrom: + secretKeyRef: + name: $(params.continuous-delivery-context-secret) + key: $(params.ibmcloud-apikey-secret-key) + optional: true + - name: PIPELINE_DEBUG + value: $(params.pipeline-debug) + steps: + - name: ssh-key-deletion + onError: continue + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + workingDir: "/artifacts" + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + + if [[ "${PIPELINE_DEBUG}" == "true" ]]; then + pwd + env + trap env EXIT + set -x + fi + + ibmcloud_login () { + local regions=("$1") + local API_KEY=("$2") + local for_create_or_delete="$3" + echo "$regions" + CICD_SSH_KEY="cicd-toolchain" + # Looping all region to create SSH-KEYS + for region in ${regions[@]}; + do + echo "$region" + disable_update_check=$(eval "ibmcloud config --check-version=false") + auhtenticate=$(eval "ibmcloud login --apikey $API_KEY -r $region") + echo "$auhtenticate" + if [[ $auhtenticate = *OK* ]]; then + if echo $for_create_or_delete | grep -q "delete"; then + ssh_key_delete=$(eval "ibmcloud is key-delete $CICD_SSH_KEY -f") + if [[ $ssh_key_delete = *deleted* ]]; then + echo "$CICD_SSH_KEY deleted in $region" + else + echo "ssh-key deletion failed in $region" + fi + fi + else + echo "Issue Login with IBMCLOUD $auhtenticate" + exit 1 + fi + done + } + + regions="us-south eu-de us-east" + ibmcloud_login "${regions}" "${API_KEY}" "delete" diff --git a/.tekton/test-infra-ubuntu.yaml b/.tekton/test-infra-ubuntu.yaml new file mode 100644 index 00000000..e60dacc8 --- /dev/null +++ b/.tekton/test-infra-ubuntu.yaml @@ -0,0 +1,360 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: wes-hpc-da-ubuntu +spec: + params: + - name: ibmcloud-api + description: the ibmcloud api + default: https://cloud.ibm.com + - name: continuous-delivery-context-secret + description: name of the secret containing the continuous delivery pipeline context secrets + default: secure-properties + - name: ibmcloud-apikey-secret-key + description: field in the secret that contains the api key used to login to ibmcloud + default: ibmcloud_api_key + - name: pipeline-debug + description: Pipeline debug mode. Value can be 0 or 1. Default to 0 + default: "0" + - name: pr-branch + description: The source branch for the PullRequest + default: "" + - name: directory-name + default: "." + - name: repository + description: the git repo url + - name: ssh_keys + default: "" + description: List of names of the SSH keys that is configured in your IBM Cloud account, used to establish a connection to the IBM Cloud HPC bastion and login node. Ensure that the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by according to [SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys). + - name: zones + default: "" + description: IBM Cloud zone names within the selected region where the IBM Cloud HPC cluster should be deployed. Two zone names are required as input value and supported zones for eu-de are eu-de-2, eu-de-3 and for us-east us-east-1, us-east-3. The management nodes and file storage shares will be deployed to the first zone in the list. Compute nodes will be deployed across both first and second zones, where the first zone in the list will be considered as the most preferred zone for compute nodes deployment. [Learn more](https://cloud.ibm.com/docs/vpc?topic=vpc-creating-a-vpc-in-a-different-region#get-zones-using-the-cli). + - name: cluster_prefix + description: Prefix that is used to name the IBM Cloud HPC cluster and IBM Cloud resources that are provisioned to build the IBM Cloud HPC cluster instance. You cannot create more than one instance of the IBM Cloud HPC cluster with the same name. Ensure that the name is unique. + default: cicd-wes + - name: resource_group + description: Resource group name from your IBM Cloud account where the VPC resources should be deployed. Note. If the resource group value is set as null, automation creates two different RG with the name (workload-rg and service-rg). For additional information on resource groups, see [Managing resource groups](https://cloud.ibm.com/docs/account?topic=account-rgs). + default: Default + - name: remote_allowed_ips + default: "" + description: Comma-separated list of IP addresses that can access the IBM Cloud HPC cluster instance through an SSH interface. For security purposes, provide the public IP addresses assigned to the devices that are authorized to establish SSH connections (for example, [\"169.45.117.34\"]). To fetch the IP address of the device, use [https://ipv4.icanhazip.com/](https://ipv4.icanhazip.com/). + - name: compute_image_name_rhel + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: compute_image_name_ubuntu + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v1). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: login_image_name + description: Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-6 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/hpc-spectrum-LSF#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v2). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. + default: "" + - name: cluster_id + description: Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservation. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment. + default: "" + - name: reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: us_east_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: eu_de_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + - name: us_south_reservation_id + description: Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_). + default: "" + workspaces: + - name: workspace + mountPath: /artifacts + stepTemplate: + env: + - name: API_KEY + valueFrom: + secretKeyRef: + name: $(params.continuous-delivery-context-secret) + key: $(params.ibmcloud-apikey-secret-key) + optional: true + - name: PIPELINE_DEBUG + value: $(params.pipeline-debug) + steps: + - name: ubuntu-suite-1 + onError: continue + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + env: + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: resource_group + value: $(params.resource_group) + - name: compute_image_name_ubuntu + value: $(params.compute_image_name_ubuntu) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + - name: us_east_reservation_id + value: $(params.us_east_reservation_id) + - name: eu_de_reservation_id + value: $(params.eu_de_reservation_id) + - name: us_south_reservation_id + value: $(params.us_south_reservation_id) + workingDir: "/artifacts" + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + + if [[ "${PIPELINE_DEBUG}" == "true" ]]; then + pwd + env + trap env EXIT + set -x + fi + + unzip -n terraform_1.5.7_linux_amd64.zip + mv terraform /usr/local/bin/terraform + terraform --version + + cd $(pwd)/ && + echo "export PATH=\$PATH:$(pwd)/go/bin:\$HOME/go/bin" >> ~/.bashrc && + echo "export GOROOT=$(pwd)/go" >> ~/.bashrc + source ~/.bashrc + go version + + echo "************** Going to run in UBUNTU-Suite-1 ************** + 1.TestRunBasic + 2.TestRunInUsEastRegion + 3.TestRunInUSSouthRegion + 4.TestRunNoKMSAndHTOff" + + export TF_VAR_ibmcloud_api_key=$API_KEY + export SSH_FILE_PATH="/artifacts/.ssh/id_rsa" + CICD_SSH_KEY="cicd-toolchain" + + # Check artifacts/tests folder exists or not + DIRECTORY="/artifacts/tests" + if [ -d "$DIRECTORY" ]; then + cd $DIRECTORY + LOG_FILE=pipeline-UBUNTU-Suite-1-$(date +%d%m%Y).cicd + echo "**************Validating on UBUNTU-Suite-1**************" + SSH_KEY=$CICD_SSH_KEY US_EAST_RESERVATION_ID=$us_east_reservation_id US_SOUTH_RESERVATION_ID=$us_south_reservation_id EU_DE_RESERVATION_ID=$eu_de_reservation_id COMPUTE_IMAGE_NAME=$compute_image_name_ubuntu LOGIN_NODE_IMAGE_NAME=$login_image_name ZONE=$zones RESERVATION_ID=$reservation_id CLUSTER_ID=$cluster_id RESOURCE_GROUP=$resource_group go test -v -timeout 9000m -run "TestRunBasic|TestRunInUsEastRegion|TestRunInUSSouthRegion|TestRunNoKMSAndHTOff" | tee -a $LOG_FILE + else + pwd + ls -a + echo "$DIRECTORY does not exists" + exit 1 + fi + + # Check any error message on the plan/apply log + error_check=$(eval "grep -E -w 'FAIL|Error|ERROR' $DIRECTORY/$LOG_FILE") + if [[ "$error_check" ]]; then + echo "Found Error/FAIL/ERROR in plan/apply log. Please check log." + exit 1 + fi + + echo "*******************************************************" + count=`ls -1 $DIRECTORY/test_output/log* 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated and log file not created, check with packages or binaries installation" + exit 1 + else + cat $DIRECTORY/test_output/log* + fi + echo "*******************************************************" + + count=`ls -1 $DIRECTORY/*.cicd 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated, check with packages or binaries installation" + exit 1 + fi + - name: ubuntu-suite-2 + onError: continue + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + env: + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: resource_group + value: $(params.resource_group) + - name: compute_image_name_ubuntu + value: $(params.compute_image_name_ubuntu) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + - name: us_east_reservation_id + value: $(params.us_east_reservation_id) + - name: eu_de_reservation_id + value: $(params.eu_de_reservation_id) + - name: us_south_reservation_id + value: $(params.us_south_reservation_id) + workingDir: "/artifacts" + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + + if [[ "${PIPELINE_DEBUG}" == "true" ]]; then + pwd + env + trap env EXIT + set -x + fi + + unzip -n terraform_1.5.7_linux_amd64.zip + mv terraform /usr/local/bin/terraform + terraform --version + + cd $(pwd)/ && + echo "export PATH=\$PATH:$(pwd)/go/bin:\$HOME/go/bin" >> ~/.bashrc && + echo "export GOROOT=$(pwd)/go" >> ~/.bashrc + source ~/.bashrc + go version + + echo "************** Going to run in UBUNTU-Suite-2 ************** + 1.TestRunUsingExistingKMS + 2.TestRunLDAPAndPac + 3.TestRunCustomRGAsNull + 4.TestRunCustomRGAsNonDefault" + + export TF_VAR_ibmcloud_api_key=$API_KEY + export SSH_FILE_PATH="/artifacts/.ssh/id_rsa" + CICD_SSH_KEY="cicd-toolchain" + + # Check artifacts/tests folder exists or not + DIRECTORY="/artifacts/tests" + if [ -d "$DIRECTORY" ]; then + cd $DIRECTORY + LOG_FILE=pipeline-UBUNTU-Suite-2-$(date +%d%m%Y).cicd + echo "**************Validating on UBUNTU-Suite-2**************" + SSH_KEY=$CICD_SSH_KEY US_EAST_RESERVATION_ID=$us_east_reservation_id US_SOUTH_RESERVATION_ID=$us_south_reservation_id EU_DE_RESERVATION_ID=$eu_de_reservation_id COMPUTE_IMAGE_NAME=$compute_image_name_ubuntu LOGIN_NODE_IMAGE_NAME=$login_image_name ZONE=$zones RESERVATION_ID=$reservation_id CLUSTER_ID=$cluster_id RESOURCE_GROUP=$resource_group go test -v -timeout 9000m -run "TestRunUsingExistingKMS|TestRunLDAPAndPac|TestRunCustomRGAsNull|TestRunCustomRGAsNonDefault" | tee -a $LOG_FILE + else + pwd + ls -a + echo "$DIRECTORY does not exists" + exit 1 + fi + + # Check any error message on the plan/apply log + error_check=$(eval "grep -E -w 'FAIL|Error|ERROR' $DIRECTORY/$LOG_FILE") + if [[ "$error_check" ]]; then + echo "Found Error/FAIL/ERROR in plan/apply log. Please check log." + exit 1 + fi + + echo "*******************************************************" + count=`ls -1 $DIRECTORY/test_output/log* 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated and log file not created, check with packages or binaries installation" + exit 1 + else + cat $DIRECTORY/test_output/log* + fi + echo "*******************************************************" + + count=`ls -1 $DIRECTORY/*.cicd 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated, check with packages or binaries installation" + exit 1 + fi + - name: ubuntu-suite-3 + onError: continue + image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:latest + env: + - name: ssh_keys + value: $(params.ssh_keys) + - name: zones + value: $(params.zones) + - name: resource_group + value: $(params.resource_group) + - name: compute_image_name_ubuntu + value: $(params.compute_image_name_ubuntu) + - name: login_image_name + value: $(params.login_image_name) + - name: cluster_id + value: $(params.cluster_id) + - name: reservation_id + value: $(params.reservation_id) + - name: us_east_reservation_id + value: $(params.us_east_reservation_id) + - name: eu_de_reservation_id + value: $(params.eu_de_reservation_id) + - name: us_south_reservation_id + value: $(params.us_south_reservation_id) + workingDir: "/artifacts" + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + - | + #!/bin/bash + + if [[ "${PIPELINE_DEBUG}" == "true" ]]; then + pwd + env + trap env EXIT + set -x + fi + + unzip -n terraform_1.5.7_linux_amd64.zip + mv terraform /usr/local/bin/terraform + terraform --version + + cd $(pwd)/ && + echo "export PATH=\$PATH:$(pwd)/go/bin:\$HOME/go/bin" >> ~/.bashrc && + echo "export GOROOT=$(pwd)/go" >> ~/.bashrc + source ~/.bashrc + go version + + echo "************** Going to run in UBUNTU-Suite-3 ************** + 1.TestRunCreateVpc + 1.1 RunHpcExistingVpcSubnetId + 1.2 RunHpcExistingVpcCidr" + + export TF_VAR_ibmcloud_api_key=$API_KEY + export SSH_FILE_PATH="/artifacts/.ssh/id_rsa" + CICD_SSH_KEY="cicd-toolchain" + + # Check artifacts/tests folder exists or not + DIRECTORY="/artifacts/tests" + if [ -d "$DIRECTORY" ]; then + cd $DIRECTORY + LOG_FILE=pipeline-UBUNTU-Suite-3-$(date +%d%m%Y).cicd + echo "**************Validating on UBUNTU-Suite-3**************" + SSH_KEY=$CICD_SSH_KEY US_EAST_RESERVATION_ID=$us_east_reservation_id US_SOUTH_RESERVATION_ID=$us_south_reservation_id EU_DE_RESERVATION_ID=$eu_de_reservation_id COMPUTE_IMAGE_NAME=$compute_image_name_ubuntu LOGIN_NODE_IMAGE_NAME=$login_image_name ZONE=$zones RESERVATION_ID=$reservation_id CLUSTER_ID=$cluster_id RESOURCE_GROUP=$resource_group go test -v -timeout 9000m -run "TestRunCreateVpc" | tee -a $LOG_FILE + else + pwd + ls -a + echo "$DIRECTORY does not exists" + exit 1 + fi + + # Check any error message on the plan/apply log + error_check=$(eval "grep -E -w 'FAIL|Error|ERROR' $DIRECTORY/$LOG_FILE") + if [[ "$error_check" ]]; then + echo "Found Error/FAIL/ERROR in plan/apply log. Please check log." + exit 1 + fi + + echo "*******************************************************" + count=`ls -1 $DIRECTORY/test_output/log* 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated and log file not created, check with packages or binaries installation" + exit 1 + else + cat $DIRECTORY/test_output/log* + fi + echo "*******************************************************" + + count=`ls -1 $DIRECTORY/*.cicd 2>/dev/null | wc -l` + if [ $count == 0 ]; then + echo "Test Suite have not initated, check with packages or binaries installation" + exit 1 + fi diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..2bc6b412 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,40 @@ +# **CHANGELOG** + +## **1.4.1** +- HA variable name change for application centre. +- Cluster Status remote execution completion depends on the cloud init status successful status. + +## **1.4.0** +- Support for Deployable architecture framework code base. +- Support for existing DNS service instance and DNS custom resolvers. +- Default support for Customer-managed encryption through IBM Key Protect for IBM Cloud. +- Support Existing VPC and mandate for having three subnets with existing subnets scenario. +- Support for different SSH keys attribute for login node and cluster nodes for establishing connections. +- Support for creation of New resource groups. + +## **1.3.1** +- Bug Fixes for the support of ldap users to access Aplication centre URL. + +## **1.3.0** +- Support for dedicate LSF login client node to monitor/manage LSF cluster. +- Support for LDAP users to access the LSF cluster nodes and also access to Application centre GUI with LDAP username and password. + +## **1.2.0** +- Support for Boot drive encryption for dynamic worker nodes. +- Support of OpenLDAP integration for user authentication. + +## **1.1.1** +- Support for Application Center through VNC (remote) consoles. + +## **1.1.0** +- Enable LSF Application Center support. +- Support Boot drive encryption for management and storage nodes. +- Support for dynamic compute nodes creation across two availability zones. +- Enable VPC flow flog support. +- Cross zone VPC file share access. +- Support existing subnets of an existing VPC. +- Support for Contract ID and Cluster ID. +- Multi instance profile support for dynamic compute nodes. + +## **1.0.0** +- Initial Release. diff --git a/README.md b/README.md index 5f2e6378..a6dea173 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,199 @@ +# IBM Cloud HPC +Repository for the IBM Cloud HPC project with IBM Spectrum schedulers + +## Deploying the environment using CLI: + +### Initial configuration: + +``` +$ ibmcloud iam api-key-create user-api-key --file ~/.ibm-api-key.json -d "ibmcloud_api_key" +$ cat ~/.ibm-api-key.json | jq -r ."apikey" +# copy your apikey +``` + +### 1. Deployment with Catalog CLI on IBM Cloud + +Use the below IBM Cloud CLI command to create catalog workspace with specific product version. Update the custom configuration and cluster name as per the requirement. +Note: IBM Catalog management plug-in must be pre-installed. [Learn more](https://cloud.ibm.com/docs/cli?topic=cli-manage-catalogs-plugin) + +``` +$ cp sample/configs/hpc_catalog_values.json values.json +$ vim values.json +# Paste your API key and other mandatory parameters value for IBM Cloud HPC cluster +# Login to the IBM Cloud CLI +$ ibmcloud catalog install --vl --override-values values.json + +Note: You can retrieve the by accessing the CLI section within the Deployment options of the IBM Cloud HPC tile. + +It bears resemblance to something along these lines: +$ ibmcloud catalog install --vl 1082e7d2-5e2f-0a11-a3bc-f88a8e1931fc.c7645085-5f49-4d5f-8786-45ac376e60fe-global --override-values values.json +Attempting install of IBM Cloud HPC version x.x.x... +Schematics workspace: https://cloud.ibm.com/schematics/workspaces/us-south.workspace.globalcatalog-collection.40b1c1e4/jobs?region= +Workspace status: DRAFT +Workspace status: INACTIVE +Workspace status: INPROGRESS +Workspace status: ACTIVE +Installation successful +OK +``` + +You can refer the Schematics workspace url (next to Schematics workspace:) as part of the install command output. + +### 2. Deployment with Schematics CLI on IBM Cloud + +**Note**: You also need to generate GitHub token if you use private GitHub repository. + +``` +$ cp sample/configs/hpc_schematics_values.json values.json +$ vim values.json +# Paste your API key and other mandatory parameters value for IBM Cloud HPC cluster +# Login to the IBM Cloud CLI +$ ibmcloud schematics workspace new -f values.json --github-token xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +$ ibmcloud schematics workspace list +Name ID Description Status Frozen +hpcc-cluster us-east.workspace.hpcc-cluster.7cbc3f6b INACTIVE False +OK + +$ ibmcloud schematics plan --id us-east.workspace.hpcc-cluster.7cbc3f6b +Activity ID 51b6330e913d23636d706b084755a737 +OK + +$ ibmcloud schematics apply --id us-east.workspace.hpcc-cluster.7cbc3f6b +Do you really want to perform this action? [y/N]> y +Activity ID b0a909030f071f51d6ceb48b62ee1671 +OK + +$ ibmcloud schematics logs --id us-east.workspace.hpcc-cluster.7cbc3f6b +... + 2023/06/05 22:14:29 Terraform apply | Apply complete! Resources: 41 added, 0 changed, 0 destroyed. + 2023/06/05 22:14:29 Terraform apply | + 2023/06/05 22:14:29 Terraform apply | Outputs: + 2023/06/05 22:14:29 Terraform apply | + 2023/06/05 22:14:29 Terraform apply | image_map_entry_found = "true -- - hpcaas-lsf10-rhel86-v1" + 2023/06/05 22:14:29 Terraform apply | region_name = "us-east" + 2023/06/05 22:14:29 Terraform apply | ssh_command = "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J vpcuser@150.239.215.145 lsfadmin@10.241.0.4" + 2023/06/05 22:14:29 Terraform apply | vpc_name = "dv-hpcaas-vpc -- - r014-e7485f03-6797-4633-b140-2822ce8e1893" + 2023/06/05 22:14:29 Command finished successfully. +OK +``` + +### Accessing the deployed environment: + +* Connect to an LSF login node through SSH by using the `ssh_to_login_node` command from the Schematics log output. +``` +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J vpcuser@ lsfadmin@ +``` +* where `floating_IP_address` is the floating IP address for the bastion node and `login_node_IP_address` is the IP address for the login node. + +### Steps to access the Application Center GUI/Dashboard: + +* Open a new command line terminal. +* Connect to an LSF management node through SSH by using the `application_center_tunnel` command from the Schematics log output. + +``` +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -L 8443:localhost:8443 -L 6080:localhost:6080 -J vpcuser@> lsfadmin@ +``` +* where `floating_IP_address` is the floating IP address for the bastion node and `management_node_IP_address` is the IP address for the management node. + +* Open a browser on the local machine, and run https://localhost:8443 + +* To access the Application Center GUI, enter the password you configured when you created your workspace and the default user as "lsfadmin". + +* If LDAP is enabled, you can access the LSF Application Center using the LDAP username and password that you configured during IBM Cloud® HPC cluster deployment or using an existing LDAP username and password. + +* If IBM Spectrum LSF Application Center GUI is installed in High Availability. The `application_center_tunnel` command is a bit different. Then read also `application_center_url_note` line. +``` +"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -L 8443:pac.:8443 -L 6080:pac.:6080 -J vpcuser@ lsfadmin@" +application_center_url = "https://pac.:8443" +``` + +### Steps to validate the OpenLDAP: +* Connect to your OpenLDAP server through SSH by using the `ssh_to_ldap_node` command from the Schematics log output. + +``` +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -J vpcuser@ ubuntu@ +``` +* where `floating_IP_address` is the floating IP address for the bastion node and `LDAP_server_IP` is the IP address for the OpenLDAP node. + +* Verifiy the LDAP service status: + +``` +systemctl status slapd +``` + +* Verify the LDAP groups and users created: + +``` +ldapsearch -Q -LLL -Y EXTERNAL -H ldapi:/// +``` + +* Submit a Job from HPC cluster Management node with LDAP user : Log into the management node using the `ssh_to_management_node` value as shown as part of output section of Schematics job log: + +``` +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J vpcuser@ lsfadmin@ +``` +* where `floating_IP_address` is the floating IP address for the bastion node and `management_node_IP_address` is the IP address for the management node. + +* Switch to the LDAP user (for example, switch to lsfuser05): + +``` +[lsfadmin@hpccluster-mgmt-1 ~]$ su lsfuser05 +Password: +[lsfuser05@hpccluster-mgmt-1 lsfadmin]# +``` + +* Submit an LSF job as the LDAP user: + +``` +[lsfuser05@hpccluster-mgmt-1 lsfadmin]$ bsub -J myjob[1-4] -R "rusage[mem=2G]" sleep 10 +Job <1> is submitted to default queue . +``` + +### Cleaning up the deployed environment: + +If you no longer need your deployed IBM Cloud HPC cluster, you can clean it up from your environment. The process is threefold: ensure that the cluster is free of running jobs or working compute nodes, destroy all the associated VPC resources and remove them from your IBM Cloud account, and remove the project from the IBM Cloud console. + +**Note**: Ensuring that the cluster is free of running jobs and working compute nodes + +Ensure that it is safe to destroy resources: + +1. As the `lsfadmin` user, close all LSF queues and kill all jobs: + ``` + badmin qclose all + bkill -u all 0 + ``` + +2. Wait ten minutes (this is the default idle time), and then check for running jobs: + ``` + bjobs -u all + ``` + + Look for a `No unfinished job found` message. + + +3. Check that there are no compute nodes (only management nodes should be listed): + ``` + bhosts -w + ``` + +If the cluster has no running jobs or compute nodes, then it is safe to destroy resources from this environment. + +#### Destroying resources + +1. In the IBM Cloud console, from the **Schematics > Workspaces** view, select **Actions > Destroy resources** > **Confirm** the action by entering the workspace name in the text box and click Destroy to delete all the related VPC resources that were deployed. +2. If you select the option to destroy resources, decide whether you want to destroy all of them. This action cannot be undone. +3. Confirm the action by entering the workspace name in the text box and click **Destroy**. +You can now safely remove the resources from your account. + +#### Accessing HPC Tile on Other Regions +1. If there is a requirement to create HPC cluster other than us-east/us-south/eu-de region, HPC cluster can still be provisioned on another regions. +2. Instead of using the IBMCloudHPC provider, the automation uses the IBMCloudgen2 providers to spun up the dynamic nodes. +3. Also instead of using the proxy API URL, a vpc generic API shall be used to spin up the dynamic nodes in the same account as user's. +4. When creating HPC cluster on different different regions, contract id and cluster id basically not needed. So provide a random contract id and cluster_id. + ## Requirements -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0, <1.6.0 | +No requirements. ## Providers @@ -10,9 +201,7 @@ No providers. ## Modules -| Name | Source | Version | -|------|--------|---------| -| [hpc](#module\_hpc) | ./solutions/hpc | n/a | +No modules. ## Resources @@ -20,57 +209,8 @@ No resources. ## Inputs -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allowed\_cidr](#input\_allowed\_cidr) | Network CIDR to access the VPC. This is used to manage network ACL rules for accessing the cluster. | `list(string)` | 
[
  "10.0.0.0/8"
]
| no | -| [bastion\_ssh\_keys](#input\_bastion\_ssh\_keys) | The key pair to use to access the bastion host. | `list(string)` | n/a | yes | -| [bastion\_subnets\_cidr](#input\_bastion\_subnets\_cidr) | Subnet CIDR block to launch the bastion host. | `list(string)` | 
[
  "10.0.0.0/24"
]
| no | -| [boot\_volume\_encryption\_enabled](#input\_boot\_volume\_encryption\_enabled) | Set to true when key management is set | `bool` | `true` | no | -| [bootstrap\_instance\_profile](#input\_bootstrap\_instance\_profile) | Bootstrap should be only used for better deployment performance | `string` | `"mx2-4x32"` | no | -| [compute\_image\_name](#input\_compute\_image\_name) | Image name to use for provisioning the compute cluster instances. | `string` | `"ibm-redhat-8-6-minimal-amd64-5"` | no | -| [compute\_ssh\_keys](#input\_compute\_ssh\_keys) | The key pair to use to launch the compute host. | `list(string)` | n/a | yes | -| [compute\_subnets\_cidr](#input\_compute\_subnets\_cidr) | Subnet CIDR block to launch the compute cluster host. | `list(string)` | 
[
  "10.10.20.0/24",
  "10.20.20.0/24",
  "10.30.20.0/24"
]
| no | -| [cos\_instance\_name](#input\_cos\_instance\_name) | Exiting COS instance name | `string` | `null` | no | -| [dns\_custom\_resolver\_id](#input\_dns\_custom\_resolver\_id) | IBM Cloud DNS custom resolver id. | `string` | `null` | no | -| [dns\_domain\_names](#input\_dns\_domain\_names) | IBM Cloud HPC DNS domain names. | 
object({
    compute  = string
    storage  = string
    protocol = string
  })
| 
{
  "compute": "comp.com",
  "protocol": "ces.com",
  "storage": "strg.com"
}
| no | -| [dns\_instance\_id](#input\_dns\_instance\_id) | IBM Cloud HPC DNS service instance id. | `string` | `null` | no | -| [dynamic\_compute\_instances](#input\_dynamic\_compute\_instances) | MaxNumber of instances to be launched for compute cluster. | 
list(
    object({
      profile = string
      count   = number
    })
  )
| 
[
  {
    "count": 250,
    "profile": "cx2-2x4"
  }
]
| no | -| [enable\_atracker](#input\_enable\_atracker) | Enable Activity tracker | `bool` | `true` | no | -| [enable\_bastion](#input\_enable\_bastion) | The solution supports multiple ways to connect to your HPC cluster for example, using bastion node, via VPN or direct connection. If connecting to the HPC cluster via VPN or direct connection, set this value to false. | `bool` | `true` | no | -| [enable\_bootstrap](#input\_enable\_bootstrap) | Bootstrap should be only used for better deployment performance | `bool` | `false` | no | -| [enable\_cos\_integration](#input\_enable\_cos\_integration) | Integrate COS with HPC solution | `bool` | `true` | no | -| [enable\_vpc\_flow\_logs](#input\_enable\_vpc\_flow\_logs) | Enable Activity tracker | `bool` | `true` | no | -| [enable\_vpn](#input\_enable\_vpn) | The solution supports multiple ways to connect to your HPC cluster for example, using bastion node, via VPN or direct connection. If connecting to the HPC cluster via VPN, set this value to true. | `bool` | `false` | no | -| [file\_shares](#input\_file\_shares) | Custom file shares to access shared storage | 
list(
    object({
      mount_path = string,
      size       = number,
      iops       = number
    })
  )
| 
[
  {
    "iops": 1000,
    "mount_path": "/mnt/binaries",
    "size": 100
  },
  {
    "iops": 1000,
    "mount_path": "/mnt/data",
    "size": 100
  }
]
| no | -| [hpcs\_instance\_name](#input\_hpcs\_instance\_name) | Hyper Protect Crypto Service instance | `string` | `null` | no | -| [ibm\_customer\_number](#input\_ibm\_customer\_number) | Comma-separated list of the IBM Customer Number(s) (ICN) that is used for the Bring Your Own License (BYOL) entitlement check. For more information on how to find your ICN, see [What is my IBM Customer Number (ICN)?](https://www.ibm.com/support/pages/what-my-ibm-customer-number-icn). | `string` | `""` | no | -| [ibmcloud\_api\_key](#input\_ibmcloud\_api\_key) | IBM Cloud API Key that will be used for authentication in scripts run in this module. Only required if certain options are required. | `string` | `null` | no | -| [key\_management](#input\_key\_management) | null/key\_protect/hs\_crypto | `string` | `"key_protect"` | no | -| [login\_image\_name](#input\_login\_image\_name) | Image name to use for provisioning the login instances. | `string` | `"ibm-redhat-8-6-minimal-amd64-5"` | no | -| [login\_instances](#input\_login\_instances) | Number of instances to be launched for login. | 
list(
    object({
      profile = string
      count   = number
    })
  )
| 
[
  {
    "count": 1,
    "profile": "cx2-2x4"
  }
]
| no | -| [login\_ssh\_keys](#input\_login\_ssh\_keys) | The key pair to use to launch the login host. | `list(string)` | n/a | yes | -| [management\_image\_name](#input\_management\_image\_name) | Image name to use for provisioning the management cluster instances. | `string` | `"ibm-redhat-8-6-minimal-amd64-5"` | no | -| [management\_instances](#input\_management\_instances) | Number of instances to be launched for management. | 
list(
    object({
      profile = string
      count   = number
    })
  )
| 
[
  {
    "count": 3,
    "profile": "cx2-2x4"
  }
]
| no | -| [network\_cidr](#input\_network\_cidr) | Network CIDR for the VPC. This is used to manage network ACL rules for cluster provisioning. | `string` | `"10.0.0.0/8"` | no | -| [nsd\_details](#input\_nsd\_details) | Storage scale NSD details | 
list(
    object({
      profile  = string
      capacity = optional(number)
      iops     = optional(number)
    })
  )
| 
[
  {
    "iops": 100,
    "profile": "custom",
    "size": 100
  }
]
| no | -| [placement\_strategy](#input\_placement\_strategy) | VPC placement groups to create (null / host\_spread / power\_spread) | `string` | `null` | no | -| [prefix](#input\_prefix) | A unique identifier for resources. Must begin with a letter and end with a letter or number. This prefix will be prepended to any resources provisioned by this template. Prefixes must be 16 or fewer characters. | `string` | n/a | yes | -| [protocol\_instances](#input\_protocol\_instances) | Number of instances to be launched for protocol hosts. | 
list(
    object({
      profile = string
      count   = number
    })
  )
| 
[
  {
    "count": 2,
    "profile": "bx2-2x8"
  }
]
| no | -| [protocol\_subnets\_cidr](#input\_protocol\_subnets\_cidr) | Subnet CIDR block to launch the storage cluster host. | `list(string)` | 
[
  "10.10.40.0/24",
  "10.20.40.0/24",
  "10.30.40.0/24"
]
| no | -| [resource\_group](#input\_resource\_group) | String describing resource groups to create or reference | `string` | `null` | no | -| [static\_compute\_instances](#input\_static\_compute\_instances) | Min Number of instances to be launched for compute cluster. | 
list(
    object({
      profile = string
      count   = number
    })
  )
| 
[
  {
    "count": 0,
    "profile": "cx2-2x4"
  }
]
| no | -| [storage\_image\_name](#input\_storage\_image\_name) | Image name to use for provisioning the storage cluster instances. | `string` | `"ibm-redhat-8-6-minimal-amd64-5"` | no | -| [storage\_instances](#input\_storage\_instances) | Number of instances to be launched for storage cluster. | 
list(
    object({
      profile = string
      count   = number
    })
  )
| 
[
  {
    "count": 3,
    "profile": "bx2-2x8"
  }
]
| no | -| [storage\_ssh\_keys](#input\_storage\_ssh\_keys) | The key pair to use to launch the storage cluster host. | `list(string)` | n/a | yes | -| [storage\_subnets\_cidr](#input\_storage\_subnets\_cidr) | Subnet CIDR block to launch the storage cluster host. | `list(string)` | 
[
  "10.10.30.0/24",
  "10.20.30.0/24",
  "10.30.30.0/24"
]
| no | -| [vpc](#input\_vpc) | Name of an existing VPC in which the cluster resources will be deployed. If no value is given, then a new VPC will be provisioned for the cluster. [Learn more](https://cloud.ibm.com/docs/vpc) | `string` | `null` | no | -| [vpn\_peer\_address](#input\_vpn\_peer\_address) | The peer public IP address to which the VPN will be connected. | `string` | `null` | no | -| [vpn\_peer\_cidr](#input\_vpn\_peer\_cidr) | The peer CIDRs (e.g., 192.168.0.0/24) to which the VPN will be connected. | `list(string)` | `null` | no | -| [vpn\_preshared\_key](#input\_vpn\_preshared\_key) | The pre-shared key for the VPN. | `string` | `null` | no | -| [zones](#input\_zones) | Region where VPC will be created. To find your VPC region, use `ibmcloud is regions` command to find available regions. | `list(string)` | n/a | yes | +No inputs. ## Outputs -| Name | Description | -|------|-------------| -| [ssh\_command](#output\_ssh\_command) | SSH command to connect to HPC cluster | +No outputs. diff --git a/common-dev-assets b/common-dev-assets index 0a4ea52a..7930ea5c 160000 --- a/common-dev-assets +++ b/common-dev-assets @@ -1 +1 @@ -Subproject commit 0a4ea52a38f1c23f6476d5f57c6813b44cf42f4f +Subproject commit 7930ea5ccc853a0fbab8b29cdaae590ec66b5366 diff --git a/cra-config.yaml b/cra-config.yaml index b4336009..17a96642 100644 --- a/cra-config.yaml +++ b/cra-config.yaml @@ -1,6 +1,3 @@ # More info about this file at https://github.com/terraform-ibm-modules/common-pipeline-assets/blob/main/.github/workflows/terraform-test-pipeline.md#cra-config-yaml version: "v1" -CRA_TARGETS: - - CRA_TARGET: "examples/basic" - CRA_IGNORE_RULES_FILE: "cra-tf-validate-ignore-rules.json" - PROFILE_ID: "0e6e7b5a-817d-4344-ab6f-e5d7a9c49520" # SCC profile ID (currently set to the FSCloud 1.4.0 profile). +CRA_TARGETS: [] diff --git a/examples/basic/README.md b/examples/basic/README.md deleted file mode 100644 index 86eab8eb..00000000 --- a/examples/basic/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Basic example - - - -An end-to-end basic example that will provision the following: -- A new resource group if one is not passed in. -- A new Cloud Object Storage instance. diff --git a/examples/basic/main.tf b/examples/basic/main.tf deleted file mode 100644 index 7ef6663b..00000000 --- a/examples/basic/main.tf +++ /dev/null @@ -1,16 +0,0 @@ -locals { - # Region and Zone calculations - region = join("-", slice(split("-", var.zones[0]), 0, 2)) -} - -module "hpc_basic_example" { - source = "../.." - ibmcloud_api_key = var.ibmcloud_api_key - prefix = var.prefix - zones = var.zones - resource_group = var.resource_group - bastion_ssh_keys = var.ssh_keys - login_ssh_keys = var.ssh_keys - compute_ssh_keys = var.ssh_keys - storage_ssh_keys = var.ssh_keys -} diff --git a/examples/basic/outputs.tf b/examples/basic/outputs.tf deleted file mode 100644 index 95c1a888..00000000 --- a/examples/basic/outputs.tf +++ /dev/null @@ -1,8 +0,0 @@ -# Future use -/* -output "hpc_basic_example_output" { - value = module.hpc_basic_example - sensitive = true - description = "SSH command to connect to HPC cluster" -} -*/ diff --git a/examples/complete/README.md b/examples/complete/README.md deleted file mode 100644 index 139f8ddf..00000000 --- a/examples/complete/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Complete example - - - diff --git a/examples/complete/main.tf b/examples/complete/main.tf deleted file mode 100644 index 558c2107..00000000 --- a/examples/complete/main.tf +++ /dev/null @@ -1,3 +0,0 @@ -############################################################################## -# Complete example -############################################################################## diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf deleted file mode 100644 index addadea6..00000000 --- a/examples/complete/outputs.tf +++ /dev/null @@ -1,23 +0,0 @@ -############################################################################## -# Outputs -############################################################################## - -output "region" { - description = "The region all resources were provisioned in" - value = var.region -} - -output "prefix" { - description = "The prefix used to name all provisioned resources" - value = var.prefix -} - -output "resource_group_name" { - description = "The name of the resource group used" - value = var.resource_group -} - -output "resource_tags" { - description = "List of resource tags" - value = var.resource_tags -} diff --git a/examples/complete/provider.tf b/examples/complete/provider.tf deleted file mode 100644 index 2080946b..00000000 --- a/examples/complete/provider.tf +++ /dev/null @@ -1,8 +0,0 @@ -############################################################################## -# Provider config -############################################################################## - -provider "ibm" { - ibmcloud_api_key = var.ibmcloud_api_key - region = var.region -} diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf deleted file mode 100644 index 170a5abc..00000000 --- a/examples/complete/variables.tf +++ /dev/null @@ -1,29 +0,0 @@ -variable "ibmcloud_api_key" { - type = string - description = "The IBM Cloud API Key" - sensitive = true -} - -variable "region" { - type = string - description = "Region to provision all resources created by this example" - default = "us-south" -} - -variable "prefix" { - type = string - description = "Prefix to append to all resources created by this example" - default = "complete" -} - -variable "resource_group" { - type = string - description = "An existing resource group name to use for this example, if unset a new resource group will be created" - default = null -} - -variable "resource_tags" { - type = list(string) - description = "Optional list of tags to be added to created resources" - default = [] -} diff --git a/examples/complete/version.tf b/examples/complete/version.tf deleted file mode 100644 index d70a9d2d..00000000 --- a/examples/complete/version.tf +++ /dev/null @@ -1,12 +0,0 @@ -terraform { - required_version = ">= 1.3.0, <1.7.0" - - # Ensure that there is always 1 example locked into the lowest provider version of the range defined in the main - # module's version.tf (usually a basic example), and 1 example that will always use the latest provider version. - required_providers { - ibm = { - source = "IBM-Cloud/ibm" - version = ">= 1.49.0, < 2.0.0" - } - } -} diff --git a/examples/create_vpc/modules/landing_zone_vpc/datasource.tf b/examples/create_vpc/modules/landing_zone_vpc/datasource.tf new file mode 100644 index 00000000..130d351e --- /dev/null +++ b/examples/create_vpc/modules/landing_zone_vpc/datasource.tf @@ -0,0 +1,3 @@ +data "ibm_resource_group" "itself" { + name = var.resource_group +} diff --git a/examples/create_vpc/modules/landing_zone_vpc/locals.tf b/examples/create_vpc/modules/landing_zone_vpc/locals.tf new file mode 100644 index 00000000..a896e598 --- /dev/null +++ b/examples/create_vpc/modules/landing_zone_vpc/locals.tf @@ -0,0 +1,136 @@ +locals { + # Defined values + name = "hpc" + prefix = var.prefix + tags = [local.prefix, local.name] + schematics_reserved_cidrs = [ + "169.44.0.0/14", + "169.60.0.0/14", + "158.175.0.0/16", + "158.176.0.0/15", + "141.125.0.0/16", + "161.156.0.0/16", + "149.81.0.0/16", + "159.122.111.224/27", + "150.238.230.128/27", + "169.55.82.128/27" + ] + + bastion_sg_variable_cidr = flatten([ + local.schematics_reserved_cidrs, + var.allowed_cidr + # var.network_cidr + ]) + resource_group_id = var.resource_group != null ? data.ibm_resource_group.itself.id : "" + + # Region and Zone calculations + region = join("-", slice(split("-", var.zones[0]), 0, 2)) + zones = ["zone-1", "zone-2", "zone-3"] + active_zones = [ + for zone in var.zones : + format("zone-%d", substr(zone, -1, -2)) + ] +} + +locals { + # Subnet calculation + active_subnets = { + for zone in local.zones : zone => contains(local.active_zones, zone) ? [ + { + name = "compute-subnet-${zone}" + acl_name = "vpc-acl" + cidr = var.compute_subnets_cidr[index(local.active_zones, zone)] + public_gateway = true + }, + zone == local.active_zones[0] ? { + name = "bastion-subnet" + acl_name = "vpc-acl" + cidr = var.bastion_subnets_cidr[0] + public_gateway = false + } : null + ] : [] + } + subnets = { for zone, subnets in local.active_subnets : zone => [for each in subnets : each if each != null] } + + # Use public gateway calculation + use_public_gateways = { + for zone in local.zones : zone => contains(local.active_zones, zone) ? true : false + } +} + +locals { + # Address_Prefix calculation + + bastion_sg_variable_cidr_list = split(",", var.network_cidr) + address_prefixes = { + "zone-${element(split("-", var.zones[0]), 2)}" = [local.bastion_sg_variable_cidr_list[0]] + } + + # Security group rules + bastion_security_group_rules = flatten([ + [for cidr in local.bastion_sg_variable_cidr : { + name = format("allow-variable-inbound-%s", index(local.bastion_sg_variable_cidr, cidr) + 1) + direction = "inbound" + remote = cidr + # ssh port + tcp = { + port_min = 22 + port_max = 22 + } + }], + [for cidr in local.bastion_sg_variable_cidr : { + name = format("allow-variable-outbound-%s", index(local.bastion_sg_variable_cidr, cidr) + 1) + direction = "outbound" + remote = cidr + }], + [for cidr in local.bastion_sg_variable_cidr_list : { + name = format("allow-variable-inbound-cidr-%s", index(local.bastion_sg_variable_cidr_list, cidr) + 1) + direction = "inbound" + remote = cidr + tcp = { + port_min = 22 + port_max = 22 + } + }], + [for cidr in local.bastion_sg_variable_cidr_list : { + name = format("allow-variable-outbound-cidr-%s", index(local.bastion_sg_variable_cidr_list, cidr) + 1) + direction = "outbound" + remote = cidr + }] + ]) +} + +locals { + # # VPC calculation + # # If user defined then use existing else create new + # # Calculate network acl rules (can be done inplace in vpcs) + network_acl_inbound_rules = [ + { + name = "test-1" + action = "allow" + destination = "0.0.0.0/0" + direction = "inbound" + source = "0.0.0.0/0" + } + ] + network_acl_outbound_rules = [ + { + name = "test-2" + action = "allow" + destination = "0.0.0.0/0" + direction = "outbound" + source = "0.0.0.0/0" + } + ] + network_acl_rules = flatten([local.network_acl_inbound_rules, local.network_acl_outbound_rules]) + + network_acls = [ + { + name = "vpc-acl" + add_ibm_cloud_internal_rules = true + add_vpc_connectivity_rules = true + prepend_ibm_rules = true + rules = local.network_acl_rules + } + ] +} diff --git a/examples/create_vpc/modules/landing_zone_vpc/main.tf b/examples/create_vpc/modules/landing_zone_vpc/main.tf new file mode 100644 index 00000000..19be6920 --- /dev/null +++ b/examples/create_vpc/modules/landing_zone_vpc/main.tf @@ -0,0 +1,15 @@ +module "create_vpc" { + source = "terraform-ibm-modules/landing-zone-vpc/ibm" + version = "7.18.1" + prefix = local.prefix + region = local.region + tags = local.tags + resource_group_id = local.resource_group_id + name = local.name + use_public_gateways = local.use_public_gateways + subnets = local.subnets + address_prefixes = local.address_prefixes + security_group_rules = local.bastion_security_group_rules + network_acls = local.network_acls + enable_hub = var.enable_hub +} diff --git a/examples/create_vpc/modules/landing_zone_vpc/outputs.tf b/examples/create_vpc/modules/landing_zone_vpc/outputs.tf new file mode 100644 index 00000000..8d6bb30e --- /dev/null +++ b/examples/create_vpc/modules/landing_zone_vpc/outputs.tf @@ -0,0 +1,73 @@ +############################################################################## +# VPC GUID +############################################################################## + +output "vpc_name" { + description = "VPC name" + value = module.create_vpc[*].vpc_name +} + +output "vpc_id" { + description = "VPC ID" + value = module.create_vpc[*].vpc_id +} + +output "vpc_crn" { + description = "VPC CRN" + value = module.create_vpc[*].vpc_crn +} + +############################################################################## +output "cidr_blocks" { + description = "List of CIDR blocks present in VPC stack" + value = module.create_vpc[*].cidr_blocks +} + +############################################################################## +# Hub and Spoke specific configuration +############################################################################## + +output "custom_resolver_hub" { + description = "The custom resolver created for the hub vpc. Only set if enable_hub is set and skip_custom_resolver_hub_creation is false." + value = module.create_vpc[*].custom_resolver_hub +} + +output "dns_endpoint_gateways_by_id" { + description = "The list of VPEs that are made available for DNS resolution in the created VPC. Only set if enable_hub is false and enable_hub_vpc_id are true." + value = module.create_vpc[*].dns_endpoint_gateways_by_id +} + +output "dns_endpoint_gateways_by_crn" { + description = "The list of VPEs that are made available for DNS resolution in the created VPC. Only set if enable_hub is false and enable_hub_vpc_id are true." + value = module.create_vpc[*].dns_endpoint_gateways_by_crn +} + +############################################################################## +# Public Gateways +############################################################################## + +output "public_gateways" { + description = "Map of public gateways by zone" + value = module.create_vpc[*].public_gateways +} + +############################################################################## + +############################################################################## +# Subnet Outputs +############################################################################## + +output "subnet_ids" { + description = "The IDs of the subnets" + value = module.create_vpc[*].subnet_ids +} + +output "subnet_detail_list" { + description = "The IDs of the subnets" + value = module.create_vpc[*].subnet_detail_list +} + +output "dummy_ssh_key" { + description = "The ssh key name" + value = var.ssh_keys +} diff --git a/examples/basic/variables.tf b/examples/create_vpc/modules/landing_zone_vpc/variables.tf similarity index 63% rename from examples/basic/variables.tf rename to examples/create_vpc/modules/landing_zone_vpc/variables.tf index 9d7c8a76..207f24a5 100644 --- a/examples/basic/variables.tf +++ b/examples/create_vpc/modules/landing_zone_vpc/variables.tf @@ -6,6 +6,7 @@ variable "ibmcloud_api_key" { description = "IBM Cloud API Key that will be used for authentication in scripts run in this module. Only required if certain options are required." type = string sensitive = true + default = null } ############################################################################## @@ -15,8 +16,7 @@ variable "ibmcloud_api_key" { variable "resource_group" { description = "String describing resource groups to create or reference" type = string - # TODO: Temp fix - default = "geretain-hpc-rg" + default = null } ############################################################################## @@ -26,7 +26,7 @@ variable "resource_group" { variable "prefix" { description = "A unique identifier for resources. Must begin with a letter and end with a letter or number. This prefix will be prepended to any resources provisioned by this template. Prefixes must be 16 or fewer characters." type = string - default = "tim-hpc" + validation { error_message = "Prefix must begin and end with a letter and contain only letters, numbers, and - characters." condition = can(regex("^([A-z]|[a-z][-a-z0-9]*[a-z0-9])$", var.prefix)) @@ -36,16 +36,43 @@ variable "prefix" { variable "zones" { description = "Region where VPC will be created. To find your VPC region, use `ibmcloud is regions` command to find available regions." type = list(string) - # TODO: Temp fix - default = ["ca-tor-1"] +} + +variable "ssh_keys" { + type = list(string) + description = "The key pair to use to access the servers." +} + +variable "bastion_subnets_cidr" { + type = list(string) + default = ["10.0.0.0/24"] + description = "Subnet CIDR block to launch the bastion host." +} + +variable "compute_subnets_cidr" { + type = list(string) + default = ["10.10.20.0/24", "10.20.20.0/24", "10.30.20.0/24"] + description = "Subnet CIDR block to launch the compute cluster host." } ############################################################################## -# Access Variables +# VPC Hub-Spoke support ############################################################################## -variable "ssh_keys" { + +variable "enable_hub" { + description = "Indicates whether this VPC is enabled as a DNS name resolution hub." + type = bool + default = false +} + +variable "allowed_cidr" { + description = "Network CIDR to access the VPC. This is used to manage network ACL rules for accessing the cluster." type = list(string) - description = "The key pair to use to access the bastion host." - # TODO: Temp fix - default = ["geretain-hpc-ssh-key"] + default = ["10.0.0.0/8"] +} + +variable "network_cidr" { + description = "Network CIDR for the VPC. This is used to manage network ACL rules for cluster provisioning." + type = string + default = "10.0.0.0/8" } diff --git a/examples/basic/version.tf b/examples/create_vpc/modules/landing_zone_vpc/version.tf similarity index 88% rename from examples/basic/version.tf rename to examples/create_vpc/modules/landing_zone_vpc/version.tf index 0c47ca34..b1be33fc 100644 --- a/examples/basic/version.tf +++ b/examples/create_vpc/modules/landing_zone_vpc/version.tf @@ -3,7 +3,7 @@ terraform { required_providers { ibm = { source = "IBM-Cloud/ibm" - version = ">= 1.56.2" + version = "1.65.1" } } } diff --git a/examples/create_vpc/solutions/hpc/locals.tf b/examples/create_vpc/solutions/hpc/locals.tf new file mode 100644 index 00000000..a52008ab --- /dev/null +++ b/examples/create_vpc/solutions/hpc/locals.tf @@ -0,0 +1,5 @@ +# locals needed for create_vpc +locals { + # Region and Zone calculations + region = join("-", slice(split("-", var.zones[0]), 0, 2)) +} diff --git a/examples/create_vpc/solutions/hpc/main.tf b/examples/create_vpc/solutions/hpc/main.tf new file mode 100644 index 00000000..8dae8965 --- /dev/null +++ b/examples/create_vpc/solutions/hpc/main.tf @@ -0,0 +1,13 @@ +module "create_vpc" { + source = "../../modules/landing_zone_vpc" + allowed_cidr = var.remote_allowed_ips + ibmcloud_api_key = var.ibmcloud_api_key + ssh_keys = var.bastion_ssh_keys + prefix = var.cluster_prefix + resource_group = var.resource_group + zones = var.zones + network_cidr = var.vpc_cidr + bastion_subnets_cidr = var.vpc_cluster_login_private_subnets_cidr_blocks + compute_subnets_cidr = var.vpc_cluster_private_subnets_cidr_blocks + enable_hub = var.enable_hub +} diff --git a/examples/create_vpc/solutions/hpc/outputs.tf b/examples/create_vpc/solutions/hpc/outputs.tf new file mode 100644 index 00000000..80e50976 --- /dev/null +++ b/examples/create_vpc/solutions/hpc/outputs.tf @@ -0,0 +1,65 @@ +############################################################################## +# VPC GUID +############################################################################## + +output "vpc_name" { + description = "VPC name" + value = module.create_vpc.vpc_name[0] +} + +output "vpc_id" { + description = "VPC ID" + value = module.create_vpc.vpc_id[0] +} + +output "vpc_crn" { + description = "VPC CRN" + value = module.create_vpc.vpc_crn[0] +} + +output "cidr_blocks" { + description = "List of CIDR blocks present in VPC stack" + value = module.create_vpc.cidr_blocks[0] +} + +############################################################################## +# Hub and Spoke specific configuration +############################################################################## + +output "custom_resolver_hub" { + description = "The custom resolver created for the hub vpc. Only set if enable_hub is set and skip_custom_resolver_hub_creation is false." + value = module.create_vpc.custom_resolver_hub +} + +output "dns_endpoint_gateways_by_id" { + description = "The list of VPEs that are made available for DNS resolution in the created VPC. Only set if enable_hub is false and enable_hub_vpc_id are true." + value = module.create_vpc.dns_endpoint_gateways_by_id +} + +output "dns_endpoint_gateways_by_crn" { + description = "The list of VPEs that are made available for DNS resolution in the created VPC. Only set if enable_hub is false and enable_hub_vpc_id are true." + value = module.create_vpc.dns_endpoint_gateways_by_crn +} + +############################################################################## +# Public Gateways +############################################################################## + +output "public_gateways" { + description = "Map of public gateways by zone" + value = module.create_vpc.public_gateways +} + +############################################################################## +# Subnet Outputs +############################################################################## + +output "subnet_ids" { + description = "The IDs of the subnets" + value = module.create_vpc.subnet_ids[0] +} + +output "subnet_detail_list" { + description = "The IDs of the subnets" + value = module.create_vpc.subnet_detail_list[0] +} diff --git a/examples/create_vpc/solutions/hpc/variables.tf b/examples/create_vpc/solutions/hpc/variables.tf new file mode 100644 index 00000000..ed625bb1 --- /dev/null +++ b/examples/create_vpc/solutions/hpc/variables.tf @@ -0,0 +1,126 @@ +############################################################################## +# Account Variables +############################################################################## + +variable "ibmcloud_api_key" { + description = "IBM Cloud API key for the IBM Cloud account where the IBM Cloud HPC cluster needs to be deployed. For more information on how to create an API key, see [Managing user API keys](https://cloud.ibm.com/docs/account?topic=account-userapikey)." + type = string + sensitive = true + validation { + condition = var.ibmcloud_api_key != "" + error_message = "The API key for IBM Cloud must be set." + } +} + +############################################################################## +# Resource Groups Variables +############################################################################## + +variable "resource_group" { + description = "Resource group name from your IBM Cloud account where the VPC resources should be deployed. Note. If the resource group value is set as null, automation creates two different RG with the name (workload-rg and service-rg). For additional information on resource groups, see [Managing resource groups](https://cloud.ibm.com/docs/account?topic=account-rgs)." + type = string + default = "Default" + validation { + condition = var.resource_group != null + error_message = "If you want to provide null for resource_group variable, it should be within double quotes." + } +} + +############################################################################## +# Module Level Variables +############################################################################## + +variable "cluster_prefix" { + description = "Prefix that is used to name the IBM Cloud HPC cluster and IBM Cloud resources that are provisioned to build the IBM Cloud HPC cluster instance. You cannot create more than one instance of the IBM Cloud HPC cluster with the same name. Ensure that the name is unique. Prefix must begin and end with a letter and contain only letters, numbers, and - characters." + type = string + default = "hpcaas" + + validation { + error_message = "Prefix must begin and end with a letter and contain only letters, numbers, and - characters." + condition = can(regex("^[a-zA-Z][-a-zA-Z0-9]*[a-zA-Z]$", var.cluster_prefix)) + } +} + +variable "zones" { + description = "IBM Cloud zone name within the selected region where the IBM Cloud HPC cluster should be deployed. Single zone name is required as input value and supported zones for eu-de are eu-de-2, eu-de-3 for us-east us-east-1, us-east-3 and for us-south us-south-1 and us-south-3. The management nodes, file storage shares and compute nodes will be deployed on the same zone.[Learn more](https://cloud.ibm.com/docs/vpc?topic=vpc-creating-a-vpc-in-a-different-region#get-zones-using-the-cli)." + type = list(string) + default = ["us-east-1"] + validation { + condition = length(var.zones) == 1 + error_message = "Provide list of zones to deploy the cluster." + } +} + +############################################################################## +# VPC Variables +############################################################################## + +variable "vpc_cidr" { + description = "Creates the address prefix for the new VPC, when the vpc_name variable is empty. The VPC requires an address prefix for creation of subnet in a single zone. The subnet are created with the specified CIDR blocks. For more information, see [Setting IP ranges](https://cloud.ibm.com/docs/vpc?topic=vpc-vpc-addressing-plan-design)." + type = string + default = "10.241.0.0/18" +} + +variable "vpc_cluster_private_subnets_cidr_blocks" { + type = list(string) + default = ["10.241.0.0/20"] + description = "The CIDR block that's required for the creation of the compute cluster private subnet. Modify the CIDR block if it conflicts with any on-premises CIDR blocks when using a hybrid environment. Make sure to select a CIDR block size that will accommodate the maximum number of management and dynamic compute nodes that you expect to have in your cluster. Requires one CIDR block. For more information on CIDR block size selection, see [Choosing IP ranges for your VPC](https://cloud.ibm.com/docs/vpc?topic=vpc-choosing-ip-ranges-for-your-vpc)." + validation { + condition = length(var.vpc_cluster_private_subnets_cidr_blocks) == 1 + error_message = "Single zone is supported to deploy resources. Provide a CIDR range of subnets creation." + } +} + +variable "vpc_cluster_login_private_subnets_cidr_blocks" { + type = list(string) + default = ["10.241.16.0/28"] + description = "The CIDR block that's required for the creation of the login cluster private subnet. Modify the CIDR block if it conflicts with any on-premises CIDR blocks when using a hybrid environment. Provide only one CIDR block for the creation of the login subnet. Since login subnet is used only for the creation of login virtual server instances, provide a CIDR range of /28." + validation { + condition = length(var.vpc_cluster_login_private_subnets_cidr_blocks) <= 1 + error_message = "Only a single zone is supported to deploy resources. Provide a CIDR range of subnet creation." + } + validation { + condition = tonumber(regex("/(\\d+)", join(",", var.vpc_cluster_login_private_subnets_cidr_blocks))[0]) <= 28 + error_message = "This subnet is used to create only a login virtual server instance. Providing a larger CIDR size will waste the usage of available IPs. A CIDR range of /28 is sufficient for the creation of the login subnet." + } +} + +############################################################################## +# Access Variables +############################################################################## + +variable "remote_allowed_ips" { + type = list(string) + description = "Comma-separated list of IP addresses that can access the IBM Cloud HPC cluster instance through an SSH interface. For security purposes, provide the public IP addresses assigned to the devices that are authorized to establish SSH connections (for example, [\"169.45.117.34\"]). To fetch the IP address of the device, use [https://ipv4.icanhazip.com/](https://ipv4.icanhazip.com/)." + validation { + condition = alltrue([ + for o in var.remote_allowed_ips : !contains(["0.0.0.0/0", "0.0.0.0"], o) + ]) + error_message = "For security, provide the public IP addresses assigned to the devices authorized to establish SSH connections. Use https://ipv4.icanhazip.com/ to fetch the ip address of the device." + } + validation { + condition = alltrue([ + for a in var.remote_allowed_ips : can(regex("^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(/(3[0-2]|2[0-9]|1[0-9]|[0-9]))?$", a)) + ]) + error_message = "The provided IP address format is not valid. Check if the IP address contains a comma instead of a dot, and ensure there are double quotation marks between each IP address range if using multiple IP ranges. For multiple IP address, use the format [\"169.45.117.34\",\"128.122.144.145\"]." + } +} + +############################################################################## +# Compute Variables +############################################################################## + +variable "bastion_ssh_keys" { + type = list(string) + description = "List of names of the SSH keys that is configured in your IBM Cloud account, used to establish a connection to the IBM Cloud HPC bastion and login node. Ensure that the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by according to [SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys)." +} + +############################################################################## +# VPC Hub-Spoke support +############################################################################## + +variable "enable_hub" { + description = "Indicates whether this VPC is enabled as a DNS name resolution hub." + type = bool + default = false +} diff --git a/examples/create_vpc/solutions/hpc/version.tf b/examples/create_vpc/solutions/hpc/version.tf new file mode 100644 index 00000000..b1be33fc --- /dev/null +++ b/examples/create_vpc/solutions/hpc/version.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.3, < 1.7" + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + version = "1.65.1" + } + } +} + +provider "ibm" { + ibmcloud_api_key = var.ibmcloud_api_key + region = local.region +} diff --git a/hpcaas-arch-1.5.0.svg b/hpcaas-arch-1.5.0.svg new file mode 100644 index 00000000..59fd6930 --- /dev/null +++ b/hpcaas-arch-1.5.0.svg @@ -0,0 +1,4 @@ + + + +
IBM Cloud
IBM Cloud
Public
Network
Public...
Internet
Int...
User
Use...
Region
Region
VPC
HPC
VPC...
Availability Zone
Availability Zone
Subnet
Login
Subnet...
Floating IP
Flo...
Public Gateway
Pub...
Subnet
HPC
Subnet...
Login SG
Login SG
HPC SG
HPC SG



HPC LSF Management Nodes - v10.1.014 

HPC LSF Management Nodes...
File Storage
Fil...
LDAP Server
LDA...
Bastion Node
Bas...
Login Node
Log...
IBM Storage Scale
(Optional)
IBM...
VPN Gateway (optional)
VPN...
DNS Service
DNS...

IBM Cloud HPC
VPC Endpoint

IBM Cloud HPC...
SSH
SSH
Virtual Server
Dynamic Compute Nodes
Virtual Server...





Text is not SVG - cannot display \ No newline at end of file diff --git a/ibm_catalog.json b/ibm_catalog.json index 3256b283..32a31baf 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -1,8 +1,8 @@ { "products": [ { - "name": "terraform-ibm-modules-terraform-ibm-hpc-ad6e71e", - "label": "HPC Deployable Architecture", + "name": "deploy-arch-ibm-hpc", + "label": "IBM Cloud HPC", "product_kind": "solution", "tags": [ "Deployable Architecture", @@ -43,8 +43,8 @@ ], "flavors": [ { - "label": "Advanced", - "name": "advanced", + "label": "Cluster-with-LSF-v10.1.0.14", + "name": "Cluster-with-LSF-v10.1.0.14", "install_type": "fullstack", "working_directory": "solutions/hpc", "compliance": { @@ -52,24 +52,261 @@ "profiles": [ { "profile_name": "IBM Cloud Framework for Financial Services", - "profile_version": "1.5.0" + "profile_version": "1.6.0" } ] }, "release_notes_url": "https://cloud.ibm.com/docs/allowlist/hpc-service?topic=hpc-service-release-notes", "configuration": [ { - "key": "ibmcloud_api_key", - "type": "password", - "description": "IBM Cloud API Key that will be used for authentication in scripts run in this module. Only required if certain options are required.", - "required": true + "key": "ibmcloud_api_key" }, { - "key": "nsd_details", - "type": "array", - "default_value": null, - "description": "Storage scale NSD details", - "hidden": true + "custom_config": { + "type": "resource_group", + "grouping": "deployment", + "original_grouping": "deployment", + "config_constraints": { + "identifier": "rg_name" + } + }, + "required": true, + "key": "resource_group" + }, + { + "key": "reservation_id" + }, + { + "key": "cluster_id" + }, + { + "key": "bastion_ssh_keys" + }, + { + "key": "compute_ssh_keys" + }, + { + "key": "remote_allowed_ips" + }, + { + "key": "zones", + "required": true, + "default_value": ["us-east-1"], + "options": [ + { + "displayname": "Washington DC 1", + "value": ["us-east-1"] + }, + { + "displayname": "Washington DC 3", + "value": ["us-east-3"] + }, + { + "displayname": "Frankfurt 2", + "value": ["eu-de-2"] + }, + { + "displayname": "Frankfurt 3", + "value": ["eu-de-3"] + }, + { + "displayname": "Dallas 1", + "value": ["us-south-1"] + } + ] + }, + { + "key": "cluster_prefix" + }, + { + "key": "vpc_cidr" + }, + { + "key": "vpc_cluster_private_subnets_cidr_blocks" + }, + { + "key": "vpc_cluster_login_private_subnets_cidr_blocks" + }, + { + "key": "vpc_name" + }, + { + "key": "cluster_subnet_ids" + }, + { + "key": "login_subnet_id" + }, + { + "key": "login_node_instance_type" + }, + { + "key": "management_node_instance_type" + }, + { + "key": "management_node_count" + }, + { + "key": "management_image_name" + }, + { + "key": "compute_image_name" + }, + { + "key": "login_image_name" + }, + { + "key": "custom_file_shares" + }, + { + "key": "storage_security_group_id" + }, + { + "key": "dns_instance_id" + }, + { + "key": "dns_domain_name" + }, + { + "key": "dns_custom_resolver_id" + }, + { + "key": "enable_cos_integration" + }, + { + "key": "cos_instance_name" + }, + { + "key": "observability_atracker_on_cos_enable" + }, + { + "key": "observability_monitoring_enable" + }, + { + "key": "observability_monitoring_on_compute_nodes_enable" + }, + { + "key": "observability_monitoring_plan", + "default_value": "graduated-tier", + "options": [ + { + "displayname": "graduated-tier", + "value": "graduated-tier" + }, + { + "displayname": "lite", + "value": "lite" + } + ] + }, + { + "key": "enable_vpc_flow_logs" + }, + { + "key": "vpn_enabled" + }, + { + "key": "key_management" + }, + { + "key": "kms_instance_name" + }, + { + "key": "kms_key_name" + }, + { + "key": "scc_enable" + }, + { + "key": "scc_profile" + }, + { + "key": "scc_profile_version" + }, + { + "key": "scc_location" + }, + { + "key": "scc_event_notification_plan", + "default_value": "lite", + "options": [ + { + "displayname": "lite", + "value": "lite" + }, + { + "displayname": "standard", + "value": "standard" + } + ] + }, + { + "key": "hyperthreading_enabled" + }, + { + "key": "enable_fip" + }, + { + "key": "enable_app_center" + }, + { + "key": "app_center_gui_pwd" + }, + { + "key": "app_center_high_availability" + }, + { + "key": "enable_ldap" + }, + { + "key": "ldap_basedns" + }, + { + "key": "ldap_server" + }, + { + "key": "ldap_admin_password" + }, + { + "key": "ldap_user_name" + }, + { + "key": "ldap_user_password" + }, + { + "key": "ldap_vsi_profile" + }, + { + "key": "ldap_vsi_osimage_name" + }, + { + "key": "skip_iam_authorization_policy" + }, + { + "key": "existing_certificate_instance" + }, + { + "key": "bastion_instance_name" + }, + { + "key": "bastion_instance_public_ip" + }, + { + "key": "bastion_security_group_id" + }, + { + "key": "bastion_ssh_private_key" + }, + { + "hidden": true, + "key": "TF_VERSION" + }, + { + "hidden": true, + "key": "TF_PARALLELISM" + }, + { + "hidden": true, + "key": "TF_VALIDATION_SCRIPT_FILES" } ], "iam_permissions": [ @@ -77,11 +314,11 @@ "role_crns": [ "crn:v1:bluemix:public:iam::::serviceRole:Manager" ], - "service_name": "appid" + "service_name": "schematics" }, { "role_crns": [ - "crn:v1:bluemix:public:iam::::serviceRole:Manager" + "crn:v1:bluemix:public:iam::::serviceRole:writer" ], "service_name": "cloud-object-storage" }, @@ -89,25 +326,31 @@ "role_crns": [ "crn:v1:bluemix:public:iam::::serviceRole:Manager" ], - "service_name": "hs-crypto" + "service_name": "dns-svcs" }, { "role_crns": [ - "crn:v1:bluemix:public:iam::::role:Administrator" + "crn:v1:bluemix:public:iam::::serviceRole:Manager" ], - "service_name": "iam-identity" + "service_name": "kms" }, { "role_crns": [ - "crn:v1:bluemix:public:iam::::serviceRole:Manager" + "crn:v1:bluemix:public:iam::::role:Administrator" ], - "service_name": "kms" + "service_name": "project" }, { "role_crns": [ - "crn:v1:bluemix:public:iam::::role:Administrator" + "crn:v1:bluemix:public:iam::::role:Editor" ], "service_name": "is.vpc" + }, + { + "role_crns": [ + "crn:v1:bluemix:public:iam::::role:Editor" + ], + "service_name": "dns-svcs" } ], "architecture": { @@ -149,11 +392,11 @@ "diagrams": [ { "diagram": { - "caption": "HPC variation", - "url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-landing-zone/main/reference-architectures/vsi-vsi.drawio.svg", + "caption": "IBM Cloud HPC", + "url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-hpc/main/hpcaas-arch-1.5.0.svg", "type": "image/svg+xml" }, - "description": "The HPC variation of the deployable architecture is based on the IBM Cloud for Financial Services reference architecture. The architecture creates a customizable and secure infrastructure, with virtual servers, to run your workloads with a Virtual Private Cloud (VPC) in multizone regions." + "description": "This deployable architecture creates a VPC to run your HPC workload within a single zone from IBM Cloud. A login node is deployed in a separate subnet and security group to access your HPC environment. The HPC management nodes are in a different subnet and security group. In addition, clusters of virtual server instances are provisioned for high availability and are pre-installed with the IBM Spectrum LSF scheduler for HPC workload job management. The IBM Spectrum LSF scheduler dynamically creates compute nodes and deletes them after job completion. Also, IBM Cloud File Storage for VPC is provisioned for configuration or data sharing between HPC management and compute nodes." } ] } diff --git a/main.tf b/main.tf deleted file mode 100644 index 5bf68ad6..00000000 --- a/main.tf +++ /dev/null @@ -1,55 +0,0 @@ -module "hpc" { - source = "./solutions/hpc" - allowed_cidr = var.allowed_cidr - bootstrap_instance_profile = var.bootstrap_instance_profile - bastion_ssh_keys = var.bastion_ssh_keys - bastion_subnets_cidr = var.bastion_subnets_cidr - #compute_gui_password = var.compute_gui_password - #compute_gui_username = var.compute_gui_username - compute_image_name = var.compute_image_name - compute_ssh_keys = var.compute_ssh_keys - compute_subnets_cidr = var.compute_subnets_cidr - cos_instance_name = var.cos_instance_name - dns_custom_resolver_id = var.dns_custom_resolver_id - dns_instance_id = var.dns_instance_id - dns_domain_names = var.dns_domain_names - dynamic_compute_instances = var.dynamic_compute_instances - enable_atracker = var.enable_atracker - enable_bastion = var.enable_bastion - enable_bootstrap = var.enable_bootstrap - enable_cos_integration = var.enable_cos_integration - enable_vpc_flow_logs = var.enable_vpc_flow_logs - enable_vpn = var.enable_vpn - file_shares = var.file_shares - hpcs_instance_name = var.hpcs_instance_name - # ibm_customer_number = var.ibm_customer_number - ibmcloud_api_key = var.ibmcloud_api_key - key_management = var.key_management - login_image_name = var.login_image_name - login_instances = var.login_instances - login_ssh_keys = var.login_ssh_keys - # login_subnets_cidr = var.login_subnets_cidr - management_image_name = var.management_image_name - management_instances = var.management_instances - network_cidr = var.network_cidr - nsd_details = var.nsd_details - placement_strategy = var.placement_strategy - prefix = var.prefix - protocol_instances = var.protocol_instances - protocol_subnets_cidr = var.protocol_subnets_cidr - resource_group = var.resource_group - #scheduler = var.scheduler - static_compute_instances = var.static_compute_instances - #storage_gui_password = var.storage_gui_password - #storage_gui_username = var.storage_gui_username - storage_image_name = var.storage_image_name - storage_instances = var.storage_instances - storage_ssh_keys = var.storage_ssh_keys - storage_subnets_cidr = var.storage_subnets_cidr - #storage_type = var.storage_type - vpc = var.vpc - vpn_peer_address = var.vpn_peer_address - vpn_peer_cidr = var.vpn_peer_cidr - vpn_preshared_key = var.vpn_preshared_key - zones = var.zones -} diff --git a/modules/alb/locals.tf b/modules/alb/locals.tf new file mode 100644 index 00000000..3088b553 --- /dev/null +++ b/modules/alb/locals.tf @@ -0,0 +1,3 @@ +locals { + pool_ids = { for idx, pool in ibm_is_lb_pool.alb_backend_pools : pool.name => pool.id } +} diff --git a/modules/alb/main.tf b/modules/alb/main.tf new file mode 100644 index 00000000..d230dc31 --- /dev/null +++ b/modules/alb/main.tf @@ -0,0 +1,57 @@ +resource "ibm_is_lb" "alb" { + count = var.create_load_balancer ? 1 : 0 + name = format("%s-alb", var.prefix) + resource_group = var.resource_group_id + type = var.alb_type + security_groups = var.security_group_ids + subnets = [var.bastion_subnets[0].id] +} + +resource "ibm_is_lb_pool" "alb_backend_pools" { + count = var.create_load_balancer ? length(var.alb_pools) : 0 + name = format(var.alb_pools[count.index]["name"], var.prefix) + lb = ibm_is_lb.alb[0].id + algorithm = var.alb_pools[count.index]["algorithm"] + protocol = var.alb_pools[count.index]["protocol"] + health_delay = var.alb_pools[count.index]["health_delay"] + health_retries = var.alb_pools[count.index]["health_retries"] + health_timeout = var.alb_pools[count.index]["health_timeout"] + health_type = var.alb_pools[count.index]["health_type"] + health_monitor_url = var.alb_pools[count.index]["health_monitor_url"] + health_monitor_port = var.alb_pools[count.index]["health_monitor_port"] + session_persistence_type = var.alb_pools[count.index]["session_persistence_type"] +} + +resource "ibm_is_lb_listener" "alb_frontend_listener" { + count = var.create_load_balancer ? length(var.alb_pools) : 0 + lb = ibm_is_lb.alb[0].id + port = var.alb_pools[count.index]["lb_pool_listener"]["port"] + protocol = var.alb_pools[count.index]["lb_pool_listener"]["protocol"] + idle_connection_timeout = var.alb_pools[count.index]["lb_pool_listener"]["idle_connection_timeout"] + certificate_instance = var.certificate_instance + default_pool = lookup(local.pool_ids, format(var.alb_pools[count.index]["name"], var.prefix), null) +} + +resource "ibm_is_lb_pool_member" "alb_candidate_members_8443" { + count = var.create_load_balancer ? length(var.vsi_ids) : 0 + lb = ibm_is_lb.alb[0].id + pool = element(split("/", lookup(local.pool_ids, format(var.alb_pools[0]["name"], var.prefix), null)), 1) + port = var.alb_pools[0]["lb_pool_members_port"] + target_id = var.vsi_ids[count.index]["id"] +} + +resource "ibm_is_lb_pool_member" "alb_candidate_members_8444" { + count = var.create_load_balancer ? 1 : 0 + lb = ibm_is_lb.alb[0].id + pool = element(split("/", lookup(local.pool_ids, format(var.alb_pools[1]["name"], var.prefix), null)), 1) + port = var.alb_pools[1]["lb_pool_members_port"] + target_id = var.vsi_ids[0]["id"] +} + +resource "ibm_is_lb_pool_member" "alb_candidate_members_6080" { + count = var.create_load_balancer ? length(var.vsi_ids) : 0 + lb = ibm_is_lb.alb[0].id + pool = element(split("/", lookup(local.pool_ids, format(var.alb_pools[2]["name"], var.prefix), null)), 1) + port = var.alb_pools[2]["lb_pool_members_port"] + target_id = var.vsi_ids[count.index]["id"] +} diff --git a/modules/alb/outputs.tf b/modules/alb/outputs.tf new file mode 100644 index 00000000..aa22195f --- /dev/null +++ b/modules/alb/outputs.tf @@ -0,0 +1,5 @@ + +output "alb_hostname" { + description = "ALB hostname" + value = var.create_load_balancer ? ibm_is_lb.alb[0].hostname : "" +} diff --git a/modules/alb/variables.tf b/modules/alb/variables.tf new file mode 100644 index 00000000..b225d06d --- /dev/null +++ b/modules/alb/variables.tf @@ -0,0 +1,136 @@ + +variable "resource_group_id" { + description = "String describing resource groups to create or reference" + type = string + default = null +} + +variable "prefix" { + description = "A unique identifier for resources. Must begin with a letter and end with a letter or number. This prefix will be prepended to any resources provisioned by this template. Prefixes must be 16 or fewer characters." + type = string + + validation { + error_message = "Prefix must begin and end with a letter and contain only letters, numbers, and - characters." + condition = can(regex("^([A-z]|[a-z][-a-z0-9]*[a-z0-9])$", var.prefix)) + } +} + +variable "certificate_instance" { + description = "Certificate instance CRN value. It's the CRN value of a certificate stored in the Secret Manager" + type = string + default = "" +} + +variable "security_group_ids" { + type = list(string) + description = "List of Security group IDs to allow File share access" + default = null +} + +variable "bastion_subnets" { + type = list(object({ + name = string + id = string + zone = string + cidr = string + })) + default = [] + description = "Subnets to launch the bastion host." +} + +variable "create_load_balancer" { + description = "True to create new Load Balancer." + type = bool +} + +variable "vsi_ids" { + type = list( + object({ + id = string, + }) + ) + description = "VSI data" +} + +variable "alb_type" { + description = "ALB type" + type = string + default = "private" +} + +variable "alb_pools" { + description = "List of Load Balancer Pools" + type = list(object({ + name = string + algorithm = string + protocol = string + health_delay = number + health_retries = number + health_timeout = number + health_type = string + health_monitor_url = string + health_monitor_port = number + session_persistence_type = string + lb_pool_members_port = number + lb_pool_listener = object({ + port = number + protocol = string + idle_connection_timeout = number + }) + })) + default = [ + { + name = "%s-alb-pool-8443" + algorithm = "round_robin" + protocol = "https" + health_delay = 5 + health_retries = 5 + health_timeout = 2 + health_type = "https" + health_monitor_url = "/platform/" + health_monitor_port = 8443 + session_persistence_type = "http_cookie" + lb_pool_members_port = 8443 + lb_pool_listener = { + port = 8443 + protocol = "https" + idle_connection_timeout = 50 + } + }, + { + name = "%s-alb-pool-8444" + algorithm = "round_robin" + protocol = "https" + health_delay = 5 + health_retries = 5 + health_timeout = 2 + health_type = "https" + health_monitor_url = "/" + health_monitor_port = 8444 + session_persistence_type = "http_cookie" + lb_pool_members_port = 8444 + lb_pool_listener = { + port = 8444 + protocol = "https" + idle_connection_timeout = 7200 + } + }, + { + name = "%s-alb-pool-6080" + algorithm = "round_robin" + protocol = "https" + health_delay = 5 + health_retries = 5 + health_timeout = 2 + health_type = "https" + health_monitor_url = "/" + health_monitor_port = 6080 + session_persistence_type = "http_cookie" + lb_pool_members_port = 6080 + lb_pool_listener = { + port = 6080 + protocol = "https" + idle_connection_timeout = 50 + } + }] +} diff --git a/modules/alb/version.tf b/modules/alb/version.tf new file mode 100644 index 00000000..2fb790db --- /dev/null +++ b/modules/alb/version.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.3, < 1.7" + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + version = ">= 1.56.2" + } + } +} diff --git a/modules/alb_api/.gitignore b/modules/alb_api/.gitignore new file mode 100644 index 00000000..06eab7ae --- /dev/null +++ b/modules/alb_api/.gitignore @@ -0,0 +1 @@ +debug_*.txt diff --git a/modules/alb_api/locals.tf b/modules/alb_api/locals.tf new file mode 100644 index 00000000..54a975c2 --- /dev/null +++ b/modules/alb_api/locals.tf @@ -0,0 +1,2 @@ +locals { +} diff --git a/modules/alb_api/main.tf b/modules/alb_api/main.tf new file mode 100644 index 00000000..b4041bd2 --- /dev/null +++ b/modules/alb_api/main.tf @@ -0,0 +1,36 @@ +provider "shell" { + environment = { + } + interpreter = ["/bin/bash", "-c"] + enable_parallelism = false +} + +resource "shell_script" "alb_api" { + count = var.create_load_balancer ? 1 : 0 + lifecycle_commands { + create = "scripts/alb-create.sh" + # read = "scripts/alb-read.sh" + # update = "scripts/alb-update.sh" + delete = "scripts/alb-delete.sh" + } + working_directory = path.module + # interpreter = ["/bin/bash", "-c"] + sensitive_environment = { + ibmcloud_api_key = var.ibmcloud_api_key + } + environment = { + region = var.region + resource_group_id = var.resource_group_id + prefix = var.prefix + bastion_subnet_id = var.bastion_subnets[0].id + certificate_instance = var.certificate_instance + firstip = var.vsi_ips[0] + pool_ips = join(",", var.vsi_ips[*]) + security_group_ids = join(",", var.security_group_ids[*]) + } + triggers = { + # We actually always do delete/create, since "update" is not implemented. + # when_value_changed = var.region + # ... + } +} diff --git a/modules/alb_api/outputs.tf b/modules/alb_api/outputs.tf new file mode 100644 index 00000000..0d294c09 --- /dev/null +++ b/modules/alb_api/outputs.tf @@ -0,0 +1,4 @@ +output "alb_hostname" { + description = "ALB hostname" + value = var.create_load_balancer ? shell_script.alb_api[0].output["hostname"] : "" +} diff --git a/modules/alb_api/scripts/alb-create.sh b/modules/alb_api/scripts/alb-create.sh new file mode 100755 index 00000000..fe9a946b --- /dev/null +++ b/modules/alb_api/scripts/alb-create.sh @@ -0,0 +1,261 @@ +#!/bin/bash +# shellcheck disable=all + +# inputs we assume to get: +# - bastion_subnet_id +# - certificate_instance +# - pool_ips (comma separated) +# - prefix +# - resource_group_id +# - security_group_ids (comma separated) + +debug=true # "true" or "false" + +exec 111>&1 >&2 # use fd 111 later to emit json, other output goes to stderr + +$debug && echo "CREATE $(date +%Y%m%dT%H%M%S.%N)" >>debug_shell_log.txt + +$debug && sort -z /proc/self/environ|tr \\0 \\n >debug_shell_create_env.txt + +# json input from stdin; nothing really expected, input values come from env variables when creating +in="$(cat)" +$debug && echo >debug_shell_create_in.txt "in=<<<$in>>>" + + +# Going to build the complex json request for the ALB creation. +# Many pieces have to be customized, duplicated and finally merged together. + +# pieces +jreq="$(cat <>>$jreq<<<" + +# rough sanity check +if [ "${#jreq}" -lt 100 ]; then + echo "failed to create the JSON request" + exit 1 +fi + +# Step 1. Get a IAM token. + +out="$(curl -X POST 'https://iam.cloud.ibm.com/identity/token' \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=${ibmcloud_api_key}")" +$debug && echo "$out" +iam_token="$(jq -r '.access_token' <<<"$out")" +$debug && echo "$iam_token" + +# rough sanity check +if [ "${#iam_token}" -lt 100 ]; then + echo "failed to get a IAM token" + exit 1 +fi + +# Step 2. Create the LB. + +out="$(curl -X POST "https://${region}.iaas.cloud.ibm.com/v1/load_balancers?version=2024-04-25&generation=2" \ + -H "Authorization: Bearer $iam_token" \ + -H 'Content-Type: application/json' \ + -H 'accept: application/json' \ + -d "$jreq")" +$debug && echo "$out" +lbid="$(jq -r '.id' <<<"$out")" +$debug && echo "$lbid" + +# rough sanity check +if [ "${#lbid}" -lt 10 ]; then + echo "failed to get a LB id" + exit 1 +fi + +# Other interesting outputs can be collected. + +name="$(jq -r '.name' <<<"$out")" +hostname="$(jq -r '.hostname' <<<"$out")" +crn="$(jq -r '.crn' <<<"$out")" +href="$(jq -r '.href' <<<"$out")" + + +# Step 3. Finally wait for the LB to be really running. + +max_wait_seconds=$((20*60)) +start_at="$(date +%s)" +while true; do + now="$(date +%s)" + if [ "$now" -gt "$((start_at+max_wait_seconds))" ]; then + echo "timeout waiting for LB creation" + exit 1 + fi + + out="$(curl -X GET "https://${region}.iaas.cloud.ibm.com/v1/load_balancers/$lbid?version=2024-04-25&generation=2" \ + -H "Authorization: Bearer $iam_token")" + status="$(jq -r '.provisioning_status' <<<"$out")" + error="$(jq -r '.errors[].code' <<<"$out" )" + $debug && echo "$(date -Is) $status" + $debug && echo "$(date -Is) $error" + + if [ "$status" == "active" ]; then + echo "LB successfully created" + break + elif [ "$status" == "create_pending" ]; then + delay=5 + else # this also handles connection problems + delay=4 + fi + + echo "waiting $delay seconds" + sleep $delay +done +# Note possibile status we can get: +# - create_pending +# - active +# - delete_pending +# Or a specific error if LB is not existent (.errors[].code) +# - load_balancer_not_found + +# All done, prepare final output including interesting values to consume. + +res="$(cat <&111 "$res" + +exit 0 diff --git a/modules/alb_api/scripts/alb-delete.sh b/modules/alb_api/scripts/alb-delete.sh new file mode 100755 index 00000000..b6faabc9 --- /dev/null +++ b/modules/alb_api/scripts/alb-delete.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# shellcheck disable=all + +debug=true # "true" or "false" + +exec 111>&1 >&2 # use fd 111 later to emit json, other output goes to stderr + +$debug && echo "DELETE $(date +%Y%m%dT%H%M%S.%N)" >>debug_shell_log.txt + +$debug && sort -z /proc/self/environ|tr \\0 \\n >debug_shell_delete_env.txt + +# json input from stdin; we get the "id" of the LB here +in=$(cat) +$debug && echo >debug_shell_delete_in.txt "in=<<<$in>>>" + +lbid="$(jq -r .id <<<"$in")" + +# rough sanity check +if [ "${#lbid}" -lt 10 ]; then + echo "failed to get a LB id" + exit 1 +fi + +# Step 1. Get a IAM token. + +out="$(curl -X POST 'https://iam.cloud.ibm.com/identity/token' \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=${ibmcloud_api_key}")" +$debug && echo "$out" +iam_token="$(jq -r '.access_token' <<<"$out")" +$debug && echo "$iam_token" + +# rough sanity check +if [ "${#iam_token}" -lt 100 ]; then + echo "failed to get a IAM token" + exit 1 +fi + +# Step 2. Delete the LB. + +out="$(curl -X DELETE "https://${region}.iaas.cloud.ibm.com/v1/load_balancers/$lbid?version=2024-04-25&generation=2" \ + -H "Authorization: Bearer $iam_token")" +$debug && echo "$out" + +# Step 3. Finally wait for the LB to really disappear. + +max_wait_seconds=$((15*60)) +start_at="$(date +%s)" +while true; do + now="$(date +%s)" + if [ "$now" -gt "$((start_at+max_wait_seconds))" ]; then + echo "timeout waiting for LB deletion" + exit 1 + fi + + out="$(curl -X GET "https://${region}.iaas.cloud.ibm.com/v1/load_balancers/$lbid?version=2024-04-25&generation=2" \ + -H "Authorization: Bearer $iam_token")" + status="$(jq -r '.provisioning_status' <<<"$out")" + error="$(jq -r '.errors[].code' <<<"$out" )" + $debug && echo "$(date -Is) $status" + $debug && echo "$(date -Is) $error" + + if [ "$error" == "load_balancer_not_found" ]; then + echo "LB successfully deleted" + break + elif [ "$status" == "delete_pending" ]; then + delay=5 + else # this also handles connection problems + delay=4 + fi + + echo "waiting $delay seconds" + sleep $delay +done +# Note possibile status we can get: +# - create_pending +# - active +# - delete_pending +# Or a specific error if LB is not existent (.errors[].code) +# - load_balancer_not_found + +# All done, no output has to be generated. + +exit 0 diff --git a/modules/alb_api/scripts/alb-read.sh b/modules/alb_api/scripts/alb-read.sh new file mode 100755 index 00000000..e087922a --- /dev/null +++ b/modules/alb_api/scripts/alb-read.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# Optional, implementation of "resource state sync" functionality. +# We do not need this. diff --git a/modules/alb_api/scripts/alb-update.sh b/modules/alb_api/scripts/alb-update.sh new file mode 100755 index 00000000..a348884a --- /dev/null +++ b/modules/alb_api/scripts/alb-update.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# Optional, implementation of "resource change" functionality. +# We do not need this, the delete+create fallback is ok. diff --git a/modules/alb_api/variables.tf b/modules/alb_api/variables.tf new file mode 100644 index 00000000..7c785181 --- /dev/null +++ b/modules/alb_api/variables.tf @@ -0,0 +1,60 @@ +variable "ibmcloud_api_key" { + description = "IBM Cloud API Key that will be used for authentication in scripts run in this module. Only required if certain options are required." + type = string + sensitive = true + default = null +} + +variable "region" { + description = "The region where the ALB must be instantiated" + type = string +} + +variable "resource_group_id" { + description = "String describing resource groups to create or reference" + type = string + default = null +} + +variable "prefix" { + description = "A unique identifier for resources. Must begin with a letter and end with a letter or number. This prefix will be prepended to any resources provisioned by this template. Prefixes must be 16 or fewer characters." + type = string + + validation { + error_message = "Prefix must begin and end with a letter and contain only letters, numbers, and - characters." + condition = can(regex("^([A-z]|[a-z][-a-z0-9]*[a-z0-9])$", var.prefix)) + } +} + +variable "certificate_instance" { + description = "Certificate instance CRN value. It's the CRN value of a certificate stored in the Secret Manager" + type = string + default = "" +} + +variable "security_group_ids" { + type = list(string) + description = "List of Security group IDs to allow File share access" + default = null +} + +variable "bastion_subnets" { + type = list(object({ + name = string + id = string + zone = string + cidr = string + })) + default = [] + description = "Subnets to launch the bastion host." +} + +variable "create_load_balancer" { + description = "True to create new Load Balancer." + type = bool +} + +variable "vsi_ips" { + type = list(string) + description = "VSI IPv4 addresses" +} diff --git a/modules/alb_api/version.tf b/modules/alb_api/version.tf new file mode 100644 index 00000000..82290e46 --- /dev/null +++ b/modules/alb_api/version.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.3, < 1.7" + required_providers { + shell = { + source = "scottwinkler/shell" + version = "1.7.10" + } + } +} diff --git a/modules/bootstrap/datasource.tf b/modules/bootstrap/datasource.tf index c86bf016..5b171679 100644 --- a/modules/bootstrap/datasource.tf +++ b/modules/bootstrap/datasource.tf @@ -1,16 +1,14 @@ -data "ibm_resource_group" "itself" { - name = var.resource_group -} - data "ibm_is_image" "bastion" { name = local.bastion_image_name } -data "ibm_is_image" "bootstrap" { - name = local.bootstrap_image_name -} - data "ibm_is_ssh_key" "bastion" { for_each = toset(var.ssh_keys) name = each.key } + +# Existing Bastion details +data "ibm_is_instance" "bastion_instance_name" { + count = var.bastion_instance_name != null ? 1 : 0 + name = var.bastion_instance_name +} diff --git a/modules/bootstrap/locals.tf b/modules/bootstrap/locals.tf index 948f43bc..a89e43af 100644 --- a/modules/bootstrap/locals.tf +++ b/modules/bootstrap/locals.tf @@ -1,6 +1,5 @@ # define variables locals { - #products = "scale" name = "hpc" prefix = var.prefix tags = [local.prefix, local.name] @@ -17,41 +16,23 @@ locals { "150.238.230.128/27", "169.55.82.128/27" ] - bastion_sg_variable_cidr = var.enable_bootstrap == false ? flatten([ - local.schematics_reserved_cidrs, - var.allowed_cidr, - var.network_cidr - ]) : flatten([var.allowed_cidr, var.network_cidr]) - enable_bastion = var.enable_bastion || var.enable_bootstrap - enable_bootstrap = var.enable_bootstrap + bastion_sg_variable_cidr = flatten([ + local.schematics_reserved_cidrs, + var.allowed_cidr + # var.network_cidr + ]) - bastion_node_name = format("%s-%s", local.prefix, "bastion") - bootstrap_node_name = format("%s-%s", local.prefix, "bootstrap") + bastion_node_name = format("%s-%s", local.prefix, "bastion") bastion_machine_type = "cx2-4x8" bastion_image_name = "ibm-ubuntu-22-04-3-minimal-amd64-1" - bootstrap_image_name = "ibm-redhat-8-6-minimal-amd64-6" - bastion_image_id = data.ibm_is_image.bastion.id - bootstrap_image_id = data.ibm_is_image.bootstrap.id + bastion_image_id = data.ibm_is_image.bastion.id bastion_ssh_keys = [for name in var.ssh_keys : data.ibm_is_ssh_key.bastion[name].id] - /* - # Scale static configs - scale_cloud_deployer_path = "/opt/IBM/ibm-spectrumscale-cloud-deploy" - scale_cloud_install_repo_url = "https://github.com/IBM/ibm-spectrum-scale-cloud-install" - scale_cloud_install_repo_name = "ibm-spectrum-scale-cloud-install" - scale_cloud_install_branch = "5.1.8.1" - scale_cloud_infra_repo_url = "https://github.com/IBM/ibm-spectrum-scale-install-infra" - scale_cloud_infra_repo_name = "ibm-spectrum-scale-install-infra" - scale_cloud_infra_repo_tag = "v2.7.0" - */ - - # Region and Zone calculations - region = join("-", slice(split("-", var.zones[0]), 0, 2)) - + bastion_sg_variable_cidr_list = var.network_cidr # Security group rules # TODO: Fix SG rules bastion_security_group_rules = flatten([ @@ -69,13 +50,48 @@ locals { name = format("allow-variable-outbound-%s", index(local.bastion_sg_variable_cidr, cidr) + 1) direction = "outbound" remote = cidr + }], + [for cidr in local.bastion_sg_variable_cidr_list : { + name = format("allow-variable-inbound-cidr-%s", index(local.bastion_sg_variable_cidr_list, cidr) + 1) + direction = "inbound" + remote = cidr + tcp = { + port_min = 22 + port_max = 22 + } + }], + [for cidr in local.bastion_sg_variable_cidr_list : { + name = format("allow-variable-outbound-cidr-%s", index(local.bastion_sg_variable_cidr_list, cidr) + 1) + direction = "outbound" + remote = cidr }] ]) # Derived configs # VPC - resource_group_id = data.ibm_resource_group.itself.id # Subnets bastion_subnets = var.bastion_subnets + + # Bastion Security group rule update to connect with login node + bastion_security_group_rule_update = [ + { + name = "inbound-rule-for-login-node-connection" + direction = "inbound" + remote = var.bastion_security_group_id + } + ] + + # Bastion Security Group rule update with LDAP server + ldap_security_group_rule = [ + { + name = "inbound-rule-for-ldap-node-connection" + direction = "inbound" + remote = var.ldap_server + tcp = { + port_min = 389 + port_max = 389 + } + } + ] } diff --git a/modules/bootstrap/main.tf b/modules/bootstrap/main.tf index d489903b..7975823b 100644 --- a/modules/bootstrap/main.tf +++ b/modules/bootstrap/main.tf @@ -1,65 +1,67 @@ module "ssh_key" { - count = local.enable_bastion ? 1 : 0 - source = "./../key" - private_key_path = "bastion_id_rsa" #checkov:skip=CKV_SECRET_6 + count = 1 + source = "./../key" + # private_key_path = "bastion_id_rsa" #checkov:skip=CKV_SECRET_6 } module "bastion_sg" { - count = local.enable_bastion ? 1 : 0 + count = 1 source = "terraform-ibm-modules/security-group/ibm" - version = "1.0.1" + version = "2.6.1" add_ibm_cloud_internal_rules = true - resource_group = local.resource_group_id + resource_group = var.resource_group security_group_name = format("%s-bastion-sg", local.prefix) security_group_rules = local.bastion_security_group_rules vpc_id = var.vpc_id + tags = local.tags } +module "bastion_sg_with_ldap_update" { + count = var.ldap_server == "null" ? 0 : 1 + source = "terraform-ibm-modules/security-group/ibm" + version = "2.6.1" + resource_group = var.resource_group + add_ibm_cloud_internal_rules = true + use_existing_security_group_id = true + existing_security_group_id = module.bastion_sg[0].security_group_id + security_group_rules = local.ldap_security_group_rule + vpc_id = var.vpc_id + depends_on = [module.bastion_sg] +} + +module "existing_bastion_sg_update" { + count = var.bastion_security_group_id != null ? 1 : 0 + source = "terraform-ibm-modules/security-group/ibm" + version = "2.6.1" + resource_group = var.resource_group + add_ibm_cloud_internal_rules = true + use_existing_security_group_id = true + existing_security_group_id = var.bastion_security_group_id + security_group_rules = local.bastion_security_group_rule_update + vpc_id = var.vpc_id + depends_on = [module.bastion_sg] +} module "bastion_vsi" { - count = var.enable_bastion ? 1 : 0 + count = var.bastion_instance_name != null ? 0 : 1 source = "terraform-ibm-modules/landing-zone-vsi/ibm" - version = "2.13.0" + version = "4.0.0" vsi_per_subnet = 1 create_security_group = false security_group = null image_id = local.bastion_image_id machine_type = local.bastion_machine_type prefix = local.bastion_node_name - resource_group_id = local.resource_group_id + resource_group_id = var.resource_group enable_floating_ip = true security_group_ids = module.bastion_sg[*].security_group_id ssh_key_ids = local.bastion_ssh_keys - subnets = local.bastion_subnets + subnets = length(var.bastion_subnets) == 2 ? [local.bastion_subnets[1]] : [local.bastion_subnets[0]] tags = local.tags user_data = data.template_file.bastion_user_data.rendered vpc_id = var.vpc_id kms_encryption_enabled = var.kms_encryption_enabled - skip_iam_authorization_policy = false - boot_volume_encryption_key = var.boot_volume_encryption_key - existing_kms_instance_guid = var.existing_kms_instance_guid -} - -module "bootstrap_vsi" { - count = local.enable_bootstrap ? 1 : 0 - source = "terraform-ibm-modules/landing-zone-vsi/ibm" - version = "2.13.0" - vsi_per_subnet = 1 - create_security_group = false - security_group = null - image_id = local.bootstrap_image_id - machine_type = var.bootstrap_instance_profile - prefix = local.bootstrap_node_name - resource_group_id = local.resource_group_id - enable_floating_ip = false - security_group_ids = module.bastion_sg[*].security_group_id - ssh_key_ids = local.bastion_ssh_keys - subnets = local.bastion_subnets - tags = local.tags - user_data = data.template_file.bootstrap_user_data.rendered - vpc_id = var.vpc_id - kms_encryption_enabled = var.kms_encryption_enabled - skip_iam_authorization_policy = var.enable_bastion ? true : false + skip_iam_authorization_policy = var.skip_iam_authorization_policy boot_volume_encryption_key = var.boot_volume_encryption_key existing_kms_instance_guid = var.existing_kms_instance_guid } diff --git a/modules/bootstrap/outputs.tf b/modules/bootstrap/outputs.tf index 73efaab0..dd9c746d 100644 --- a/modules/bootstrap/outputs.tf +++ b/modules/bootstrap/outputs.tf @@ -1,21 +1,21 @@ -/* -output "bastion_vsi_data" { - value = module.bastion_vsi[*] +output "bastion_primary_ip" { + description = "Bastion primary IP" + value = var.bastion_instance_name != null && var.bastion_instance_public_ip != null ? data.ibm_is_instance.bastion_instance_name[0].primary_network_interface[0].primary_ip[0].address : one(module.bastion_vsi[*]["fip_list"][0]["ipv4_address"]) } -output "bootstrap_vsi_data" { - value = module.bootstrap_vsi[*] -} -*/ - output "bastion_fip" { description = "Bastion FIP" - value = one(module.bastion_vsi[*]["fip_list"][0]["floating_ip"]) + value = var.bastion_instance_public_ip != null && var.bastion_instance_name != null ? [var.bastion_instance_public_ip] : module.bastion_vsi[*]["fip_list"][0]["floating_ip"] +} + +output "bastion_fip_id" { + description = "Bastion FIP ID" + value = var.bastion_instance_name != null && var.bastion_instance_public_ip != null ? null : one(module.bastion_vsi[*]["fip_list"][0]["floating_ip_id"]) } output "bastion_security_group_id" { description = "Bastion SG" - value = one(module.bastion_sg[*].security_group_id) + value = var.bastion_security_group_id != null ? var.bastion_security_group_id : one(module.bastion_sg[*].security_group_id) } output "bastion_public_key_content" { @@ -23,3 +23,9 @@ output "bastion_public_key_content" { sensitive = true value = one(module.ssh_key[*].public_key_content) } + +output "bastion_private_key_content" { + description = "Bastion private key content" + sensitive = true + value = one(module.ssh_key[*].private_key_content) +} diff --git a/modules/bootstrap/template_files.tf b/modules/bootstrap/template_files.tf index c556de8d..9cb9ce11 100644 --- a/modules/bootstrap/template_files.tf +++ b/modules/bootstrap/template_files.tf @@ -1,13 +1,6 @@ data "template_file" "bastion_user_data" { template = file("${path.module}/templates/bastion_user_data.tpl") vars = { - ssh_public_key_content = local.enable_bastion ? module.ssh_key[0].public_key_content : "" - } -} - -data "template_file" "bootstrap_user_data" { - template = file("${path.module}/templates/bootstrap_user_data.tpl") - vars = { - bastion_public_key_content = local.enable_bastion ? module.ssh_key[0].public_key_content : "" + ssh_public_key_content = module.ssh_key[0].public_key_content } } diff --git a/modules/bootstrap/templates/bootstrap_user_data.tpl b/modules/bootstrap/templates/bootstrap_user_data.tpl deleted file mode 100644 index a5782268..00000000 --- a/modules/bootstrap/templates/bootstrap_user_data.tpl +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/bash - -################################################### -# Copyright (C) IBM Corp. 2023 All Rights Reserved. -# Licensed under the Apache License v2.0 -################################################### - -#!/usr/bin/env bash -if grep -E -q "CentOS|Red Hat" /etc/os-release -then - USER=vpcuser -elif grep -q "Ubuntu" /etc/os-release -then - USER=ubuntu -fi -sed -i -e "s/^/no-port-forwarding,no-agent-forwarding,no-X11-forwarding,command=\"echo \'Please login as the user \\\\\"$USER\\\\\" rather than the user \\\\\"root\\\\\".\';echo;sleep 5; exit 142\" /" /root/.ssh/authorized_keys - -# input parameters -echo "${bastion_public_key_content}" >> /home/$USER/.ssh/authorized_keys -echo "StrictHostKeyChecking no" >> /home/$USER/.ssh/config - -# setup env -# TODO: Conditional installation (python3, terraform & ansible) -if grep -E -q "CentOS|Red Hat" /etc/os-release -then - # TODO: Terraform Repo access - #yum install -y yum-utils - #yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo - #if (which terraform); then echo "Terraform exists, skipping the installation"; else (yum install -y terraform - if (which python3); then echo "Python3 exists, skipping the installation"; else (yum install -y python38); fi - if (which ansible-playbook); then echo "Ansible exists, skipping the installation"; else (yum install -y ansible); fi -elif grep -q "Ubuntu" /etc/os-release -then - apt update - # TODO: Terraform Repo access - #apt-get update && sudo apt-get install -y gnupg software-properties-common - #wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg - #gpg --no-default-keyring --keyring /usr/share/keyrings/hashicorp-archive-keyring.gpg --fingerprint - apt install software-properties-common - apt-add-repository --yes --update ppa:ansible/ansible - if (which python3); then echo "Python3 exists, skipping the installation"; else (apt install python38); fi - if (which ansible-playbook); then echo "Ansible exists, skipping the installation"; else (apt install ansible); fi -fi - -# TODO: run terraform diff --git a/modules/bootstrap/variables.tf b/modules/bootstrap/variables.tf index b519c68a..a370470e 100644 --- a/modules/bootstrap/variables.tf +++ b/modules/bootstrap/variables.tf @@ -1,14 +1,3 @@ -############################################################################## -# Account Variables -############################################################################## - -variable "ibmcloud_api_key" { - description = "IBM Cloud API Key that will be used for authentication in scripts run in this module. Only required if certain options are required." - type = string - sensitive = true - default = null -} - ############################################################################## # Resource Groups Variables ############################################################################## @@ -33,11 +22,6 @@ variable "prefix" { } } -variable "zones" { - description = "Region where VPC will be created. To find your VPC region, use `ibmcloud is regions` command to find available regions." - type = list(string) -} - ############################################################################## # VPC Variables ############################################################################## @@ -49,17 +33,13 @@ variable "vpc_id" { variable "network_cidr" { description = "Network CIDR for the VPC. This is used to manage network ACL rules for cluster provisioning." - type = string - default = "10.0.0.0/8" + type = list(string) + default = null } + ############################################################################## # Access Variables ############################################################################## -variable "enable_bastion" { - type = bool - default = true - description = "The solution supports multiple ways to connect to your HPC cluster for example, using bastion node, via VPN or direct connection. If connecting to the HPC cluster via VPN or direct connection, set this value to false." -} variable "bastion_subnets" { type = list(object({ @@ -72,18 +52,6 @@ variable "bastion_subnets" { description = "Subnets to launch the bastion host." } -variable "enable_bootstrap" { - type = bool - default = false - description = "Bootstrap should be only used for better deployment performance" -} - -variable "bootstrap_instance_profile" { - type = string - default = "mx2-4x32" - description = "Bootstrap should be only used for better deployment performance" -} - variable "ssh_keys" { type = list(string) description = "The key pair to use to access the host." @@ -113,3 +81,41 @@ variable "existing_kms_instance_guid" { default = null description = "GUID of boot volume encryption key" } + +variable "skip_iam_authorization_policy" { + type = string + default = null + description = "Skip IAM Authorization policy" +} + +########################################################################### +# Existing Bastion Support variables +########################################################################### + +variable "bastion_instance_name" { + type = string + default = null + description = "Bastion instance name." +} + +variable "bastion_instance_public_ip" { + type = string + default = null + description = "Bastion instance public ip address." +} + +variable "bastion_security_group_id" { + type = string + default = null + description = "Bastion security group id." +} + +########################################################################### +# LDAP Server variables +########################################################################### + +variable "ldap_server" { + type = string + default = "null" + description = "Provide the IP address for the existing LDAP server. If no address is given, a new LDAP server will be created." +} diff --git a/modules/bootstrap/version.tf b/modules/bootstrap/version.tf index beb59d86..c917feb9 100644 --- a/modules/bootstrap/version.tf +++ b/modules/bootstrap/version.tf @@ -11,8 +11,3 @@ terraform { } } } - -provider "ibm" { - ibmcloud_api_key = var.ibmcloud_api_key - region = local.region -} diff --git a/modules/custom/subnet_cidr_check/main.tf b/modules/custom/subnet_cidr_check/main.tf new file mode 100644 index 00000000..c69da089 --- /dev/null +++ b/modules/custom/subnet_cidr_check/main.tf @@ -0,0 +1,17 @@ + +locals { + subnet_cidr = [for i in [var.subnet_cidr] : [(((split(".", cidrhost(i, 0))[0]) * pow(256, 3)) #192 + + ((split(".", cidrhost(i, 0))[1]) * pow(256, 2)) + + ((split(".", cidrhost(i, 0))[2]) * pow(256, 1)) + + ((split(".", cidrhost(i, 0))[3]) * pow(256, 0))), (((split(".", cidrhost(i, -1))[0]) * pow(256, 3)) #192 + + ((split(".", cidrhost(i, -1))[1]) * pow(256, 2)) + + ((split(".", cidrhost(i, -1))[2]) * pow(256, 1)) + + ((split(".", cidrhost(i, -1))[3]) * pow(256, 0)))]] + vpc_address_prefix = [for i in var.vpc_address_prefix : [(((split(".", cidrhost(i, 0))[0]) * pow(256, 3)) #192 + + ((split(".", cidrhost(i, 0))[1]) * pow(256, 2)) + + ((split(".", cidrhost(i, 0))[2]) * pow(256, 1)) + + ((split(".", cidrhost(i, 0))[3]) * pow(256, 0))), (((split(".", cidrhost(i, -1))[0]) * pow(256, 3)) + + ((split(".", cidrhost(i, -1))[1]) * pow(256, 2)) + + ((split(".", cidrhost(i, -1))[2]) * pow(256, 1)) + + ((split(".", cidrhost(i, -1))[3]) * pow(256, 0)))]] +} diff --git a/modules/custom/subnet_cidr_check/outputs.tf b/modules/custom/subnet_cidr_check/outputs.tf new file mode 100644 index 00000000..da2c0d02 --- /dev/null +++ b/modules/custom/subnet_cidr_check/outputs.tf @@ -0,0 +1,4 @@ +output "results" { + description = "Result of the calculation" + value = [for ip in local.vpc_address_prefix : ip[0] <= local.subnet_cidr[0][0] && ip[1] >= local.subnet_cidr[0][1]] +} diff --git a/modules/custom/subnet_cidr_check/variables.tf b/modules/custom/subnet_cidr_check/variables.tf new file mode 100644 index 00000000..8249beb2 --- /dev/null +++ b/modules/custom/subnet_cidr_check/variables.tf @@ -0,0 +1,12 @@ +#subnet_cidr is the cidr range of input subnet. +variable "subnet_cidr" { + description = "CIDR range of input subnet." + type = string +} + +#vpc_address_prefix is the cidr range of vpc address prefixes. +variable "vpc_address_prefix" { + description = "CIDR range of VPC address prefixes." + type = list(string) + default = [] +} diff --git a/modules/custom/subnet_cidr_check/version.tf b/modules/custom/subnet_cidr_check/version.tf new file mode 100644 index 00000000..01db83a5 --- /dev/null +++ b/modules/custom/subnet_cidr_check/version.tf @@ -0,0 +1,3 @@ +terraform { + required_version = ">= 1.3, < 1.7" +} diff --git a/modules/database/mysql/main.tf b/modules/database/mysql/main.tf new file mode 100644 index 00000000..42abdaae --- /dev/null +++ b/modules/database/mysql/main.tf @@ -0,0 +1,24 @@ +################################################################################ +# database/mysql/main.tf - Creating a MySQL database +################################################################################ +# Copyright 2023 IBM +# +# Licensed under the MIT License. See the LICENSE file for details. +# +# Maintainer: Salvatore D'Angelo +################################################################################ + +module "db" { + source = "terraform-ibm-modules/icd-mysql/ibm" + version = "1.2.6" + resource_group_id = var.resource_group_id + name = var.name + region = var.region + service_endpoints = var.service_endpoints + mysql_version = var.mysql_version + admin_pass = var.adminpassword + members = var.members + member_memory_mb = var.memory + member_disk_mb = var.disks + member_cpu_count = var.vcpu +} diff --git a/modules/database/mysql/outputs.tf b/modules/database/mysql/outputs.tf new file mode 100644 index 00000000..a23377bf --- /dev/null +++ b/modules/database/mysql/outputs.tf @@ -0,0 +1,11 @@ +output "db_instance_info" { + description = "Database instance information" + value = { + id = module.db.id + adminuser = module.db.adminuser + adminpassword = var.adminpassword + hostname = module.db.hostname + port = module.db.port + certificate = module.db.certificate_base64 + } +} diff --git a/modules/database/mysql/variables.tf b/modules/database/mysql/variables.tf new file mode 100644 index 00000000..7b9120fd --- /dev/null +++ b/modules/database/mysql/variables.tf @@ -0,0 +1,56 @@ +variable "name" { + description = "Name of the Database" + type = string +} + +variable "mysql_version" { + description = "MySQL version of the Database" + type = string + default = "8.0" +} + +variable "region" { + description = "The region where the database must be instantiated" + type = string +} + +variable "adminpassword" { + description = "The administrator password" + type = string +} + +variable "resource_group_id" { + description = "Resource group ID" + type = string + default = null +} + +variable "members" { + description = "Number of members" + type = number + default = null +} + +variable "memory" { + description = "Ram in megabyte" + type = number + default = null +} + +variable "disks" { + description = "Rom in megabyte" + type = number + default = null +} + +variable "vcpu" { + description = "Number of cpu cores" + type = number + default = null +} + +variable "service_endpoints" { + description = "The service endpoints" + type = string + default = "private" +} diff --git a/modules/database/mysql/version.tf b/modules/database/mysql/version.tf new file mode 100644 index 00000000..01db83a5 --- /dev/null +++ b/modules/database/mysql/version.tf @@ -0,0 +1,3 @@ +terraform { + required_version = ">= 1.3, < 1.7" +} diff --git a/modules/dns/main.tf b/modules/dns/main.tf index ba5fcd73..b6e9fabc 100644 --- a/modules/dns/main.tf +++ b/modules/dns/main.tf @@ -5,12 +5,19 @@ resource "ibm_resource_instance" "itself" { location = "global" service = "dns-svcs" plan = "standard-dns" + tags = local.tags } locals { dns_instance_id = var.dns_instance_id == null ? ibm_resource_instance.itself[0].guid : var.dns_instance_id } +locals { + name = "hpc" + prefix = var.prefix + tags = [local.prefix, local.name] +} + resource "ibm_dns_custom_resolver" "itself" { count = var.dns_custom_resolver_id == null ? 1 : 0 name = format("%s-custom-resolver", var.prefix) @@ -18,7 +25,7 @@ resource "ibm_dns_custom_resolver" "itself" { enabled = true high_availability = length(var.subnets_crn) > 1 ? true : false dynamic "locations" { - for_each = length(var.subnets_crn) > 3 ? slice(var.subnets_crn, 0, 3) : var.subnets_crn + for_each = length(var.subnets_crn) > 3 ? slice(var.subnets_crn, 0, 2) : var.subnets_crn content { subnet_crn = locations.value enabled = true @@ -26,19 +33,10 @@ resource "ibm_dns_custom_resolver" "itself" { } } -data "ibm_dns_zones" "conditional" { - count = var.dns_instance_id != null ? 1 : 0 - instance_id = var.dns_instance_id -} - -locals { - dns_domain_names = flatten([setsubtract(var.dns_domain_names == null ? [] : var.dns_domain_names, flatten(data.ibm_dns_zones.conditional[*].dns_zones[*]["name"]))]) -} - resource "ibm_dns_zone" "itself" { - count = length(local.dns_domain_names) + count = 1 instance_id = local.dns_instance_id - name = local.dns_domain_names[count.index] + name = var.dns_domain_names[0] } data "ibm_dns_zones" "itself" { @@ -53,9 +51,9 @@ locals { } resource "ibm_dns_permitted_network" "itself" { - count = length(var.dns_domain_names) + count = 1 instance_id = local.dns_instance_id vpc_crn = var.vpc_crn - zone_id = one(values(local.dns_zone_maps[count.index])) + zone_id = split("/", ibm_dns_zone.itself[0].id)[1] type = "vpc" } diff --git a/modules/dns/outputs.tf b/modules/dns/outputs.tf index 39022f0c..23937e1e 100644 --- a/modules/dns/outputs.tf +++ b/modules/dns/outputs.tf @@ -3,11 +3,6 @@ output "dns_instance_id" { value = local.dns_instance_id } -output "dns_custom_resolver_id" { - description = "DNS custom resolver ID" - value = var.dns_custom_resolver_id == null ? one(ibm_dns_custom_resolver.itself[*].id) : var.dns_custom_resolver_id -} - output "dns_zone_maps" { description = "DNS zones" value = local.dns_zone_maps diff --git a/modules/dns/variables.tf b/modules/dns/variables.tf index 4565fb1f..a16bfeea 100644 --- a/modules/dns/variables.tf +++ b/modules/dns/variables.tf @@ -1,14 +1,3 @@ -############################################################################## -# Account Variables -############################################################################## - -variable "ibmcloud_api_key" { - description = "IBM Cloud API Key that will be used for authentication in scripts run in this module. Only required if certain options are required." - type = string - sensitive = true - default = null -} - ############################################################################## # Resource Groups Variables ############################################################################## diff --git a/modules/dns/version.tf b/modules/dns/version.tf index fc2eae13..2fb790db 100644 --- a/modules/dns/version.tf +++ b/modules/dns/version.tf @@ -7,7 +7,3 @@ terraform { } } } - -provider "ibm" { - ibmcloud_api_key = var.ibmcloud_api_key -} diff --git a/modules/dns_record/main.tf b/modules/dns_record/main.tf index aa84e997..a5755e7b 100644 --- a/modules/dns_record/main.tf +++ b/modules/dns_record/main.tf @@ -1,13 +1,3 @@ -data "ibm_dns_zones" "itself" { - instance_id = var.dns_instance_id -} - -locals { - dns_domain_name = [ - for zone in data.ibm_dns_zones.itself.dns_zones : zone["name"] if zone["zone_id"] == var.dns_zone_id - ] -} - resource "ibm_dns_resource_record" "a" { count = length(var.dns_records) instance_id = var.dns_instance_id @@ -17,6 +7,9 @@ resource "ibm_dns_resource_record" "a" { rdata = var.dns_records[count.index]["rdata"] ttl = 300 } +########################### +# TODO: on line number30 update the var.dns_domain_names to pick up existing domain name when we support scale/protocol domain names +########################## resource "ibm_dns_resource_record" "ptr" { count = length(var.dns_records) @@ -24,7 +17,8 @@ resource "ibm_dns_resource_record" "ptr" { zone_id = var.dns_zone_id type = "PTR" name = var.dns_records[count.index]["rdata"] - rdata = format("%s.%s", var.dns_records[count.index]["name"], one(local.dns_domain_name)) - ttl = 300 - depends_on = [ibm_dns_resource_record.a] + rdata = format("%s.%s", var.dns_records[count.index]["name"], var.dns_domain_names["compute"]) + #rdata = format("%s.%s", var.dns_records[count.index]["name"], one(local.dns_domain_name)) + ttl = 300 + depends_on = [ibm_dns_resource_record.a] } diff --git a/modules/dns_record/variables.tf b/modules/dns_record/variables.tf index 295b5715..473e760c 100644 --- a/modules/dns_record/variables.tf +++ b/modules/dns_record/variables.tf @@ -1,14 +1,3 @@ -############################################################################## -# Account Variables -############################################################################## - -variable "ibmcloud_api_key" { - description = "IBM Cloud API Key that will be used for authentication in scripts run in this module. Only required if certain options are required." - type = string - sensitive = true - default = null -} - ############################################################################## # DNS Variables ############################################################################## @@ -33,3 +22,15 @@ variable "dns_records" { default = null description = "IBM Cloud HPC DNS record." } + +variable "dns_domain_names" { + type = object({ + compute = string + #storage = string + #protocol = string + }) + default = { + compute = "comp.com" + } + description = "IBM Cloud HPC DNS domain names." +} diff --git a/modules/dns_record/version.tf b/modules/dns_record/version.tf index fc2eae13..2fb790db 100644 --- a/modules/dns_record/version.tf +++ b/modules/dns_record/version.tf @@ -7,7 +7,3 @@ terraform { } } } - -provider "ibm" { - ibmcloud_api_key = var.ibmcloud_api_key -} diff --git a/modules/file_storage/locals.tf b/modules/file_storage/locals.tf index c31aeaa7..8f34e4bc 100644 --- a/modules/file_storage/locals.tf +++ b/modules/file_storage/locals.tf @@ -1,4 +1,5 @@ locals { - # Region and Zone calculations - region = join("-", slice(split("-", var.zone), 0, 2)) + name = "hpc" + prefix = var.prefix + tags = [local.prefix, local.name] } diff --git a/modules/file_storage/main.tf b/modules/file_storage/main.tf index 043e26f3..a6340a07 100644 --- a/modules/file_storage/main.tf +++ b/modules/file_storage/main.tf @@ -7,6 +7,24 @@ resource "ibm_is_share" "share" { iops = var.file_shares[count.index]["iops"] zone = var.zone encryption_key = var.encryption_key_crn + resource_group = var.resource_group + tags = local.tags + depends_on = [time_sleep.wait_for_authorization_policy] +} + +resource "ibm_iam_authorization_policy" "policy" { + count = var.kms_encryption_enabled == false || var.skip_iam_share_authorization_policy ? 0 : 1 + source_service_name = "is" + source_resource_type = "share" + target_service_name = "kms" + target_resource_instance_id = var.existing_kms_instance_guid + roles = ["Reader"] +} + +resource "time_sleep" "wait_for_authorization_policy" { + depends_on = [ibm_iam_authorization_policy.policy[0]] + + create_duration = "30s" } resource "ibm_is_share_mount_target" "share_target_vpc" { @@ -28,5 +46,5 @@ resource "ibm_is_share_mount_target" "share_target_sg" { name = format("%s-fs-vni", var.file_shares[count.index]["name"]) security_groups = var.security_group_ids } - transit_encryption = "user_managed" + #transit_encryption = "user_managed" } diff --git a/modules/file_storage/outputs.tf b/modules/file_storage/outputs.tf index e7a7b245..ae7a5df9 100644 --- a/modules/file_storage/outputs.tf +++ b/modules/file_storage/outputs.tf @@ -5,3 +5,44 @@ output "mount_path" { ibm_is_share_mount_target.share_target_sg[*].mount_path ]) } + +output "mount_paths_info" { + description = "Information about mount paths" + value = { + original_list_length = length(ibm_is_share_mount_target.share_target_sg[*].mount_path) + original_list = ibm_is_share_mount_target.share_target_sg[*].mount_path + exclude_first_element = length(ibm_is_share_mount_target.share_target_sg[*].mount_path) > 1 ? slice(ibm_is_share_mount_target.share_target_sg[*].mount_path, 1, length(ibm_is_share_mount_target.share_target_sg[*].mount_path) - 1) : [] + } +} + +output "mount_path_1" { + description = "Mount path" + #value = ibm_is_share_mount_target.share_target_sg[0].mount_path + #value = output "mount_path_1" { + value = length(ibm_is_share_mount_target.share_target_sg) > 0 ? ibm_is_share_mount_target.share_target_sg[0].mount_path : null +} + +output "mount_paths_excluding_first" { + description = "Mount paths excluding the first element" + value = length(ibm_is_share_mount_target.share_target_sg[*].mount_path) > 1 ? slice(ibm_is_share_mount_target.share_target_sg[*].mount_path, 1, length(ibm_is_share_mount_target.share_target_sg[*].mount_path)) : [] +} + +#output "mount_paths_excluding_first" { +# description = "Mount paths excluding the first element" +# value = ibm_is_share_mount_target.share_target_vpc[*].mount_path[1:] +#} + +#output "mount_paths_excluding_first" { +# description = "Mount paths excluding the first element" +# value = length(ibm_is_share_mount_target.share_target_sg[*].mount_path) > 1 ? slice(ibm_is_share_mount_target.share_target_sg[*].mount_path, 1, length(ibm_is_share_mount_target.share_target_sg[*].mount_path) - 1) : [] +#} + +#output "mount_paths_excluding_first" { +# description = "Mount paths excluding the first element" +# value = length(ibm_is_share_mount_target.share_target_sg[*].mount_path) > 1 ? slice(ibm_is_share_mount_target.share_target_sg[*].mount_path, 1, length(ibm_is_share_mount_target.share_target_sg[*].mount_path) - 1) : [] +#} + +#output "mount_paths_excluding_first" { +# description = "Mount paths excluding the first element" +# value = length(ibm_is_share_mount_target.share_target_sg[*].mount_path) > 1 ? tail(ibm_is_share_mount_target.share_target_sg[*].mount_path, length(ibm_is_share_mount_target.share_target_sg[*].mount_path) - 1) : [] +#} diff --git a/modules/file_storage/variables.tf b/modules/file_storage/variables.tf index c2ae92dd..9de6e76b 100644 --- a/modules/file_storage/variables.tf +++ b/modules/file_storage/variables.tf @@ -1,13 +1,16 @@ -variable "ibmcloud_api_key" { - description = "IBM Cloud API Key that will be used for authentication in scripts run in this module. Only required if certain options are required." +variable "zone" { + description = "Region where VPC will be created. To find your VPC region, use `ibmcloud is regions` command to find available regions." type = string - sensitive = true - default = null } -variable "zone" { - description = "Region where VPC will be created. To find your VPC region, use `ibmcloud is regions` command to find available regions." +############################################################################## +# Resource Groups Variables +############################################################################## + +variable "resource_group" { + description = "String describing resource groups to create or reference" type = string + default = null } variable "file_shares" { @@ -45,3 +48,26 @@ variable "subnet_id" { description = "Subnet ID to mount file share" default = null } + +variable "prefix" { + description = "A unique identifier for resources. Must begin with a letter and end with a letter or number. This prefix will be prepended to any resources provisioned by this template. Prefixes must be 16 or fewer characters." + type = string +} + +variable "existing_kms_instance_guid" { + type = string + default = null + description = "GUID of boot volume encryption key" +} + +variable "skip_iam_share_authorization_policy" { + type = bool + default = false + description = "Set it to false if authorization policy is required for VPC file share to access kms. This can be set to true if authorization policy already exists. For more information on how to create authorization policy manually, see [creating authorization policies for VPC file share](https://cloud.ibm.com/docs/vpc?topic=vpc-file-s2s-auth&interface=ui)." +} + +variable "kms_encryption_enabled" { + description = "Enable Key management" + type = bool + default = true +} diff --git a/modules/file_storage/version.tf b/modules/file_storage/version.tf index 0c47ca34..46c93b5b 100644 --- a/modules/file_storage/version.tf +++ b/modules/file_storage/version.tf @@ -5,10 +5,9 @@ terraform { source = "IBM-Cloud/ibm" version = ">= 1.56.2" } + time = { + source = "hashicorp/time" + version = ">=0.11.2" + } } } - -provider "ibm" { - ibmcloud_api_key = var.ibmcloud_api_key - region = local.region -} diff --git a/modules/inventory/main.tf b/modules/inventory/main.tf index d7eb0797..f2a2697f 100644 --- a/modules/inventory/main.tf +++ b/modules/inventory/main.tf @@ -1,4 +1,4 @@ resource "local_sensitive_file" "itself" { - content = join("\n", var.hosts) + content = join("\n", concat([var.server_name, var.user], var.hosts)) filename = var.inventory_path } diff --git a/modules/inventory/variables.tf b/modules/inventory/variables.tf index 1082b2f3..dedf2c20 100644 --- a/modules/inventory/variables.tf +++ b/modules/inventory/variables.tf @@ -4,6 +4,16 @@ variable "hosts" { default = ["localhost"] } +variable "user" { + description = "user" + type = string +} + +variable "server_name" { + description = "server_name" + type = string +} + variable "inventory_path" { description = "Inventory file path" type = string diff --git a/modules/key/main.tf b/modules/key/main.tf index d5081e27..abcc5097 100644 --- a/modules/key/main.tf +++ b/modules/key/main.tf @@ -2,10 +2,3 @@ resource "tls_private_key" "itself" { algorithm = "RSA" rsa_bits = 4096 } - -resource "local_sensitive_file" "write_private_key" { - count = var.private_key_path != null ? 1 : 0 - content = tls_private_key.itself.private_key_pem - filename = var.private_key_path - file_permission = "0600" -} diff --git a/modules/key/variables.tf b/modules/key/variables.tf index 09fe711c..bdb2b8cd 100644 --- a/modules/key/variables.tf +++ b/modules/key/variables.tf @@ -1,5 +1 @@ -variable "private_key_path" { - description = "Private key file path" - type = string - default = null -} +# This empty file exists to suppress TFLint Warning on the terraform_standard_module_structure diff --git a/modules/key/version.tf b/modules/key/version.tf index 358d5a57..bba7b669 100644 --- a/modules/key/version.tf +++ b/modules/key/version.tf @@ -1,10 +1,6 @@ terraform { required_version = ">= 1.3, < 1.7" required_providers { - local = { - source = "hashicorp/local" - version = "~> 2" - } tls = { source = "hashicorp/tls" version = "~> 4" diff --git a/modules/landing_zone/datasource.tf b/modules/landing_zone/datasource.tf new file mode 100644 index 00000000..27fa72ea --- /dev/null +++ b/modules/landing_zone/datasource.tf @@ -0,0 +1,21 @@ +data "ibm_resource_instance" "kms_instance" { + count = (var.key_management == "key_protect" && var.kms_instance_name != null) ? 1 : 0 + name = var.kms_instance_name + service = "kms" +} + +data "ibm_kms_key" "kms_key" { + count = (var.key_management == "key_protect" && var.kms_key_name != null) ? 1 : 0 + instance_id = data.ibm_resource_instance.kms_instance[0].id + key_name = var.kms_key_name +} + +data "ibm_is_vpc" "itself" { + count = var.vpc == null ? 0 : 1 + name = var.vpc +} + +data "ibm_is_subnet" "subnet" { + count = (var.vpc != null && length(var.subnet_id) > 0) ? 1 : 0 + identifier = var.subnet_id[count.index] +} diff --git a/modules/landing_zone/locals.tf b/modules/landing_zone/locals.tf index c36a66b7..966b940b 100644 --- a/modules/landing_zone/locals.tf +++ b/modules/landing_zone/locals.tf @@ -3,35 +3,30 @@ locals { name = "hpc" prefix = var.prefix tags = [local.prefix, local.name] - schematics_reserved_cidrs = [ - "169.44.0.0/14", - "169.60.0.0/14", - "158.175.0.0/16", - "158.176.0.0/15", - "141.125.0.0/16", - "161.156.0.0/16", - "149.81.0.0/16", - "159.122.111.224/27", - "150.238.230.128/27", - "169.55.82.128/27" - ] + # schematics_reserved_cidrs = [ + # "169.44.0.0/14", + # "169.60.0.0/14", + # "158.175.0.0/16", + # "158.176.0.0/15", + # "141.125.0.0/16", + # "161.156.0.0/16", + # "149.81.0.0/16", + # "159.122.111.224/27", + # "150.238.230.128/27", + # "169.55.82.128/27" + # ] # Derived values # Resource group calculation # If user defined then use existing else create new - create_resource_group = var.resource_group == null ? true : false - resource_groups = var.resource_group == null ? [ + create_resource_group = var.resource_group == "null" ? true : false + resource_groups = var.resource_group == "null" ? [ { name = "service-rg", create = local.create_resource_group, use_prefix : false }, - { - name = "management-rg", - create = local.create_resource_group, - use_prefix : false - }, { name = "workload-rg", create = local.create_resource_group, @@ -44,68 +39,35 @@ locals { } ] # For the variables looking for resource group names only (transit_gateway, key_management, atracker) - resource_group = var.resource_group == null ? "service-rg" : var.resource_group - - login_instance_count = sum(var.login_instances[*]["count"]) - management_instance_count = sum(var.management_instances[*]["count"]) - static_compute_instance_count = sum(var.compute_instances[*]["count"]) - storage_instance_count = sum(var.storage_instances[*]["count"]) - protocol_instance_count = sum(var.protocol_instances[*]["count"]) - - # Region and Zone calculations - region = join("-", slice(split("-", var.zones[0]), 0, 2)) - zones = ["zone-1", "zone-2", "zone-3"] + resource_group = var.resource_group == "null" ? "service-rg" : var.resource_group + region = join("-", slice(split("-", var.zones[0]), 0, 2)) + zones = ["zone-1", "zone-2", "zone-3"] active_zones = [ for zone in var.zones : format("zone-%d", substr(zone, -1, -2)) ] - # Future use - #zone_count = length(local.active_zones) - - # Address Prefixes calculation + bastion_sg_variable_cidr_list = split(",", var.network_cidr) address_prefixes = { - for zone in local.zones : zone => contains(local.active_zones, zone) ? distinct(compact([ - local.login_instance_count != 0 && local.management_instance_count != 0 ? var.login_subnets_cidr[index(local.active_zones, zone)] : null, - var.compute_subnets_cidr[index(local.active_zones, zone)], - local.storage_instance_count != 0 ? var.storage_subnets_cidr[index(local.active_zones, zone)] : null, - local.storage_instance_count != 0 && local.protocol_instance_count != 0 ? var.protocol_subnets_cidr[index(local.active_zones, zone)] : null, - # bastion subnet and instance will always be in first active zone - zone == local.active_zones[0] ? var.bastion_subnets_cidr[0] : null - ])) : [] + "zone-${element(split("-", var.zones[0]), 2)}" = [local.bastion_sg_variable_cidr_list[0]] } # Subnet calculation active_subnets = { for zone in local.zones : zone => contains(local.active_zones, zone) ? [ - local.login_instance_count != 0 && local.management_instance_count != 0 ? { - name = "login-subnet-${zone}" - acl_name = "hpc-acl" - cidr = var.login_subnets_cidr[index(local.active_zones, zone)] - public_gateway = false - } : null, { name = "compute-subnet-${zone}" acl_name = "hpc-acl" cidr = var.compute_subnets_cidr[index(local.active_zones, zone)] - public_gateway = true + public_gateway = var.vpc == null ? true : false + no_addr_prefix = var.no_addr_prefix + }, - local.storage_instance_count != 0 ? { - name = "storage-subnet-${zone}" - acl_name = "hpc-acl" - cidr = var.storage_subnets_cidr[index(local.active_zones, zone)] - public_gateway = true - } : null, - local.storage_instance_count != 0 && local.protocol_instance_count != 0 ? { - name = "protocol-subnet-${zone}" - acl_name = "hpc-acl" - cidr = var.protocol_subnets_cidr[index(local.active_zones, zone)] - public_gateway = false - } : null, zone == local.active_zones[0] ? { name = "bastion-subnet" acl_name = "hpc-acl" cidr = var.bastion_subnets_cidr[0] public_gateway = false + no_addr_prefix = var.no_addr_prefix } : null ] : [] } @@ -115,36 +77,47 @@ locals { use_public_gateways = { for zone in local.zones : zone => contains(local.active_zones, zone) ? true : false } - - # VPC calculation - # If user defined then use existing else create new - # Calculate network acl rules (can be done inplace in vpcs) - # TODO: VPN expectation - cidrs_network_acl_rules = compact(flatten([local.schematics_reserved_cidrs, var.allowed_cidr, var.network_cidr, "161.26.0.0/16", "166.8.0.0/14"])) network_acl_inbound_rules = [ - for cidr_index in range(length(local.cidrs_network_acl_rules)) : { - name = format("allow-inbound-%s", cidr_index + 1) + { + name = "test-1" action = "allow" - destination = var.network_cidr + destination = "0.0.0.0/0" direction = "inbound" - source = element(local.cidrs_network_acl_rules, cidr_index) + source = "0.0.0.0/0" } ] network_acl_outbound_rules = [ - for cidr_index in range(length(local.cidrs_network_acl_rules)) : { - name = format("allow-outbound-%s", cidr_index + 1) + { + name = "test-2" action = "allow" - destination = element(local.cidrs_network_acl_rules, cidr_index) + destination = "0.0.0.0/0" direction = "outbound" - source = var.network_cidr + source = "0.0.0.0/0" } ] network_acl_rules = flatten([local.network_acl_inbound_rules, local.network_acl_outbound_rules]) - vpcs = var.vpc == null ? [ + use_public_gateways_existing_vpc = { + "zone-1" = false + "zone-2" = false + "zone-3" = false + } + + vpcs = [ { + existing_vpc_id = var.vpc == null ? null : data.ibm_is_vpc.itself[0].id + existing_subnets = (var.vpc != null && length(var.subnet_id) > 0) ? [ + { + id = var.subnet_id[0] + public_gateway = false + }, + { + id = var.login_subnet_id + public_gateway = false + } + ] : null prefix = local.name - resource_group = var.resource_group == null ? "workload-rg" : var.resource_group + resource_group = var.resource_group == "null" ? "workload-rg" : var.resource_group clean_default_security_group = true clean_default_acl = true flow_logs_bucket_name = var.enable_vpc_flow_logs ? "vpc-flow-logs-bucket" : null @@ -155,11 +128,11 @@ locals { rules = local.network_acl_rules } ], - subnets = local.subnets - use_public_gateways = local.use_public_gateways - address_prefixes = local.address_prefixes + subnets = (var.vpc != null && length(var.subnet_id) > 0) ? null : local.subnets + use_public_gateways = var.vpc == null ? local.use_public_gateways : local.use_public_gateways_existing_vpc + address_prefixes = var.vpc == null ? local.address_prefixes : null } - ] : [] + ] # Define SSH key ssh_keys = [ @@ -167,105 +140,16 @@ locals { name = item } ] - #bastion_ssh_keys = var.bastion_ssh_keys - - # Sample to spin VSI - /* - bastion_vsi = { - name = "bastion-vsi" - resource_group = var.resource_group == null ? "management-rg" : var.resource_group - image_name = "ibm-ubuntu-22-04-1-minimal-amd64-4" - machine_type = "cx2-4x8" - vpc_name = var.vpc == null ? local.name : var.vpc - subnet_names = ["bastion-subnet"] - ssh_keys = local.bastion_ssh_keys - vsi_per_subnet = 1 - user_data = var.enable_bastion ? data.template_file.bastion_user_data.rendered : null - enable_floating_ip = true - boot_volume_encryption_key_name = var.key_management == null ? null : format("%s-vsi-key", var.prefix) - security_group = { - name = "bastion-sg" - rules = flatten([ - { - name = "allow-ibm-inbound" - direction = "inbound" - source = "161.26.0.0/16" - }, - { - name = "allow-vpc-inbound" - direction = "inbound" - source = var.network_cidr - }, - [for cidr_index in range(length(flatten([local.schematics_reserved_cidrs, var.allowed_cidr]))) : { - name = format("allow-variable-inbound-%s", cidr_index + 1) - direction = "inbound" - source = element(var.allowed_cidr, cidr_index) - # ssh port - tcp = { - port_min = 22 - port_max = 22 - } - }], - { - name = "allow-ibm-http-outbound" - direction = "outbound" - source = "161.26.0.0/16" - tcp = { - port_min = 80 - port_max = 80 - } - }, - { - name = "allow-ibm-https-outbound" - direction = "outbound" - source = "161.26.0.0/16" - tcp = { - port_min = 443 - port_max = 443 - } - }, - { - name = "allow-ibm-dns-outbound" - direction = "outbound" - source = "161.26.0.0/16" - tcp = { - port_min = 53 - port_max = 53 - } - }, - { - name = "allow-vpc-outbound" - direction = "outbound" - source = var.network_cidr - }, - [for cidr_index in range(length(flatten([local.schematics_reserved_cidrs, var.allowed_cidr]))) : { - name = format("allow-variable-outbound-%s", cidr_index + 1) - direction = "outbound" - source = element(var.allowed_cidr, cidr_index) - }] - ]) - } - } - vsi = [local.bastion_vsi] - */ vsi = [] # Define VPN vpn_gateways = var.enable_vpn ? [ { name = "vpn-gw" - vpc_name = var.vpc == null ? local.name : var.vpc - subnet_name = "bastion-subnet" + vpc_name = local.name + subnet_name = length(var.subnet_id) == 0 ? "bastion-subnet" : data.ibm_is_subnet.subnet[0].name mode = "policy" resource_group = local.resource_group - connections = [ - { - peer_address = var.vpn_peer_address - preshared_key = var.vpn_preshared_key - peer_cidrs = var.vpn_peer_cidr - local_cidrs = var.bastion_subnets_cidr - } - ] } ] : [] @@ -276,7 +160,7 @@ locals { active_cos = [ ( - var.enable_cos_integration || var.enable_vpc_flow_logs || var.enable_atracker + var.enable_cos_integration || var.enable_vpc_flow_logs || var.enable_atracker || var.scc_enable ) ? { name = var.cos_instance_name == null ? "hpc-cos" : var.cos_instance_name resource_group = local.resource_group @@ -292,21 +176,28 @@ locals { storage_class = "standard" endpoint_type = "public" force_delete = true - kms_key = var.key_management == "key_protect" ? format("%s-key", var.prefix) : null + kms_key = var.key_management == "key_protect" ? (var.kms_key_name == null ? format("%s-key", var.prefix) : var.kms_key_name) : null } : null, var.enable_vpc_flow_logs ? { name = "vpc-flow-logs-bucket" storage_class = "standard" endpoint_type = "public" force_delete = true - kms_key = var.key_management == "key_protect" ? format("%s-slz-key", var.prefix) : null + kms_key = var.key_management == "key_protect" ? (var.kms_key_name == null ? format("%s-slz-key", var.prefix) : var.kms_key_name) : null } : null, var.enable_atracker ? { name = "atracker-bucket" storage_class = "standard" endpoint_type = "public" force_delete = true - kms_key = var.key_management == "key_protect" ? format("%s-atracker-key", var.prefix) : null + kms_key = var.key_management == "key_protect" ? (var.kms_key_name == null ? format("%s-atracker-key", var.prefix) : var.kms_key_name) : null + } : null, + var.scc_enable ? { + name = "scc-bucket" + storage_class = "standard" + endpoint_type = "public" + force_delete = true + kms_key = null } : null ] } : null @@ -336,9 +227,8 @@ locals { if instance != null ] - # Prerequisite: Existing key protect instance is not supported, always create a key management instance - active_keys = [ - var.key_management != null ? { + active_keys = var.key_management == "key_protect" ? (var.kms_key_name == null ? [ + var.key_management == "key_protect" ? { name = format("%s-vsi-key", var.prefix) } : null, var.enable_cos_integration ? { @@ -350,48 +240,45 @@ locals { var.enable_atracker ? { name = format("%s-atracker-key", var.prefix) } : null - ] - key_management = { - name = var.key_management == "hs_crypto" ? var.hpcs_instance_name : format("%s-kms", var.prefix) + ] : [ + { + name = var.kms_key_name + existing_key_crn = data.ibm_kms_key.kms_key[0].keys[0].crn + } + ]) : null + key_management = var.key_management == "key_protect" ? { + name = var.kms_instance_name != null ? var.kms_instance_name : format("%s-kms", var.prefix) # var.key_management == "hs_crypto" ? var.hpcs_instance_name : format("%s-kms", var.prefix) resource_group = local.resource_group - use_hs_crypto = var.key_management == "hs_crypto" ? true : false + use_hs_crypto = false keys = [for each in local.active_keys : each if each != null] + use_data = var.kms_instance_name != null ? true : false + } : { + name = null + resource_group = null + use_hs_crypto = null + keys = [] + use_data = null } - - total_vsis = sum([ - local.management_instance_count, - local.static_compute_instance_count, - local.storage_instance_count, - local.protocol_instance_count - ]) * length(local.active_zones) - placement_groups_count = var.placement_strategy == "host_spread" ? local.total_vsis / 12 : var.placement_strategy == "power_spread" ? local.total_vsis / 4 : 0 - vpc_placement_groups = [ - for placement_group in range(local.placement_groups_count) : { - name = format("%s", placement_group + 1) - resource_group = local.resource_group - strategy = var.placement_strategy - } - ] - # Unexplored variables security_groups = [] virtual_private_endpoints = [] service_endpoints = "private" atracker = { resource_group = local.resource_group - receive_global_events = var.enable_atracker + receive_global_events = false collector_bucket_name = "atracker-bucket" - add_route = var.enable_atracker + add_route = var.enable_atracker ? true : false } secrets_manager = { use_secrets_manager = false } - access_groups = [] - f5_vsi = [] - add_kms_block_storage_s2s = false - clusters = [] - wait_till = "IngressReady" - teleport_vsi = [] + access_groups = [] + f5_vsi = [] + #add_kms_block_storage_s2s = false + skip_kms_block_storage_s2s_auth_policy = true + clusters = [] + wait_till = "IngressReady" + teleport_vsi = [] iam_account_settings = { enable = false } @@ -411,32 +298,31 @@ locals { locals { env = { #ibmcloud_api_key = var.ibmcloud_api_key - resource_groups = local.resource_groups - network_cidr = var.network_cidr - vpcs = local.vpcs - vpn_gateways = local.vpn_gateways - enable_transit_gateway = local.enable_transit_gateway - transit_gateway_resource_group = local.transit_gateway_resource_group - transit_gateway_connections = local.transit_gateway_connections - vsi = local.vsi - ssh_keys = local.ssh_keys - cos = local.cos - key_management = local.key_management - atracker = local.atracker - vpc_placement_groups = local.vpc_placement_groups - security_groups = local.security_groups - virtual_private_endpoints = local.virtual_private_endpoints - service_endpoints = local.service_endpoints - add_kms_block_storage_s2s = local.add_kms_block_storage_s2s - clusters = local.clusters - wait_till = local.wait_till - iam_account_settings = local.iam_account_settings - access_groups = local.access_groups - f5_vsi = local.f5_vsi - f5_template_data = local.f5_template_data - appid = local.appid - teleport_config_data = local.teleport_config_data - teleport_vsi = local.teleport_vsi - secrets_manager = local.secrets_manager + resource_groups = local.resource_groups + network_cidr = var.network_cidr + vpcs = local.vpcs + vpn_gateways = local.vpn_gateways + enable_transit_gateway = local.enable_transit_gateway + transit_gateway_resource_group = local.transit_gateway_resource_group + transit_gateway_connections = local.transit_gateway_connections + vsi = local.vsi + ssh_keys = local.ssh_keys + cos = local.cos + key_management = local.key_management + atracker = local.atracker + security_groups = local.security_groups + virtual_private_endpoints = local.virtual_private_endpoints + service_endpoints = local.service_endpoints + skip_kms_block_storage_s2s_auth_policy = local.skip_kms_block_storage_s2s_auth_policy + clusters = local.clusters + wait_till = local.wait_till + iam_account_settings = local.iam_account_settings + access_groups = local.access_groups + f5_vsi = local.f5_vsi + f5_template_data = local.f5_template_data + appid = local.appid + teleport_config_data = local.teleport_config_data + teleport_vsi = local.teleport_vsi + secrets_manager = local.secrets_manager } } diff --git a/modules/landing_zone/main.tf b/modules/landing_zone/main.tf index 4e3ee2a1..0fa23148 100644 --- a/modules/landing_zone/main.tf +++ b/modules/landing_zone/main.tf @@ -1,35 +1,31 @@ module "landing_zone" { - count = var.enable_landing_zone ? 1 : 0 - source = "terraform-ibm-modules/landing-zone/ibm" - version = "4.15.0" - prefix = local.prefix - region = local.region - tags = local.tags - resource_groups = local.env.resource_groups - network_cidr = local.env.network_cidr - vpcs = local.env.vpcs - vpn_gateways = local.env.vpn_gateways - enable_transit_gateway = local.env.enable_transit_gateway - transit_gateway_resource_group = local.env.transit_gateway_resource_group - transit_gateway_connections = local.env.transit_gateway_connections - ssh_keys = local.env.ssh_keys - vsi = local.env.vsi - security_groups = local.env.security_groups - virtual_private_endpoints = local.env.virtual_private_endpoints - cos = local.env.cos - service_endpoints = local.env.service_endpoints - key_management = local.env.key_management - add_kms_block_storage_s2s = local.env.add_kms_block_storage_s2s - atracker = local.env.atracker - clusters = local.env.clusters - wait_till = local.env.wait_till - iam_account_settings = local.env.iam_account_settings - access_groups = local.env.access_groups - f5_vsi = local.env.f5_vsi - f5_template_data = local.env.f5_template_data - appid = local.env.appid - teleport_config_data = local.env.teleport_config_data - teleport_vsi = local.env.teleport_vsi - secrets_manager = local.env.secrets_manager - vpc_placement_groups = local.env.vpc_placement_groups + count = var.enable_landing_zone ? 1 : 0 + source = "terraform-ibm-modules/landing-zone/ibm" + version = "5.23.0" + prefix = local.prefix + region = local.region + tags = local.tags + resource_groups = local.env.resource_groups + network_cidr = local.env.network_cidr + vpcs = local.env.vpcs + vpn_gateways = local.env.vpn_gateways + enable_transit_gateway = local.env.enable_transit_gateway + transit_gateway_resource_group = local.env.transit_gateway_resource_group + transit_gateway_connections = local.env.transit_gateway_connections + ssh_keys = local.env.ssh_keys + vsi = local.env.vsi + security_groups = local.env.security_groups + virtual_private_endpoints = local.env.virtual_private_endpoints + cos = local.env.cos + service_endpoints = local.env.service_endpoints + key_management = local.env.key_management + skip_kms_block_storage_s2s_auth_policy = local.env.skip_kms_block_storage_s2s_auth_policy + atracker = local.env.atracker + clusters = local.env.clusters + wait_till = local.env.wait_till + f5_vsi = local.env.f5_vsi + f5_template_data = local.env.f5_template_data + appid = local.env.appid + teleport_config_data = local.env.teleport_config_data + teleport_vsi = local.env.teleport_vsi } diff --git a/modules/landing_zone/outputs.tf b/modules/landing_zone/outputs.tf index 0d729ffb..a9a0aa95 100644 --- a/modules/landing_zone/outputs.tf +++ b/modules/landing_zone/outputs.tf @@ -1,22 +1,3 @@ -# To debug local values -/* -output "env_var" { - value = local.env -} - -output "landing_zone_data" { - value = module.landing_zone[*] -} - -output "vpc_data" { - value = module.landing_zone[*].vpc_data -} - -output "subnet_data" { - value = module.landing_zone[*].subnet_data -} -*/ - output "resource_group_id" { description = "Resource group ID" value = module.landing_zone[*].resource_group_data @@ -37,6 +18,22 @@ output "vpc_crn" { value = module.landing_zone[*].vpc_data[0].vpc_crn } +output "public_gateways" { + description = "Public Gateway IDs" + value = module.landing_zone[*].vpc_data[0].public_gateways +} + +output "subnets" { + description = "subnets" + value = [for subnet in flatten(module.landing_zone[*].subnet_data) : { + name = subnet["name"] + id = subnet["id"] + zone = subnet["zone"] + cidr = subnet["cidr"] + crn = subnet["crn"] + } + ] +} output "bastion_subnets" { description = "Bastion subnets" @@ -67,6 +64,8 @@ output "compute_subnets" { id = subnet["id"] zone = subnet["zone"] cidr = subnet["cidr"] + crn = subnet["crn"] + #ipv4_cidr_block = subnet["ipv4_cidr_block "] } if strcontains(subnet["name"], "-hpc-compute-subnet-zone-") ] } @@ -101,12 +100,22 @@ output "subnets_crn" { # TODO: Find a way to get CRN needed for VSI boot drive encryption output "boot_volume_encryption_key" { description = "Boot volume encryption key" - value = var.key_management != null ? module.landing_zone[*].key_map[format("%s-vsi-key", var.prefix)] : null + value = var.key_management == "key_protect" ? (var.kms_key_name == null ? module.landing_zone[*].key_map[format("%s-vsi-key", var.prefix)] : module.landing_zone[*].key_map[var.kms_key_name]) : null } output "key_management_guid" { description = "GUID for KMS instance" - value = var.key_management != null ? module.landing_zone[0].key_management_guid : null + value = var.key_management == "key_protect" ? module.landing_zone[0].key_management_guid : null +} + +output "cos_instance_crns" { + description = "CRN of the COS instance created by Landing Zone Module" + value = flatten(module.landing_zone[*].cos_data[*].crn) +} + +output "cos_buckets_names" { + description = "Name of the COS Bucket created for SCC Instance" + value = flatten(module.landing_zone[*].cos_bucket_names) } # TODO: Observability data diff --git a/modules/landing_zone/variables.tf b/modules/landing_zone/variables.tf index 205e208c..cdf0f160 100644 --- a/modules/landing_zone/variables.tf +++ b/modules/landing_zone/variables.tf @@ -2,13 +2,6 @@ # Account Variables ############################################################################## -variable "ibmcloud_api_key" { - description = "IBM Cloud API Key that will be used for authentication in scripts run in this module. Only required if certain options are required." - type = string - sensitive = true - default = null -} - variable "enable_landing_zone" { type = bool default = true @@ -54,16 +47,22 @@ variable "vpc" { default = null } -variable "network_cidr" { - description = "Network CIDR for the VPC. This is used to manage network ACL rules for cluster provisioning." - type = string - default = "10.0.0.0/8" +variable "subnet_id" { + type = list(string) + default = null + description = "List of existing subnet IDs under the VPC, where the cluster will be provisioned." } -variable "placement_strategy" { +variable "login_subnet_id" { type = string default = null - description = "VPC placement groups to create (null / host_spread / power_spread)" + description = "List of existing subnet ID under the VPC, where the login/Bastion server will be provisioned." +} + +variable "network_cidr" { + description = "Network CIDR for the VPC. This is used to manage network ACL rules for cluster provisioning." + type = string + default = "10.0.0.0/8" } variable "ssh_keys" { @@ -75,7 +74,6 @@ variable "ssh_keys" { # Access Variables ############################################################################## - variable "bastion_subnets_cidr" { type = list(string) default = ["10.0.0.0/24"] @@ -88,52 +86,9 @@ variable "enable_vpn" { description = "The solution supports multiple ways to connect to your HPC cluster for example, using bastion node, via VPN or direct connection. If connecting to the HPC cluster via VPN, set this value to true." } -variable "vpn_peer_cidr" { - type = list(string) - default = null - description = "The peer CIDRs (e.g., 192.168.0.0/24) to which the VPN will be connected." -} - -variable "vpn_peer_address" { - type = string - default = null - description = "The peer public IP address to which the VPN will be connected." -} - -variable "vpn_preshared_key" { - type = string - default = null - description = "The pre-shared key for the VPN." -} - -variable "allowed_cidr" { - description = "Network CIDR to access the VPC. This is used to manage network ACL rules for accessing the cluster." - type = list(string) - default = ["10.0.0.0/8"] -} - ############################################################################## # Compute Variables ############################################################################## -variable "login_subnets_cidr" { - type = list(string) - default = ["10.10.10.0/24", "10.20.10.0/24", "10.30.10.0/24"] - description = "Subnet CIDR block to launch the login host." -} - -variable "login_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 1 - }] - description = "Number of instances to be launched for login." -} variable "compute_subnets_cidr" { type = list(string) @@ -141,78 +96,6 @@ variable "compute_subnets_cidr" { description = "Subnet CIDR block to launch the compute cluster host." } -variable "management_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 3 - }] - description = "Number of instances to be launched for management." -} - -variable "compute_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 0 - }] - description = "Min Number of instances to be launched for compute cluster." -} - -############################################################################## -# Scale Storage Variables -############################################################################## - -variable "storage_subnets_cidr" { - type = list(string) - default = ["10.10.30.0/24", "10.20.30.0/24", "10.30.30.0/24"] - description = "Subnet CIDR block to launch the storage cluster host." -} - -variable "storage_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "bx2-2x8" - count = 3 - }] - description = "Number of instances to be launched for storage cluster." -} - -variable "protocol_subnets_cidr" { - type = list(string) - default = ["10.10.40.0/24", "10.20.40.0/24", "10.30.40.0/24"] - description = "Subnet CIDR block to launch the storage cluster host." -} - -variable "protocol_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "bx2-2x8" - count = 2 - }] - description = "Number of instances to be launched for protocol hosts." -} - ############################################################################## # Observability Variables ############################################################################## @@ -232,7 +115,7 @@ variable "cos_instance_name" { variable "enable_atracker" { type = bool default = true - description = "Enable Activity tracker" + description = "Enable Activity tracker on COS" } variable "enable_vpc_flow_logs" { @@ -241,6 +124,16 @@ variable "enable_vpc_flow_logs" { description = "Enable Activity tracker" } +############################################################################## +# SCC Variables +############################################################################## + +variable "scc_enable" { + type = bool + default = false + description = "Flag to enable SCC instance creation. If true, an instance of SCC (Security and Compliance Center) will be created." +} + ############################################################################## # Encryption Variables ############################################################################## @@ -248,11 +141,22 @@ variable "enable_vpc_flow_logs" { variable "key_management" { type = string default = null - description = "null/key_protect/hs_crypto" + description = "null/key_protect" +} + +variable "kms_instance_name" { + type = string + default = null + description = "Name of the Key Protect instance associated with the Key Management Service. The ID can be found under the details of the KMS, see [View key-protect ID](https://cloud.ibm.com/docs/key-protect?topic=key-protect-retrieve-instance-ID&interface=ui)." } -variable "hpcs_instance_name" { +variable "kms_key_name" { type = string default = null - description = "Hyper Protect Crypto Service instance" + description = "Provide the existing KMS encryption key name that you want to use for the IBM Cloud HPC cluster. (for example kms_key_name: my-encryption-key)." +} + +variable "no_addr_prefix" { + type = bool + description = "Set it as true, if you don't want to create address prefixes." } diff --git a/modules/landing_zone/version.tf b/modules/landing_zone/version.tf index 0c47ca34..2fb790db 100644 --- a/modules/landing_zone/version.tf +++ b/modules/landing_zone/version.tf @@ -7,8 +7,3 @@ terraform { } } } - -provider "ibm" { - ibmcloud_api_key = var.ibmcloud_api_key - region = local.region -} diff --git a/modules/landing_zone_vsi/configuration_steps/.gitignore b/modules/landing_zone_vsi/configuration_steps/.gitignore new file mode 100644 index 00000000..c46693cd --- /dev/null +++ b/modules/landing_zone_vsi/configuration_steps/.gitignore @@ -0,0 +1 @@ +management_values diff --git a/modules/landing_zone_vsi/configuration_steps/compute_user_data_fragment.sh b/modules/landing_zone_vsi/configuration_steps/compute_user_data_fragment.sh new file mode 100644 index 00000000..c676aa80 --- /dev/null +++ b/modules/landing_zone_vsi/configuration_steps/compute_user_data_fragment.sh @@ -0,0 +1,387 @@ +#!/bin/bash +# shellcheck disable=all + +if [ "$compute_user_data_vars_ok" != "1" ]; then + echo 2>&1 "fatal: vars block is missing" + exit 1 +fi + +echo "Logging initial env variables" >> $logfile +env|sort >> $logfile + +# Disallow root login +sed -i -e "s/^/no-port-forwarding,no-agent-forwarding,no-X11-forwarding,command=\"echo \'Please login as the user \\\"lsfadmin or vpcuser\\\" rather than the user \\\"root\\\".\';echo;sleep 5; exit 142\" /" /root/.ssh/authorized_keys + +# Updates the lsfadmin user as never expire +chage -I -1 -m 0 -M 99999 -E -1 -W 14 lsfadmin + +# Setup Hostname +HostIP=$(hostname -I | awk '{print $1}') +hostname=${cluster_prefix}-${HostIP//./-} +hostnamectl set-hostname "$hostname" + +echo "START $(date '+%Y-%m-%d %H:%M:%S')" >> $logfile + +# Setup Network configuration +# Change the MTU setting as this is required for setting mtu as 9000 for communication to happen between clusters +if grep -q "NAME=\"Red Hat Enterprise Linux\"" /etc/os-release; then + # Replace the MTU value in the Netplan configuration + echo "MTU=9000" >> "/etc/sysconfig/network-scripts/ifcfg-${network_interface}" + echo "DOMAIN=\"${dns_domain}\"" >> "/etc/sysconfig/network-scripts/ifcfg-${network_interface}" + # Change the MTU setting as 9000 at router level. + gateway_ip=$(ip route | grep default | awk '{print $3}' | head -n 1) + cidr_range=$(ip route show | grep "kernel" | awk '{print $1}' | head -n 1) + echo "$cidr_range via $gateway_ip dev ${network_interface} metric 0 mtu 9000" >> /etc/sysconfig/network-scripts/route-eth0 + # Restart the Network Manager. + systemctl restart NetworkManager +elif grep -q "NAME=\"Ubuntu\"" /etc/os-release; then + net_int=$(basename /sys/class/net/en*) + netplan_config="/etc/netplan/50-cloud-init.yaml" + gateway_ip=$(ip route | grep default | awk '{print $3}' | head -n 1) + cidr_range=$(ip route show | grep "kernel" | awk '{print $1}' | head -n 1) + usermod -s /bin/bash lsfadmin + # Replace the MTU value in the Netplan configuration + if ! grep -qE "^[[:space:]]*mtu: 9000" $netplan_config; then + echo "MTU 9000 Packages entries not found" + # Append the MTU configuration to the Netplan file + sudo sed -i '/'"$net_int"':/a\ mtu: 9000' $netplan_config + sudo sed -i "/dhcp4: true/a \ nameservers:\n search: [$dns_domain]" $netplan_config + sudo sed -i '/'"$net_int"':/a\ routes:\n - to: '"$cidr_range"'\n via: '"$gateway_ip"'\n metric: 100\n mtu: 9000' $netplan_config + sudo netplan apply + echo "MTU set to 9000 on Netplan." + else + echo "MTU entry already exists in Netplan. Skipping." + fi +fi + +# TODO: Conditional NFS mount +LSF_TOP="/opt/ibm/lsf" +# Setup file share +if [ -n "${nfs_server_with_mount_path}" ]; then + echo "File share ${nfs_server_with_mount_path} found" >> $logfile + nfs_client_mount_path="/mnt/lsf" + rm -rf "${nfs_client_mount_path}" + mkdir -p "${nfs_client_mount_path}" + # Mount LSF TOP + mount -t nfs -o sec=sys "$nfs_server_with_mount_path" "$nfs_client_mount_path" >> $logfile + # Verify mount + if mount | grep "$nfs_client_mount_path"; then + echo "Mount found" >> $logfile + else + echo "No mount found, exiting!" >> $logfile + exit 1 + fi + # Update mount to fstab for automount + echo "$nfs_server_with_mount_path $nfs_client_mount_path nfs rw,sec=sys,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,_netdev 0 0 " >> /etc/fstab + for dir in conf work das_staging_area; do + rm -rf "${LSF_TOP}/$dir" # this local data can go away + ln -fs "${nfs_client_mount_path}/$dir" "${LSF_TOP}" # we link from shared fs + chown -R lsfadmin:root "${LSF_TOP}" + done +fi +echo "Setting LSF share is completed." >> $logfile + +# Setup Custom file shares +echo "Setting custom file shares." >> $logfile +# Setup file share +if [ -n "${custom_file_shares}" ]; then + echo "Custom file share ${custom_file_shares} found" >> $logfile + file_share_array=(${custom_file_shares}) + mount_path_array=(${custom_mount_paths}) + length=${#file_share_array[@]} + for (( i=0; i> $logfile + # Verify mount + if mount | grep "${file_share_array[$i]}"; then + echo "Mount found" >> $logfile + else + echo "No mount found" >> $logfile + rm -rf "${mount_path_array[$i]}" + fi + # Update permission to 777 for all users to access + chmod 777 "${mount_path_array[$i]}" + # Update mount to fstab for automount + echo "${file_share_array[$i]} ${mount_path_array[$i]} nfs rw,sec=sys,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,_netdev 0 0 " >> /etc/fstab + done +fi +echo "Setting custom file shares is completed." >> $logfile + +# Setup LSF environment variables +LSF_TOP="/opt/ibm/lsf_worker" +LSF_TOP_VERSION=10.1 +LSF_CONF=$LSF_TOP/conf +LSF_CONF_FILE=$LSF_CONF/lsf.conf +LSF_HOSTS_FILE=$LSF_CONF/hosts +. $LSF_CONF/profile.lsf # WARNING: this may unset LSF_TOP and LSF_VERSION +echo "Logging env variables" >> $logfile +env | sort >> $logfile + +# Defining ncpus based on hyper-threading +if [ "$hyperthreading" == true ]; then + ego_define_ncpus="threads" +else + ego_define_ncpus="cores" + cat << 'EOT' > /root/lsf_hyperthreading +#!/bin/sh +for vcpu in $(cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list | cut -s -d- -f2 | cut -d- -f2 | uniq); do + echo "0" > "/sys/devices/system/cpu/cpu"$vcpu"/online" +done +EOT + chmod 755 /root/lsf_hyperthreading + command="/root/lsf_hyperthreading" + sh $command && (crontab -l 2>/dev/null; echo "@reboot $command") | crontab - +fi +echo "EGO_DEFINE_NCPUS=${ego_define_ncpus}" >> $LSF_CONF_FILE + +# Update lsf configuration +echo 'LSB_MC_DISABLE_HOST_LOOKUP=Y' >> $LSF_CONF_FILE +echo "LSF_RSH=\"ssh -o 'PasswordAuthentication no' -o 'StrictHostKeyChecking no'\"" >> $LSF_CONF_FILE +sed -i "s/LSF_SERVER_HOSTS=.*/LSF_SERVER_HOSTS=\"$ManagementHostNames\"/g" $LSF_CONF_FILE + +# TODO: Understand usage +# Support rc_account resource to enable RC_ACCOUNT policy +if [ -n "${rc_account}" ]; then + sed -i "s/\(LSF_LOCAL_RESOURCES=.*\)\"/\1 [resourcemap ${rc_account}*rc_account]\"/" $LSF_CONF_FILE + echo "Update LSF_LOCAL_RESOURCES lsf.conf successfully, add [resourcemap ${rc_account}*rc_account]" >> $logfile +fi +# Support for multiprofiles for the Job submission +if [ -n "${family}" ]; then + sed -i "s/\(LSF_LOCAL_RESOURCES=.*\)\"/\1 [resourcemap ${family}*family]\"/" $LSF_CONF_FILE + echo "update LSF_LOCAL_RESOURCES lsf.conf successfully, add [resourcemap ${pricing}*family]" >> $logfile +fi +# Add additional local resources if needed +instance_id=$(dmidecode | grep Family | cut -d ' ' -f 2 |head -1) +if [ -n "$instance_id" ]; then + sed -i "s/\(LSF_LOCAL_RESOURCES=.*\)\"/\1 [resourcemap $instance_id*instanceID]\"/" $LSF_CONF_FILE + echo "Update LSF_LOCAL_RESOURCES in $LSF_CONF_FILE successfully, add [resourcemap ${instance_id}*instanceID]" >> $logfile +else + echo "Can not get instance ID" >> $logfile +fi + +#Update LSF Tuning on dynamic hosts +LSF_TUNABLES="etc/sysctl.conf" +echo 'vm.overcommit_memory=1' >> $LSF_TUNABLES +echo 'net.core.rmem_max=26214400' >> $LSF_TUNABLES +echo 'net.core.rmem_default=26214400' >> $LSF_TUNABLES +echo 'net.core.wmem_max=26214400' >> $LSF_TUNABLES +echo 'net.core.wmem_default=26214400' >> $LSF_TUNABLES +echo 'net.ipv4.tcp_fin_timeout = 5' >> $LSF_TUNABLES +echo 'net.core.somaxconn = 8000' >> $LSF_TUNABLES +sudo sysctl -p $LSF_TUNABLES + +# Setup ssh +lsfadmin_home_dir="/home/lsfadmin" +lsfadmin_ssh_dir="${lsfadmin_home_dir}/.ssh" +mkdir -p $lsfadmin_ssh_dir +if grep -q "NAME=\"Red Hat Enterprise Linux\"" /etc/os-release; then + cp /home/vpcuser/.ssh/authorized_keys $lsfadmin_ssh_dir/authorized_keys +else + cp /home/ubuntu/.ssh/authorized_keys "${lsfadmin_ssh_dir}/authorized_keys" + sudo cp /home/ubuntu/.profile $lsfadmin_home_dir +fi +echo "${lsf_public_key}" >> $lsfadmin_ssh_dir/authorized_keys +echo "StrictHostKeyChecking no" >> $lsfadmin_ssh_dir/config +chmod 600 $lsfadmin_ssh_dir/authorized_keys +chmod 700 $lsfadmin_ssh_dir +chown -R lsfadmin:lsfadmin $lsfadmin_ssh_dir +echo "SSH key setup for lsfadmin user is completed" >> $logfile +echo "source ${LSF_CONF}/profile.lsf" >> $lsfadmin_home_dir/.bashrc +echo "source /opt/intel/oneapi/setvars.sh >> /dev/null" >> $lsfadmin_home_dir/.bashrc +echo "Setting up LSF env variables for lasfadmin user is completed" >> $logfile + +# Create lsf.sudoers file to support single lsfstartup and lsfrestart command from management node +echo 'LSF_STARTUP_USERS="lsfadmin"' | sudo tee -a /etc/lsf1.sudoers +echo "LSF_STARTUP_PATH=$LSF_TOP_VERSION/linux3.10-glibc2.17-x86_64/etc/" | sudo tee -a /etc/lsf.sudoers +chmod 600 /etc/lsf.sudoers +ls -l /etc/lsf.sudoers + +# Change LSF_CONF= value in lsf_daemons +cd /opt/ibm/lsf_worker/10.1/linux3.10-glibc2.17-x86_64/etc/ +sed -i "s|/opt/ibm/lsf/|/opt/ibm/lsf_worker/|g" lsf_daemons +cd - + +sudo ${LSF_TOP}/10.1/install/hostsetup --top="${LSF_TOP}" --setuid ### WARNING: LSF_TOP may be unset here +echo "Added LSF administrators to start LSF daemons" >> $logfile + +# Install LSF as a service and start up +/opt/ibm/lsf_worker/10.1/install/hostsetup --top="/opt/ibm/lsf_worker" --boot="y" --start="y" --dynamic 2>&1 >> $logfile +cat /opt/ibm/lsf/conf/hosts >> /etc/hosts + +# Setting up the LDAP configuration +if [ "$enable_ldap" = "true" ]; then + + # Detect the operating system + if grep -q "NAME=\"Red Hat Enterprise Linux\"" /etc/os-release; then + + # Detect RHEL version + rhel_version=$(grep -oE 'release [0-9]+' /etc/redhat-release | awk '{print $2}') + + if [ "$rhel_version" == "8" ]; then + echo "Detected RHEL 8. Proceeding with LDAP client configuration...." >> $logfile + + # Allow Password authentication + sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config + systemctl restart sshd + + # Configure LDAP authentication + authconfig --enableldap --enableldapauth --ldapserver=ldap://"${ldap_server_ip}" --ldapbasedn="dc=${base_dn%%.*},dc=${base_dn#*.}" --enablemkhomedir --update + + # Check the exit status of the authconfig command + if [ $? -eq 0 ]; then + echo "LDAP Authentication enabled successfully." >> $logfile + else + echo "Failed to enable LDAP and LDAP Authentication." >> $logfile + exit 1 + fi + + # Update LDAP Client configurations in nsswitch.conf + sed -i -e 's/^passwd:.*$/passwd: files ldap/' -e 's/^shadow:.*$/shadow: files ldap/' -e 's/^group:.*$/group: files ldap/' /etc/nsswitch.conf # pragma: allowlist secret + + # Update PAM configuration files + sed -i -e '/^auth/d' /etc/pam.d/password-auth + sed -i -e '/^auth/d' /etc/pam.d/system-auth + + auth_line="\nauth required pam_env.so\nauth sufficient pam_unix.so nullok try_first_pass\nauth requisite pam_succeed_if.so uid >= 1000 quiet_success\nauth sufficient pam_ldap.so use_first_pass\nauth required pam_deny.so" + + echo -e "$auth_line" | tee -a /etc/pam.d/password-auth /etc/pam.d/system-auth + + # Copy 'password-auth' settings to 'sshd' + cat /etc/pam.d/password-auth > /etc/pam.d/sshd + + # Configure nslcd + cat < /etc/nslcd.conf +uid nslcd +gid ldap +uri ldap://${ldap_server_ip}/ +base dc=${base_dn%%.*},dc=${base_dn#*.} +EOF + + # Restart nslcd and nscd service + systemctl restart nslcd + systemctl restart nscd + + # Validate the LDAP configuration + if ldapsearch -x -H ldap://"${ldap_server_ip}"/ -b "dc=${base_dn%%.*},dc=${base_dn#*.}" > /dev/null; then + echo "LDAP configuration completed successfully !!" >> $logfile + else + echo "LDAP configuration failed !!" >> $logfile + exit 1 + fi + + # Make LSF commands available for every user. + echo ". ${LSF_CONF}/profile.lsf" >> /etc/bashrc + source /etc/bashrc + else + echo "This script is designed for RHEL 8. Detected RHEL version: $rhel_version. Exiting." >> $logfile + exit 1 + fi + + elif grep -q "NAME=\"Ubuntu\"" /etc/os-release; then + + echo "Detected as Ubuntu. Proceeding with LDAP client configuration..." >> $logfile + + # Update package repositories + sudo apt-get update -y + + # Update SSH configuration to allow password authentication + sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config + sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloudimg-settings.conf + sudo systemctl restart ssh + + # Create preseed file for LDAP configuration + cat > debconf-ldap-preseed.txt <> /etc/bash.bashrc + source /etc/bash.bashrc + + # Restart nslcd and nscd service + systemctl restart nslcd + systemctl restart nscd + + # Enable nslcd and nscd service + systemctl enable nslcd + systemctl enable nscd + + # Validate the LDAP client service status + if sudo systemctl is-active --quiet nscd; then + echo "LDAP client configuration completed successfully !!" >> $logfile + else + echo "LDAP client configuration failed. nscd service is not running." >> $logfile + exit 1 + fi + else + echo -e "debconf-ldap-preseed.txt Not found. Skipping LDAP client configuration." >> $logfile + fi + else + echo "This script is designed for Ubuntu 22, and installation is not supported. Exiting." >> $logfile + fi +fi + +#update lsf client ip address to LSF_HOSTS_FILE +echo "$login_ip_address $login_hostname" >> $LSF_HOSTS_FILE +# Startup lsf daemons +systemctl status lsfd >> "$logfile" + +# Setting up the Metrics Agent + +if [ "$cloud_monitoring_access_key" != "" ] && [ "$cloud_monitoring_ingestion_url" != "" ]; then + + SYSDIG_CONFIG_FILE="/opt/draios/etc/dragent.yaml" + + #packages installation + echo "Writing sysdig config file" >> "$logfile" + + #sysdig config file + echo "Setting customerid access key" >> "$logfile" + sed -i "s/==ACCESSKEY==/$cloud_monitoring_access_key/g" $SYSDIG_CONFIG_FILE + sed -i "s/==COLLECTOR==/$cloud_monitoring_ingestion_url/g" $SYSDIG_CONFIG_FILE + echo "tags: type:compute,lsf:true" >> $SYSDIG_CONFIG_FILE +else + echo "Skipping metrics agent configuration due to missing parameters" >> "$logfile" +fi + +if [ "$observability_monitoring_on_compute_nodes_enable" = true ]; then + + echo "Restarting sysdig agent" >> "$logfile" + systemctl enable dragent + systemctl restart dragent + else + echo "Metrics agent start skipped since monitoring provisioning is not enabled" >> "$logfile" +fi + +echo "END $(date '+%Y-%m-%d %H:%M:%S')" >> "$logfile" diff --git a/modules/landing_zone_vsi/configuration_steps/configure_management_vsi.sh b/modules/landing_zone_vsi/configuration_steps/configure_management_vsi.sh new file mode 100644 index 00000000..fb2a1b0a --- /dev/null +++ b/modules/landing_zone_vsi/configuration_steps/configure_management_vsi.sh @@ -0,0 +1,1056 @@ +#!/bin/bash +# shellcheck disable=all + +################################################### +# Copyright (C) IBM Corp. 2023 All Rights Reserved. +# Licensed under the Apache License v2.0 +################################################### + +#set -x # uncomment with care: this can log too much, including credentials + +# Setup logs +logfile="/tmp/configure_management.log" +exec > >(stdbuf -oL awk '{print strftime("%Y-%m-%dT%H:%M:%S") " " $0}' | tee "$logfile") 2>&1 +# automatic logging of stdout and stderr, including timestamps; no need to redirect explicitly + +echo "START $(date '+%Y-%m-%d %H:%M:%S')" + +source management_values + +# Local variable declaration +default_cluster_name="HPCCluster" +nfs_server_with_mount_path=${mount_path} +enable_ldap="${enable_ldap}" +ldap_server_ip="${ldap_server_ip}" +base_dn="${ldap_basedns}" + +this_hostname="$(hostname)" +mgmt_hostname_primary="$management_hostname" +mgmt_hostnames="${management_hostname},${management_cand_hostnames}" +mgmt_hostnames="${mgmt_hostnames//,/ }" # replace commas with spaces +mgmt_hostnames="${mgmt_hostnames# }" # remove an initial space +mgmt_hostnames="${mgmt_hostnames% }" # remove a final space + +LSF_TOP="/opt/ibm/lsf" +LSF_CONF="$LSF_TOP/conf" +LSF_SSH="$LSF_TOP/ssh" +LSF_CONF_FILE="$LSF_CONF/lsf.conf" +LSF_HOSTS_FILE="$LSF_CONF/hosts" +LSF_EGO_CONF_FILE="$LSF_CONF/ego/$cluster_name/kernel/ego.conf" +LSF_LSBATCH_CONF="$LSF_CONF/lsbatch/$cluster_name/configdir" +LSF_RC_CONF="$LSF_CONF/resource_connector" +LSF_RC_IC_CONF="$LSF_RC_CONF/ibmcloudgen2/conf" +LSF_DM_STAGING_AREA="$LSF_TOP/das_staging_area" +# Should be changed in the upcoming days. Since the LSF core team have mismatched the path and we have approached to make the changes. +LSF_RC_IBMCLOUDHPC_CONF="$LSF_RC_CONF/ibmcloudhpc/conf" +LSF_TOP_VERSION="$LSF_TOP/10.1" + +# Useful variables that reference the main GUI and PERF Manager folders. +LSF_SUITE_TOP="/opt/ibm/lsfsuite" +LSF_SUITE_GUI="${LSF_SUITE_TOP}/ext/gui" +LSF_SUITE_GUI_CONF="${LSF_SUITE_GUI}/conf" +LSF_SUITE_PERF="${LSF_SUITE_TOP}/ext/perf" +LSF_SUITE_PERF_CONF="${LSF_SUITE_PERF}/conf" +LSF_SUITE_PERF_BIN="${LSF_SUITE_PERF}/1.2/bin" + +# important: is this a primary or secondary management node? +if [ "$this_hostname" == "$mgmt_hostname_primary" ]; then + on_primary="true" +else + on_primary="false" +fi +echo "is this node primary: $on_primary" + +echo "umask=$(umask)" +umask 022 # since being run with 077 can cause issues +echo "umask=$(umask)" + +db_certificate_file="${LSF_SUITE_GUI_CONF}/cert.pem" + +# Function that dump the ICD certificate in the $db_certificate_file +create_certificate() { + # Dump the CA certificate in the ${db_certificate_file} file and set permissions + echo "${db_certificate}" | base64 -d > "${db_certificate_file}" + chown lsfadmin:lsfadmin "${db_certificate_file}" + chmod 644 "${db_certificate_file}" + +} + +# Function for creating PAC database in the IBM Cloud Database (ICD) service when High Availability is enabled. +# It is invoked when ${enable_app_center} and ${app_center_high_availability} are both true. +create_appcenter_database() { + # Required SQL commands to create the PAC database in the IBM Cloud Database (ICD) instance. + local create_db_command="CREATE DATABASE ${db_name} default character set utf8 default collate utf8_bin;" + local commands=( + "CREATE USER ${db_user}@'%' IDENTIFIED WITH mysql_native_password BY '${db_password}';" + "CREATE USER ${db_user}@'localhost' IDENTIFIED WITH mysql_native_password BY '${db_password}';" + "GRANT ALL ON ${db_name}.* TO ${db_user}@'%';" + "GRANT ALL ON ${db_name}.* TO ${db_user}@'localhost';" + "source ${LSF_SUITE_PERF}/ego/1.2/DBschema/MySQL/egodata.sql;" + "source ${LSF_SUITE_PERF}/lsf/10.0/DBschema/MySQL/lsfdata.sql;" + "source ${LSF_SUITE_PERF}/lsf/10.0/DBschema/MySQL/lsf_sql.sql;" + "source ${LSF_SUITE_GUI}/DBschema/MySQL/create_schema.sql;" + "source ${LSF_SUITE_GUI}/DBschema/MySQL/create_pac_schema.sql;" + "source ${LSF_SUITE_GUI}/DBschema/MySQL/init.sql;" + ) + + # On ICD you cannot change system variables so we need to comment 736 line in $LSF_SUITE_GUI/DBschema/MySQL/create_pac_schema.sql + sed -i "s|SET GLOBAL group_concat_max_len = 1000000;|/* SET GLOBAL group_concat_max_len = 1000000; */|" $LSF_SUITE_GUI/DBschema/MySQL/create_pac_schema.sql + # Create the PAC database + echo "${create_db_command}" | MYSQL_PWD="${db_adminpassword}" mysql --host="${db_hostname}" --port="${db_port}" --user="${db_adminuser}" --ssl-ca="${db_certificate_file}" ibmclouddb + # Create the pacuser, grant him all the required privileges, then create the schema and tables + for command in "${commands[@]}"; do + echo "${command}" | MYSQL_PWD="${db_adminpassword}" mysql --host="${db_hostname}" --port="${db_port}" --user="${db_adminuser}" --ssl-ca="${db_certificate_file}" pac + done +} + +# Configures the GUI JDBC datasource file ${LSF_SUITE_PERF_CONF}/datasource.xml +# to reference the IBM Cloud Database (ICD) instance. If ${enable_app_center} and +# ${app_center_high_availability} are both true, updates the connection string to +# point to the remote database service instead of the local MySQL server. +configure_icd_datasource() { + local default_connection_string="jdbc:mariadb://localhost:3306/pac?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT" + local icd_connection_string="jdbc:mariadb://${db_hostname}:${db_port}/${db_name}?useUnicode=true\&characterEncoding=UTF-8\&serverTimezone=GMT\&requireSSL=true\&useSSL=true\&serverSslCert=${db_certificate_file}" + + # Change the connection string to use ICD + sed -i "s!Connection=\"${default_connection_string}\"!Connection=\"${icd_connection_string}\"!" ${LSF_SUITE_PERF_CONF}/datasource.xml + # Change the Cipher algorithm to AES128 in the Datasource definition + sed -i "s|Cipher=\".*\"|Cipher=\"aes128\"|" ${LSF_SUITE_PERF_CONF}/datasource.xml + # Encrypt the Database user and password with AES128 Cipher. The encryptTool.sh script requires the setting of the JAVA_HOME + db_user_aes128=$(source ${LSF_SUITE_TOP}/ext/profile.platform; ${LSF_SUITE_PERF_BIN}/encryptTool.sh "${db_user}") + db_password_aes128=$(source ${LSF_SUITE_TOP}/ext/profile.platform; ${LSF_SUITE_PERF_BIN}/encryptTool.sh "${db_password}") + # Change the username password in the Datasource definition + sed -i "s|UserName=\".*\"|UserName=\"${db_user_aes128}\"|" ${LSF_SUITE_PERF_CONF}/datasource.xml + sed -i "s|Password=\".*\"|Password=\"${db_password_aes128}\"|" ${LSF_SUITE_PERF_CONF}/datasource.xml +} + +########### LSFSETUP-BEGIN ################################################################ +###################################### search LSFSETUP-END to skip this part ############## + +# Setup LSF + +if [ "$on_primary" == "true" ]; then + + echo "LSF configuration begin" + + mkdir -p $LSF_RC_IBMCLOUDHPC_CONF + chown -R lsfadmin:root $LSF_RC_IBMCLOUDHPC_CONF + + echo "Setting up LSF" + + # 0. Update LSF configuration with new cluster name if cluster_name is not default + if [ "$default_cluster_name" != "$cluster_name" ]; then + echo "New cluster name $cluster_name has been identified. Upgrading the cluster configurations accordingly." + grep -rli "$default_cluster_name" $LSF_CONF/* | xargs sed -i "s/$default_cluster_name/$cluster_name/g" + # Below directory in work has cluster_name twice in path and was resulting in a indefinite loop scenario. So, this directory has to be handled separately + mv $LSF_TOP/work/$default_cluster_name/live_confdir/lsbatch/$default_cluster_name $LSF_TOP/work/"$cluster_name"/live_confdir/lsbatch/"$cluster_name" + for file in $(find $LSF_TOP -name "*$default_cluster_name*"); do mv "$file" $(echo "$file"| sed -r "s/$default_cluster_name/$cluster_name/g"); done + fi + + # 1. setting up lsf configuration + cat <> $LSF_CONF_FILE +LSB_RC_EXTERNAL_HOST_IDLE_TIME=10 +LSF_DYNAMIC_HOST_TIMEOUT="EXPIRY[10m] THRESHOLD[250] INTERVAL[60m]" +LSB_RC_EXTERNAL_HOST_FLAG="icgen2host cloudhpchost" +LSB_RC_UPDATE_INTERVAL=15 +LSB_RC_MAX_NEWDEMAND=50 +LSF_UDP_TO_TCP_THRESHOLD=9000 +LSF_CALL_LIM_WITH_TCP=N +LSF_ANNOUNCE_MASTER_TCP_WAITTIME=600 +LSF_CLOUD_UI=Y +LSF_RSH="ssh -o 'PasswordAuthentication no' -o 'StrictHostKeyChecking no'" +EOT + sed -i "s/LSF_MASTER_LIST=.*/LSF_MASTER_LIST=\"${mgmt_hostnames}\"/g" $LSF_CONF_FILE + + if [ "$hyperthreading" == true ]; then + ego_define_ncpus="threads" + else + ego_define_ncpus="cores" + + cat << 'EOT' > /root/lsf_hyperthreading +#!/bin/sh +for vcpu in $(cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list | cut -s -d- -f2 | cut -d- -f2 | uniq); do + echo "0" > "/sys/devices/system/cpu/cpu"$vcpu"/online" +done +EOT + chmod 755 /root/lsf_hyperthreading + command="/root/lsf_hyperthreading" + sh $command && (crontab -l 2>/dev/null; echo "@reboot $command") | crontab - + fi + echo "EGO_DEFINE_NCPUS=${ego_define_ncpus}" >> $LSF_CONF_FILE + + # 2. setting up lsf.shared + sed -i "s/^# icgen2host/ icgen2host/g" $LSF_CONF/lsf.shared + sed -i '/^End Resource/i cloudhpchost Boolean () () (hpc hosts from IBM Cloud HPC pool)' $LSF_CONF/lsf.shared + sed -i '/^End Resource/i family String () () (account name for the external hosts)' $LSF_CONF/lsf.shared + + # 3. setting up lsb.module + sed -i "s/^#schmod_demand/schmod_demand/g" "$LSF_LSBATCH_CONF/lsb.modules" + + # 4. setting up lsb.queue + sed -i '/^Begin Queue$/,/^End Queue$/{/QUEUE_NAME/{N;s/\(QUEUE_NAME\s*=[^\n]*\)\n/\1\nRC_HOSTS = all\n/}}' "$LSF_LSBATCH_CONF/lsb.queues" + cat <> "$LSF_LSBATCH_CONF/lsb.queues" +Begin Queue +QUEUE_NAME=das_q +DATA_TRANSFER=Y +RC_HOSTS=all +HOSTS=all +RES_REQ=type==any +End Queue +EOT + + # 5. setting up lsb.hosts + for hostname in $mgmt_hostnames; do + sed -i "/^default !.*/a $hostname 0 () () () () () (Y)" "$LSF_LSBATCH_CONF/lsb.hosts" + done + + # 6. setting up lsf.cluster."$cluster_name" + sed -i "s/^lsfservers/#lsfservers/g" "$LSF_CONF/lsf.cluster.$cluster_name" + sed -i 's/LSF_HOST_ADDR_RANGE=\*.\*.\*.\*/LSF_HOST_ADDR_RANGE=10.*.*.*/' "$LSF_CONF/lsf.cluster.$cluster_name" + for hostname in $mgmt_hostnames; do + sed -i "/^#lsfservers.*/a $hostname ! ! 1 (mg)" "$LSF_CONF/lsf.cluster.$cluster_name" + done + + # Updating the value of login node as Intel for lsfserver to update cluster file name + sed -i "/^#lsfservers.*/a $login_hostname Intel_E5 X86_64 0 ()" "$LSF_CONF/lsf.cluster.$cluster_name" + echo "LSF_SERVER_HOSTS=\"$mgmt_hostnames\"" >> $LSF_CONF_FILE + + # Update ego.conf + sed -i "s/EGO_MASTER_LIST=.*/EGO_MASTER_LIST=\"${mgmt_hostnames}\"/g" "$LSF_EGO_CONF_FILE" + # 0.5 Update lsfservers with newly added lsf management nodes + grep -rli 'lsfservers' $LSF_CONF/*|xargs sed -i "s/lsfservers/${this_hostname}/g" + + # Setup LSF resource connector + echo "Setting up LSF resource connector" + + # 1. Create hostProviders.json + if [ "$regionName" = "eu-de" ] || [ "$regionName" = "us-east" ] || [ "$regionName" = "us-south" ] ; then + cat < "$LSF_RC_CONF"/hostProviders.json +{ + "providers":[ + { + "name": "ibmcloudhpc", + "type": "ibmcloudhpcProv", + "confPath": "resource_connector/ibmcloudhpc", + "scriptPath": "resource_connector/ibmcloudhpc" + } + ] +} +EOT + else + cat < "$LSF_RC_CONF"/hostProviders.json +{ + "providers":[ + { + "name": "ibmcloudgen2", + "type": "ibmcloudgen2Prov", + "confPath": "resource_connector/ibmcloudgen2", + "scriptPath": "resource_connector/ibmcloudgen2" + } + ] +} +EOT + fi + + # 2. Create ibmcloudgen2_config.json + cat < "$LSF_RC_IC_CONF"/ibmcloudgen2_config.json +{ + "IBMCLOUDGEN2_KEY_FILE": "${LSF_RC_IC_CONF}/credentials", + "IBMCLOUDGEN2_PROVISION_FILE": "${LSF_RC_IC_CONF}/user_data.sh", + "IBMCLOUDGEN2_MACHINE_PREFIX": "${cluster_prefix}", + "LogLevel": "INFO", + "ApiEndPoints": { + "eu-gb": "https://eu-gb.iaas.cloud.ibm.com/v1", + "au-syd": "https://au-syd.iaas.cloud.ibm.com/v1", + "ca-tor": "https://ca-tor.iaas.cloud.ibm.com/v1", + "jp-osa": "https://jp-osa.iaas.cloud.ibm.com/v1", + "jp-tok": "https://jp-tok.iaas.cloud.ibm.com/v1", + "br-sao": "https://br-sao.iaas.cloud.ibm.com/v1" + } +} +EOT + + # 3. Create ibmcloudhpc_config.json + cat < "$LSF_RC_IBMCLOUDHPC_CONF"/ibmcloudhpc_config.json +{ + "IBMCLOUDHPC_KEY_FILE": "${LSF_RC_IBMCLOUDHPC_CONF}/credentials", + "IBMCLOUDHPC_PROVISION_FILE": "${LSF_RC_IBMCLOUDHPC_CONF}/user_data.sh", + "IBMCLOUDHPC_MACHINE_PREFIX": "${cluster_prefix}", + "LogLevel": "INFO", + "CONTRACT_ID": "${contract_id}", + "CLUSTER_ID": "${cluster_name}", + "PROJECT_ID": "${ce_project_guid}", + "ApiEndPoints": { + "us-east": "${api_endpoint_us_east}", + "eu-de": "${api_endpoint_eu_de}", + "us-south": "${api_endpoint_us_south}" + } +} +EOT + + # 4. Create credentials for ibmcloudgen2 + cat < "$LSF_RC_IC_CONF"/credentials +# BEGIN ANSIBLE MANAGED BLOCK +VPC_URL=http://vpc.cloud.ibm.com/v1 +VPC_AUTH_TYPE=iam +VPC_APIKEY=$VPC_APIKEY_VALUE +RESOURCE_RECORDS_URL=https://api.dns-svcs.cloud.ibm.com/v1 +RESOURCE_RECORDS_AUTH_TYPE=iam +RESOURCE_RECORDS_APIKEY=$VPC_APIKEY_VALUE +EOT + + # 5. Create credentials for ibmcloudhpc + cat < "$LSF_RC_IBMCLOUDHPC_CONF"/credentials +# BEGIN ANSIBLE MANAGED BLOCK +CLOUD_HPC_URL=http://vpc.cloud.ibm.com/v1 +CLOUD_HPC_AUTH_TYPE=iam +CLOUD_HPC_AUTH_URL=https://iam.cloud.ibm.com +CLOUD_HPC_APIKEY=$VPC_APIKEY_VALUE +RESOURCE_RECORDS_URL=https://api.dns-svcs.cloud.ibm.com/v1 +RESOURCE_RECORDS_AUTH_TYPE=iam +RESOURCE_RECORDS_APIKEY=$VPC_APIKEY_VALUE +# END ANSIBLE MANAGED BLOCK +EOT + + # 6. Create ibmcloudgen2_templates.json + ibmcloudgen2_templates="$LSF_RC_IC_CONF/ibmcloudgen2_templates.json" + # Incrementally build a json string + json_string="" + + tab="$(cat < "$ibmcloudgen2_templates" + echo "JSON templates are created and updated on ibmcloudgen2_templates.json" + +# 7. Create resource template for ibmcloudhpc templates +# Define the output JSON file path +ibmcloudhpc_templates="$LSF_RC_IBMCLOUDHPC_CONF/ibmcloudhpc_templates.json" + +# Initialize an empty JSON string +json_string="" + +# Loop through the specified regions +for region in "eu-de" "us-east" "us-south"; do + if [ "$region" = "$regionName" ]; then + # Loop through the core counts + for i in 2 4 8 16 32 48 64 96 128 176; do + ncores=$((i / 2)) + if [ "$region" = "eu-de" ] || [ "$region" = "us-east" ]; then + family="mx2" + maxmem_mx2=$((ncores * 16 * 1024)) + mem_mx2=$((maxmem_mx2 * 9 / 10)) + elif [ "$region" = "us-south" ]; then + family="mx2,mx3d" # Include both "mx2" and "mx3d" families + maxmem_mx2=$((ncores * 16 * 1024)) + mem_mx2=$((maxmem_mx2 * 9 / 10)) + maxmem_mx3d=$((ncores * 20 * 1024)) + mem_mx3d=$((maxmem_mx3d * 9 / 10)) + fi + + vpcus=$i + + if $hyperthreading; then + ncpus=$vpcus + else + ncpus=$ncores + fi + + if [ "${imageID:0:4}" == "crn:" ]; then + imagetype="imageCrn" + else + imagetype="imageId" + fi + + # Split the family string into an array and iterate over it + IFS=',' read -ra families <<< "$family" + for fam in "${families[@]}"; do + templateId="Template-${cluster_prefix}-$((1000+i))-$fam" # Add family to templateId + if [ "$fam" = "mx2" ]; then + maxmem_val="$maxmem_mx2" # Use mx2 specific maxmem value + mem_val="$mem_mx2" # Use mx2 specific mem value + priority=10 # Priority for mx2 + elif [ "$fam" = "mx3d" ]; then + maxmem_val="$maxmem_mx3d" # Use mx3d specific maxmem value + mem_val="$mem_mx3d" # Use mx3d specific mem value + priority=20 # Priority for mx3d in us-south + fi + + # Construct JSON object and append it to the JSON string + json_string+=$(cat < "$ibmcloudhpc_templates" +echo "JSON templates are created and updated in ibmcloudhpc_templates.json" + +# 8. Define the directory to store fleet configuration files +fleet_config_dir="$LSF_RC_IBMCLOUDHPC_CONF" +# Loop through regions +for region in "eu-de" "us-east" "us-south"; do + # Define the fleet configuration family based on the region + if [ "$regionName" = "us-south" ]; then + families=("mx2" "mx3d") + else + families=("mx2") + fi + + # Loop through families + for family in "${families[@]}"; do + # Create fleet configuration file for the region and family + cat < "${fleet_config_dir}/ibmcloudhpc_fleetconfig_${family}.json" +{ + "fleet_request": { + "availability_policy": { + "host_failure": "restart" + }, + "host_name": { + "prefix": "${cluster_prefix}", + "domain": "${dns_domain}" + }, + "instance_selection": { + "type": "automatic", + "optimization": "minimum_price" + }, + "boot_volume_attachment": { + "encryption_key": { + "crn": "${bootdrive_crn}" + } + }, + "zones": [ + { + "name": "${zoneName}", + "primary_network_interface": { + "name": "eth0", + "subnet": { + "crn": "${subnetID}" + }, + "security_groups": [ + { + "id": "${securityGroupID}" + } + ] + } + } + ], + "profile_requirement": { + "families": [ + { + "name": "${family}", + "rank": 1, + "profiles": [] + } + ] + } + } +} +EOT + done +done + +# Set permissions for fleet configuration files +chown lsfadmin:root "${fleet_config_dir}/ibmcloudhpc_fleetconfig_"* +chmod 644 "${fleet_config_dir}/ibmcloudhpc_fleetconfig_"* +echo "Fleet configuration files created and updated." + + # 9. create user_data.json for compute nodes + ( + cat < "$LSF_RC_IBMCLOUDHPC_CONF"/user_data.sh + + # TODO: Setting up License Scheduler configurations + # No changes has been advised to be automated + + # 10. Copy user_data.sh from ibmcloudhpc to ibmcloudgen2 + cp $LSF_RC_IBMCLOUDHPC_CONF/user_data.sh $LSF_RC_IC_CONF/user_data.sh + + # Setting up Data Manager configurations + mkdir -p "${LSF_DM_STAGING_AREA}" + chown -R lsfadmin:root "${LSF_DM_STAGING_AREA}" + cat <> "${LSF_CONF}"/lsf.datamanager."${cluster_name}" +Begin Parameters +ADMINS = lsfadmin +STAGING_AREA = "${LSF_DM_STAGING_AREA}" +End Parameters +EOT + + # Uncomment the below line to enable Datamanager + cat <> $LSF_CONF_FILE +#LSF_DATA_HOSTS=${this_hostname} +# LSF_DATA_PORT=1729 +EOT + + echo "LSF configuration end" + ##### LSFSETUP-END ##### + + # Finally ensure ownership for conf files + chown -R lsfadmin:root $LSF_RC_IBMCLOUDHPC_CONF + +else + + # nothing to do on candidate nodes + echo "LSF configuration not to be done on secondary nodes, skipping" + +fi + +########### LSFSETUP-END ################################################################## +########################################################################################### + +echo "Setting LSF share" +# Setup file share +if [ -n "${nfs_server_with_mount_path}" ]; then + echo "File share ${nfs_server_with_mount_path} found" + # Create a data directory for sharing HPC workload data ### is this used? + mkdir -p "${LSF_TOP}/data" + nfs_client_mount_path="/mnt/lsf" + rm -rf "${nfs_client_mount_path}" + mkdir -p "${nfs_client_mount_path}" + # Mount LSF TOP + mount -t nfs -o sec=sys "$nfs_server_with_mount_path" "$nfs_client_mount_path" + # Verify mount + if mount | grep "$nfs_client_mount_path"; then + echo "Mount found" + else + echo "No mount found, exiting!" + exit 1 + fi + # Update mount to fstab for automount + echo "$nfs_server_with_mount_path $nfs_client_mount_path nfs rw,sec=sys,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,_netdev 0 0 " >> /etc/fstab + + # Move stuff to shared fs + for dir in conf work das_staging_area; do + if [ "$on_primary" == "true" ]; then + rm -rf "${nfs_client_mount_path}/$dir" # avoid old data already in shared fs + mv "${LSF_TOP}/$dir" "${nfs_client_mount_path}" # this local data goes to shared fs + else + rm -rf "${LSF_TOP}/$dir" # this local data can go away + fi + ln -fs "${nfs_client_mount_path}/$dir" "${LSF_TOP}" # locally link to shared fs + chown -R lsfadmin:root "${LSF_TOP}" + done + + # Sharing the lsfsuite..conf folder + if [ "$on_primary" == "true" ]; then + rm -rf "${nfs_client_mount_path}/gui-conf" + mv "${LSF_SUITE_GUI_CONF}" "${nfs_client_mount_path}/gui-conf" + chown -R lsfadmin:root "${nfs_client_mount_path}/gui-conf" + else + rm -rf "${LSF_SUITE_GUI_CONF}" + fi + ln -fs "${nfs_client_mount_path}/gui-conf" "${LSF_SUITE_GUI_CONF}" + chown -R lsfadmin:root "${LSF_SUITE_GUI_CONF}" + + # VNC Sessions + if [ "$on_primary" == "true" ]; then + mkdir -p "${nfs_client_mount_path}/repository-path" + # With this change, LDAP User can able to submit the job from App Center UI. + chmod -R 777 "${nfs_client_mount_path}/repository-path" + chown -R lsfadmin:root "${nfs_client_mount_path}/repository-path" + fi + + # Create folder in shared file system to store logs + mkdir -p "${nfs_client_mount_path}/log/${HOSTNAME}" + chown -R lsfadmin:root "${nfs_client_mount_path}/log" + if [ "$(ls -A ${LSF_TOP}/log)" ]; then + # Move all existing logs to the new folder + mv ${LSF_TOP}/log/* "${nfs_client_mount_path}/log/${HOSTNAME}" + fi + # Remove the original folder and create symlink so the user can still access to default location + rm -rf "${LSF_TOP}/log" + ln -fs "${nfs_client_mount_path}/log/${HOSTNAME}" "${LSF_TOP}/log" + chown -R lsfadmin:root "${LSF_TOP}/log" + + # Create log folder for pac and set proper owner + mkdir -p "${nfs_client_mount_path}/gui-logs" + chown -R lsfadmin:root "${nfs_client_mount_path}/gui-logs" + # Move PAC logs to shared folder + mkdir -p "${nfs_client_mount_path}/gui-logs/${HOSTNAME}" + if [ -d "${LSF_SUITE_GUI}/logs/${HOSTNAME}" ] && [ "$(ls -A ${LSF_SUITE_GUI}/logs/"${HOSTNAME}")" ]; then + mv "${LSF_SUITE_GUI}/logs/${HOSTNAME}" "${nfs_client_mount_path}/gui-logs/${HOSTNAME}" + fi + chown -R lsfadmin:root "${nfs_client_mount_path}/gui-logs/${HOSTNAME}" + ln -fs "${nfs_client_mount_path}/gui-logs/${HOSTNAME}" "${LSF_SUITE_GUI}/logs/${HOSTNAME}" + chown -R lsfadmin:root "${LSF_SUITE_GUI}/logs/${HOSTNAME}" +else + echo "No mount point value found, exiting!" + exit 1 +fi +echo "Setting LSF share is completed." + +# Setup Custom file shares +echo "Setting custom file shares." +# Setup file share +if [ -n "${custom_file_shares}" ]; then + echo "Custom file share ${custom_file_shares} found" + file_share_array=(${custom_file_shares}) + mount_path_array=(${custom_mount_paths}) + length=${#file_share_array[@]} + for (( i=0; i> /etc/fstab + done +fi +echo "Setting custom file shares is completed." + +# Setup ip-host mapping in LSF_HOSTS_FILE +if [ "$on_primary" == "true" ]; then + python3 -c "import ipaddress; print('\n'.join([str(ip) + ' ${cluster_prefix}-' + str(ip).replace('.', '-') for ip in ipaddress.IPv4Network('${rc_cidr_block}')]))" >> "$LSF_HOSTS_FILE" +else + while [ ! -f "$LSF_HOSTS_FILE" ]; do + echo "Waiting for cluster configuration created by management node to be shared." + sleep 5s + done +fi + +# Update the entry to LSF_HOSTS_FILE +if [ "$on_primary" == "true" ]; then + echo "$login_ip $login_hostname" >> $LSF_HOSTS_FILE + for hostname in $mgmt_hostnames; do + # we map hostnames to ips with DNS, even if we have the ips list already + while true; do + echo "querying DNS: $hostname" + ip="$(dig +short "$hostname.${dns_domain}")" + if [ "$ip" != "" ]; then + sed -i "s/^$ip .*/$ip $hostname/g" $LSF_HOSTS_FILE + break + fi + sleep 2 + done + echo "$hostname $ip added to LSF host file" + done +fi + +for hostname in $mgmt_hostnames; do + while ! grep "$hostname" "$LSF_HOSTS_FILE"; do + echo "Waiting for $hostname to be added to LSF host file" + sleep 5 + done + echo "$hostname found in LSF host file" +done +cat $LSF_HOSTS_FILE >> /etc/hosts + +if [ "$enable_app_center" = true ] && [ "${app_center_high_availability}" = true ]; then + # Add entry for VNC scenario + echo "127.0.0.1 pac pac.$dns_domain" >> /etc/hosts +fi + +# Create lsf.sudoers file to support single lsfstartup and lsfrestart command from management node +cat < "/etc/lsf.sudoers" +LSF_STARTUP_USERS="lsfadmin" +LSF_STARTUP_PATH=$LSF_TOP_VERSION/linux3.10-glibc2.17-x86_64/etc/ +EOT +chmod 600 /etc/lsf.sudoers +ls -l /etc/lsf.sudoers + +$LSF_TOP_VERSION/install/hostsetup --top="$LSF_TOP" --setuid +echo "Added LSF administrators to start LSF daemons" + +lsfadmin_home_dir="/home/lsfadmin" +echo "source ${LSF_CONF}/profile.lsf" >> /root/.bashrc +echo "source ${LSF_CONF}/profile.lsf" >> "${lsfadmin_home_dir}"/.bashrc + +if [ "$on_primary" == "true" ]; then + # Configure and start perfmon, used for lsf prometheus monitoring + sed -i '/^End Parameters/i SCHED_METRIC_ENABLE=Y' $LSF_CONF/lsbatch/"$cluster_name"/configdir/lsb.params +fi + +do_app_center=false +if [ "$enable_app_center" = true ] ; then + if [ "$on_primary" == "true" ] || [ "${app_center_high_availability}" = true ] ; then + do_app_center=true + fi +fi +# Setting up the Application Center +if [ "$do_app_center" = true ] ; then + if rpm -q lsf-appcenter; then + echo "Application center packages are found..." + echo "${app_center_gui_pwd}" | passwd --stdin lsfadmin + sed -i '$i\\ALLOW_EVENT_TYPE=JOB_NEW JOB_STATUS JOB_FINISH2 JOB_START JOB_EXECUTE JOB_EXT_MSG JOB_SIGNAL JOB_REQUEUE JOB_MODIFY2 JOB_SWITCH METRIC_LOG' $LSF_CONF/lsbatch/"$cluster_name"/configdir/lsb.params + sed -i 's/NEWJOB_REFRESH=y/NEWJOB_REFRESH=Y/g' $LSF_CONF/lsbatch/"$cluster_name"/configdir/lsb.params + + if [ "${app_center_high_availability}" = true ]; then + create_certificate + configure_icd_datasource + fi + + if [ "$on_primary" == "true" ]; then + # Update the Job directory, needed for VNC Sessions + sed -i 's|/home|/mnt/lsf/repository-path|' "$LSF_SUITE_GUI_CONF/Repository.xml" + if [ "${app_center_high_availability}" = true ]; then + echo "LSF_ADDON_HOSTS=\"${mgmt_hostnames}\"" >> $LSF_CONF/lsf.conf + create_appcenter_database + sed -i "s/NoVNCProxyHost=.*/NoVNCProxyHost=pac.${dns_domain}/g" "$LSF_SUITE_GUI_CONF/pmc.conf" + sed -i "s|.*|${mgmt_hostname_primary}|" $LSF_SUITE_GUI_CONF/pnc-config.xml + sed -i "s|.*|pac.${dns_domain}|" $LSF_SUITE_GUI_CONF/pnc-config.xml + else + echo "LSF_ADDON_HOSTS=$HOSTNAME" >> $LSF_CONF/lsf.conf + sed -i 's/NoVNCProxyHost=.*/NoVNCProxyHost=localhost/g' "$LSF_SUITE_GUI_CONF/pmc.conf" + sed -i "s|.*|${mgmt_hostname_primary}|" $LSF_SUITE_GUI_CONF/pnc-config.xml + sed -i "s|.*|localhost|" $LSF_SUITE_GUI_CONF/pnc-config.xml + fi + fi + + echo "source $LSF_SUITE_TOP/ext/profile.platform" >> ~/.bashrc + echo "source $LSF_SUITE_TOP/ext/profile.platform" >> "${lsfadmin_home_dir}"/.bashrc + rm -rf $LSF_SUITE_GUI/3.0/bin/novnc.pem + fi +else + echo 'Application Center installation skipped...' +fi + +# Startup lsf daemons + +echo 'Ready to start daemons' + +# only start after the primary node gives a green-light +if [ "$on_primary" == "true" ]; then + touch /mnt/lsf/config_done +fi +while true; do + [ -f /mnt/lsf/config_done ] && break + echo "waiting, not starting yet" + sleep 3 + ls -l /mnt/lsf /mnt/lsf/config_done 1>/dev/null 2>&1 # creating some NFS activity +done +echo "got green light for starting" + +### useless and this dangerously unsets LSF_TOP and LSF_VERSION +#if [ "$on_primary" == "true" ]; then +# . $LSF_TOP/conf/profile.lsf +#fi + +$LSF_TOP_VERSION/install/hostsetup --top="$LSF_TOP" --boot="y" --start="y" +systemctl status lsfd + +### warning: this dangerously unsets LSF_TOP and LSF_VERSION +source ~/.bashrc + +if [ "$do_app_center" = true ] ; then + # Start all the PerfMonitor and WEBUI processes. + nohup >/tmp/perfout setsid perfadmin start all; perfadmin list + sleep 5 + nohup >/tmp/pmcout setsid pmcadmin start; pmcadmin list + appcenter_status=$(pmcadmin list | grep "WEBGUI" | awk '{print $2}') + if [ "$appcenter_status" = "STARTED" ]; then + echo "Application Center installation completed..." + else + echo "Application Center installation failed..." + fi +fi + + +# Setup start at boot +# Lsf processes are started by systemctl. +# The script '/root/lsf_start_pac' manages the start of PAC processes if in HA. +if [ "$do_app_center" = "true" ]; then + echo "Configuring the start of the Pac" + cat < /root/lsf_start_pac +#!/bin/sh + +logfile=/tmp/lsf_start_pac.log +echo "\$(date +'%Y%m%d_%H%M%S'): START" > \$logfile + +# Wait mount point just to be sure it is ready +while [ ! mountpoint /mnt/lsf ]; do + sleep 1; +done +echo "\$(date +'%Y%m%d_%H%M%S'): File system '/mnt/lsf' is mounted" >> \$logfile + +# Waiting lsf processes before starting PAC +source ~/.bashrc +RC=1 +x=1 +while [ \$RC -eq 1 ] && [ \$x -le 600 ]; do + lsf_daemons status >> \$logfile; RC=\$? + echo "\$(date +'%Y%m%d_%H%M%S'): RC=\$RC; attempt #\$x" >> \$logfile + x=\$((x+1)) + sleep \$((\$x / 10 + 1)) +done +echo "END" >> \$logfile +perfadmin start all >> \$logfile +sleep 5 +pmcadmin start >> \$logfile +echo "EXIT" >> \$logfile + +EOT + chmod 755 /root/lsf_start_pac + command="/root/lsf_start_pac" + (crontab -l 2>/dev/null; echo "@reboot $command") | crontab - +fi + + +# Setting up the LDAP configuration +if [ "$enable_ldap" = "true" ]; then + + # Detect RHEL version + rhel_version=$(grep -oE 'release [0-9]+' /etc/redhat-release | awk '{print $2}') + + if [ "$rhel_version" = "8" ]; then + echo "Detected RHEL 8. Proceeding with LDAP client configuration...." + + # Allow Password authentication + sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config + systemctl restart sshd + + # Configure LDAP authentication + authconfig --enableldap --enableldapauth \ + --ldapserver=ldap://"${ldap_server_ip}" \ + --ldapbasedn="dc=${base_dn%%.*},dc=${base_dn#*.}" \ + --enablemkhomedir --update + + # Check the exit status of the authconfig command + if [ $? -eq 0 ]; then + echo "LDAP Authentication enabled successfully." + else + echo "Failed to enable LDAP and LDAP Authentication." + exit 1 + fi + + # Update LDAP Client configurations in nsswitch.conf + sed -i -e 's/^passwd:.*$/passwd: files ldap/' -e 's/^shadow:.*$/shadow: files ldap/' -e 's/^group:.*$/group: files ldap/' /etc/nsswitch.conf # pragma: allowlist secret + + # Update PAM configuration files + sed -i -e '/^auth/d' /etc/pam.d/password-auth + sed -i -e '/^auth/d' /etc/pam.d/system-auth + + auth_line="\nauth required pam_env.so\n\ +auth sufficient pam_unix.so nullok try_first_pass\n\ +auth requisite pam_succeed_if.so uid >= 1000 quiet_success\n\ +auth sufficient pam_ldap.so use_first_pass\n\ +auth required pam_deny.so" + + echo -e "$auth_line" | tee -a /etc/pam.d/password-auth /etc/pam.d/system-auth + + # Copy 'password-auth' settings to 'sshd' + cat /etc/pam.d/password-auth > /etc/pam.d/sshd + + # Configure nslcd + cat < /etc/nslcd.conf +uid nslcd +gid ldap +uri ldap://${ldap_server_ip}/ +base dc=${base_dn%%.*},dc=${base_dn#*.} +EOF + + # Restart nslcd and nscd service + systemctl restart nslcd + systemctl restart nscd + + # Enable nslcd and nscd service + systemctl enable nslcd + systemctl enable nscd + + # Validate the LDAP configuration + if ldapsearch -x -H ldap://"${ldap_server_ip}"/ -b "dc=${base_dn%%.*},dc=${base_dn#*.}" > /dev/null; then + echo "LDAP configuration completed successfully !!" + else + echo "LDAP configuration failed !!" + exit 1 + fi + + # Make LSF commands available for every user. + echo ". ${LSF_CONF}/profile.lsf" >> /etc/bashrc + source /etc/bashrc + else + echo "This script is designed for RHEL 8. Detected RHEL version: $rhel_version. Exiting." + exit 1 + fi +fi + +# Manually start perfmon, used by monitoring +# This is not needed, given that SCHED_METRIC_ENABLE=Y +#su - lsfadmin -c "badmin perfmon start" + +# Ensure lsf_prometheus_exporter service to be executed after shared filesystem mount +sed -i 's/After=network-online.target/After=network-online.target mnt-lsf.mount/g' /etc/systemd/system/lsf_prometheus_exporter.service +systemctl daemon-reload + +# Enable LSF prometheus exporter +systemctl enable lsf_prometheus_exporter +systemctl restart lsf_prometheus_exporter + +# Setting up the Metrics Agent +if [ "$observability_monitoring_enable" = true ]; then + + if [ "$cloud_monitoring_access_key" != "" ] && [ "$cloud_monitoring_ingestion_url" != "" ]; then + + SYSDIG_CONFIG_FILE="/opt/draios/etc/dragent.yaml" + PROMETHEUS_CONFIG_FILE="/opt/prometheus/prometheus.yml" + + #packages installation + echo "Writing sysdig config file" + + #sysdig config file + echo "Setting customerid access key" + sed -i "s/==ACCESSKEY==/$cloud_monitoring_access_key/g" $SYSDIG_CONFIG_FILE + sed -i "s/==COLLECTOR==/$cloud_monitoring_ingestion_url/g" $SYSDIG_CONFIG_FILE + echo "tags: type:management,lsf:true" >> $SYSDIG_CONFIG_FILE + + cat < $PROMETHEUS_CONFIG_FILE +global: + scrape_interval: 60s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +scrape_configs: + - job_name: "lsf_prometheus_exporter" + static_configs: + - targets: ["localhost:9405"] +remote_write: +- url: "$cloud_monitoring_prws_url" + authorization: + credentials: "$cloud_monitoring_prws_key" +EOTF + + # Enable prometheus + systemctl enable prometheus + systemctl restart prometheus + + echo "Restarting sysdig agent" + systemctl enable dragent + systemctl restart dragent + else + echo "Skipping metrics agent configuration due to missing parameters" + fi +else + echo "Metrics agent configuration skipped since monitoring provisioning is not enabled" +fi + +echo "END $(date '+%Y-%m-%d %H:%M:%S')" +sleep 0.1 # don't race against the log diff --git a/modules/landing_zone_vsi/configuration_steps/management_values.tpl b/modules/landing_zone_vsi/configuration_steps/management_values.tpl new file mode 100644 index 00000000..564fbaf7 --- /dev/null +++ b/modules/landing_zone_vsi/configuration_steps/management_values.tpl @@ -0,0 +1,72 @@ +#!/usr/bin/bash + +################################################### +# Copyright (C) IBM Corp. 2023 All Rights Reserved. +# Licensed under the Apache License v2.0 +################################################### + +### EXPORT_USER_DATA ### + +#input parameters +VPC_APIKEY_VALUE="${vpc_apikey_value}" +RESOURCE_RECORDS_APIKEY_VALUE="${vpc_apikey_value}" +management_node_count="${management_node_count}" +api_endpoint_eu_de="${api_endpoint_eu_de}" +api_endpoint_us_east="${api_endpoint_us_east}" +api_endpoint_us_south="${api_endpoint_us_south}" +imageID="${image_id}" +subnetID="${subnet_id}" +vpcID="${vpc_id}" +securityGroupID="${security_group_id}" +sshkey_ID="${sshkey_id}" +regionName="${region_name}" +zoneName="${zone_name}" +# the CIDR block for dyanmic hosts +rc_cidr_block="${rc_cidr_block}" +# the maximum allowed dynamic hosts created by RC +rc_max_num=${rc_max_num} +rc_rg=${rc_rg} +cluster_name="${cluster_name}" +ce_project_guid="${ce_project_guid}" +cluster_prefix="${cluster_prefix}" +cluster_private_key_content="${cluster_private_key_content}" +cluster_public_key_content="${cluster_public_key_content}" +bastion_public_key_content="${bastion_public_key_content}" +hyperthreading="${hyperthreading}" +network_interface=${network_interface} +dns_domain="${dns_domain}" +mount_path="${mount_path}" +custom_file_shares="${custom_file_shares}" +custom_mount_paths="${custom_mount_paths}" +contract_id="${contract_id}" +app_center_gui_pwd="${app_center_gui_pwd}" +enable_app_center="${enable_app_center}" +management_ip=${management_ip} +management_hostname=${management_hostname} +management_cand_ips=${management_cand_ips} +management_cand_hostnames=${management_cand_hostnames} +login_ip=${login_ip} +login_hostname=${login_hostname} +# PAC High Availability +app_center_high_availability="${app_center_high_availability}" +db_adminuser="${db_adminuser}" +db_adminpassword="${db_adminpassword}" +db_hostname="${db_hostname}" +db_port="${db_port}" +db_name="${db_name}" +db_user="${db_user}" +db_password="${db_password}" +db_certificate="${db_certificate}" +# LDAP Server +enable_ldap="${enable_ldap}" +ldap_server_ip="${ldap_server_ip}" +ldap_server_hostname="${ldap_server_hostname}" +ldap_basedns="${ldap_basedns}" +bootdrive_crn="${bootdrive_crn}" +# Observability +observability_monitoring_enable="${observability_monitoring_enable}" +observability_monitoring_on_compute_nodes_enable="${observability_monitoring_on_compute_nodes_enable}" +cloud_monitoring_access_key="${cloud_monitoring_access_key}" +cloud_monitoring_ingestion_url="${cloud_monitoring_ingestion_url}" +cloud_monitoring_prws_key="${cloud_monitoring_prws_key}" +cloud_monitoring_prws_url="${cloud_monitoring_prws_url}" diff --git a/modules/landing_zone_vsi/datasource.tf b/modules/landing_zone_vsi/datasource.tf index 3ca2f7ff..2e31f62a 100644 --- a/modules/landing_zone_vsi/datasource.tf +++ b/modules/landing_zone_vsi/datasource.tf @@ -1,54 +1,37 @@ -data "ibm_resource_group" "itself" { - name = var.resource_group -} - -# TODO: Verify distinct profiles -/* -data "ibm_is_instance_profile" "management" { - name = var.management_profile -} - -data "ibm_is_instance_profile" "compute" { - name = var.compute_profile -} - -data "ibm_is_instance_profile" "storage" { - name = var.storage_profile +data "ibm_is_image" "management" { + name = var.management_image_name + count = local.image_mapping_entry_found ? 0 : 1 } -data "ibm_is_instance_profile" "protocol" { - name = var.protocol_profile +data "ibm_is_image" "compute" { + name = var.compute_image_name + count = local.compute_image_from_data ? 1 : 0 } -*/ data "ibm_is_image" "login" { - name = var.login_image_name -} - -data "ibm_is_image" "management" { - name = var.management_image_name + name = var.login_image_name + count = local.login_image_mapping_entry_found ? 0 : 1 } -data "ibm_is_image" "compute" { - name = var.compute_image_name +data "ibm_is_ssh_key" "compute" { + for_each = toset(var.compute_ssh_keys) + name = each.key } -data "ibm_is_image" "storage" { - name = var.storage_image_name +data "ibm_is_region" "region" { + name = local.region } - -data "ibm_is_ssh_key" "login" { - for_each = toset(var.login_ssh_keys) - name = each.key +data "ibm_is_instance_profile" "management_node" { + name = var.management_node_instance_type } -data "ibm_is_ssh_key" "compute" { - for_each = toset(var.compute_ssh_keys) +data "ibm_is_ssh_key" "bastion" { + for_each = toset(var.ssh_keys) name = each.key } -data "ibm_is_ssh_key" "storage" { - for_each = toset(var.storage_ssh_keys) - name = each.key +data "ibm_is_image" "ldap_vsi_image" { + name = var.ldap_vsi_osimage_name + count = var.ldap_basedns != null && var.ldap_server == "null" ? 1 : 0 } diff --git a/modules/landing_zone_vsi/image_map.tf b/modules/landing_zone_vsi/image_map.tf new file mode 100644 index 00000000..17adf743 --- /dev/null +++ b/modules/landing_zone_vsi/image_map.tf @@ -0,0 +1,19 @@ +locals { + image_region_map = { + "hpcaas-lsf10-rhel88-v6" = { + "us-east" = "r014-7c8ff827-42f9-4e52-8ac5-0cabfa83cc08" + "eu-de" = "r010-ef5c9c76-88c9-461a-9ea9-ae3483b12463" + "us-south" = "r006-56948288-f03a-452f-a4e8-13c9523e5aac" + }, + "hpcaas-lsf10-rhel88-compute-v5" = { + "us-east" = "r014-deb34fb1-edbf-464c-9af3-7efa2efcff3f" + "eu-de" = "r010-2d04cfff-6f54-45d1-b3b3-7e259083d71f" + "us-south" = "r006-236ee1f4-38de-4845-b7ec-e2ffa7df5d08" + }, + "hpcaas-lsf10-ubuntu2204-compute-v4" = { + "us-east" = "r014-b15b5e51-ccb6-40e4-9d6b-d0d47864a8a2" + "eu-de" = "r010-39f4de94-2a55-431e-ad86-613c5b23a030" + "us-south" = "r006-fe0e6afd-4d01-4794-a9ed-dd5353dda482" + } + } +} diff --git a/modules/landing_zone_vsi/locals.tf b/modules/landing_zone_vsi/locals.tf index 4909fb5b..1ccbe93f 100644 --- a/modules/landing_zone_vsi/locals.tf +++ b/modules/landing_zone_vsi/locals.tf @@ -1,203 +1,192 @@ # define variables locals { - # Future use - # products = "scale" name = "hpc" prefix = var.prefix tags = [local.prefix, local.name] vsi_interfaces = ["eth0", "eth1"] - bms_interfaces = ["ens1", "ens2"] # TODO: explore (DA always keep it true) skip_iam_authorization_policy = true - - block_storage_volumes = [for volume in var.nsd_details : { - name = format("nsd-%s", index(var.nsd_details, volume) + 1) - profile = volume["profile"] - capacity = volume["capacity"] - iops = volume["iops"] - resource_group = local.resource_group_id - # TODO: Encryption - # encryption_key = - }] - # TODO: Update the LB configurable - # Bug: 5847 - LB profile & subnets are not configurable - /* - load_balancers = [{ - name = "hpc" - type = "private" - listener_port = 80 - listener_protocol = "http" - connection_limit = 10 - algorithm = "round_robin" - protocol = "http" - health_delay = 60 - health_retries = 5 - health_timeout = 30 - health_type = "http" - pool_member_port = 80 - }] - */ - - management_instance_count = sum(var.management_instances[*]["count"]) - storage_instance_count = sum(var.storage_instances[*]["count"]) - protocol_instance_count = sum(var.protocol_instances[*]["count"]) - static_compute_instance_count = sum(var.static_compute_instances[*]["count"]) - - enable_login = local.management_instance_count > 0 - enable_management = local.management_instance_count > 0 - enable_compute = local.management_instance_count > 0 || local.static_compute_instance_count > 0 || local.protocol_instance_count > 0 - enable_storage = local.storage_instance_count > 0 - # TODO: Fix the logic - enable_block_storage = var.storage_type == "scratch" ? true : false - enable_protocol = local.storage_instance_count > 0 && local.protocol_instance_count > 0 - # Future use - # TODO: Fix the logic - # enable_load_balancer = false - - login_node_name = format("%s-%s", local.prefix, "login") - management_node_name = format("%s-%s", local.prefix, "mgmt") - compute_node_name = format("%s-%s", local.prefix, "comp") - storage_node_name = format("%s-%s", local.prefix, "strg") - protocol_node_name = format("%s-%s", local.prefix, "proto") - - # Future use - /* - management_instance_count = sum(var.management_instances[*]["count"]) - management_instance_profile = flatten([for item in var.management_instances: [ - for count in range(item["count"]) : var.management_instances[index(var.management_instances, item)]["profile"] - ]]) - static_compute_instance_count = sum(var.static_compute_instances[*]["count"]) - storage_instance_count = sum(var.storage_instances[*]["count"]) - protocol_instance_count = sum(var.protocol_instances[*]["count"]) - */ - - # Future use - /* - login_image_name = var.login_image_name - management_image_name = var.management_image_name - compute_image_name = var.compute_image_name - storage_image_name = var.storage_image_name - protocol_image_name = var.storage_image_name - */ - - management_image_id = data.ibm_is_image.management.id - login_image_id = data.ibm_is_image.login.id - compute_image_id = data.ibm_is_image.compute.id - storage_image_id = data.ibm_is_image.storage.id - protocol_image_id = data.ibm_is_image.storage.id - - storage_ssh_keys = [for name in var.storage_ssh_keys : data.ibm_is_ssh_key.storage[name].id] - compute_ssh_keys = [for name in var.compute_ssh_keys : data.ibm_is_ssh_key.compute[name].id] - login_ssh_keys = [for name in var.login_ssh_keys : data.ibm_is_ssh_key.login[name].id] - management_ssh_keys = local.compute_ssh_keys - protocol_ssh_keys = local.storage_ssh_keys - - # Future use - /* - # Scale static configs - scale_cloud_deployer_path = "/opt/IBM/ibm-spectrumscale-cloud-deploy" - scale_cloud_install_repo_url = "https://github.com/IBM/ibm-spectrum-scale-cloud-install" - scale_cloud_install_repo_name = "ibm-spectrum-scale-cloud-install" - scale_cloud_install_branch = "5.1.8.1" - scale_cloud_infra_repo_url = "https://github.com/IBM/ibm-spectrum-scale-install-infra" - scale_cloud_infra_repo_name = "ibm-spectrum-scale-install-infra" - scale_cloud_infra_repo_tag = "v2.7.0" - */ + enable_compute = true + enable_management = true + ldap_node_name = format("%s-%s", local.prefix, "ldap") + login_node_name = format("%s-%s", local.prefix, "login") + management_node_name = format("%s-%s", local.prefix, "mgmt") + compute_ssh_keys = [for name in var.compute_ssh_keys : data.ibm_is_ssh_key.compute[name].id] + management_ssh_keys = local.compute_ssh_keys + ldap_enable = var.enable_ldap == true && var.ldap_server == "null" ? 1 : 0 # Region and Zone calculations region = join("-", slice(split("-", var.zones[0]), 0, 2)) + # # TODO: Compute & storage can't be added due to SG rule limitation + /* [ERROR] Error while creating Security Group Rule Exceeded limit of remote rules per security group + (the limit is 5 remote rules per security group)*/ - # TODO: DNS configs - - # Security group rules - login_security_group_rules = [ + compute_security_group_rules = [ { - name = "allow-all-bastion" + name = "allow-all-bastion-inbound" direction = "inbound" remote = var.bastion_security_group_id }, { - name = "allow-all-compute" + name = "allow-port-22-inbound" direction = "inbound" - remote = module.compute_sg[0].security_group_id - }, - { - name = "allow-all-bastion" - direction = "outbound" remote = var.bastion_security_group_id + tcp = { + port_min = 22 + port_max = 22 + } }, { - name = "allow-all-compute" - direction = "outbound" - remote = module.compute_sg[0].security_group_id - } - ] - # TODO: Compute & storage can't be added due to SG rule limitation - /* [ERROR] Error while creating Security Group Rule Exceeded limit of remote rules per security group - (the limit is 5 remote rules per security group)*/ - - compute_security_group_rules = [ - { - name = "allow-all-bastion" + name = "allow-all-compute-inbound" direction = "inbound" - remote = var.bastion_security_group_id + remote = module.compute_sg[0].security_group_id_for_ref }, { - name = "allow-all-login" + name = "allow-all-compute-0-inbound" direction = "inbound" - remote = module.login_sg[0].security_group_id + remote = local.compute_subnets[0].cidr + tcp = { + port_min = 2049 + port_max = 2049 + } }, { - name = "allow-all-bastion" - direction = "outbound" - remote = var.bastion_security_group_id + name = "allow-all-storage-inbound" + direction = "inbound" + remote = var.storage_security_group_id != null ? var.storage_security_group_id : module.compute_sg[0].security_group_id_for_ref }, { - name = "allow-all-login" + name = "allow-all-bastion-outbound" direction = "outbound" - remote = module.login_sg[0].security_group_id - } - ] - storage_security_group_rules = [ - { - name = "allow-all-bastion" - direction = "inbound" remote = var.bastion_security_group_id }, { - name = "allow-all-compute" - direction = "inbound" - remote = module.compute_sg[0].security_group_id + name = "allow-all-compute-0-outbound" + direction = "outbound" + remote = local.compute_subnets[0].cidr + tcp = { + port_min = 2049 + port_max = 2049 + } }, { - name = "allow-all-bastion" + name = "allow-all-outbound-outbound" direction = "outbound" - remote = var.bastion_security_group_id + remote = "0.0.0.0/0" }, + ] + + storage_nfs_security_group_rules = [ { - name = "allow-all-compute" - direction = "outbound" + name = "allow-all-hpcaas-compute-sg" + direction = "inbound" remote = module.compute_sg[0].security_group_id - }] + } + ] + # LDAP security group rule for Cluster + ldap_security_group_rule_for_cluster = [ + { + name = "inbound-rule-for-ldap-node-connection" + direction = "inbound" + remote = var.ldap_server + tcp = { + port_min = 389 + port_max = 389 + } + } + ] - # Derived configs - # VPC - resource_group_id = data.ibm_resource_group.itself.id + # SSH connection to the Login node via Cluster nodes. + ssh_connection_to_login_node_via_cluster_nodes = [ + { + name = "inbound-rule-for-login-node-ssh-connection" + direction = "inbound" + remote = module.compute_sg[0].security_group_id + tcp = { + port_min = 22 + port_max = 22 + } + } + ] # Subnets # TODO: Multi-zone multi-vNIC VSIs deployment support (bug #https://github.ibm.com/GoldenEye/issues/issues/5830) # Findings: Singe zone multi-vNICs VSIs deployment & multi-zone single vNIC VSIs deployment are supported. - login_subnets = var.login_subnets - compute_subnets = var.compute_subnets - storage_subnets = var.storage_subnets - protocol_subnets = var.protocol_subnets + compute_subnets = var.compute_subnets + + # Check whether an entry is found in the mapping file for the given management node image + image_mapping_entry_found = contains(keys(local.image_region_map), var.management_image_name) + new_image_id = local.image_mapping_entry_found ? local.image_region_map[var.management_image_name][local.region] : "Image not found with the given name" + + # Check whether an entry is found in the mapping file for the given compute node image + compute_image_found_in_map = contains(keys(local.image_region_map), var.compute_image_name) + # If not found, assume the name is the id already (customer provided image) + new_compute_image_id = local.compute_image_found_in_map ? local.image_region_map[var.compute_image_name][local.region] : var.compute_image_name + compute_image_from_data = !local.compute_image_found_in_map && !startswith(local.new_compute_image_id, "crn:") + + # Check whether an entry is found in the mapping file for the given login node image + login_image_mapping_entry_found = contains(keys(local.image_region_map), var.login_image_name) + new_login_image_id = local.login_image_mapping_entry_found ? local.image_region_map[var.login_image_name][local.region] : "Image not found with the given name" + + compute_node_max_count = 500 + rc_max_num = local.compute_node_max_count + + bastion_subnets = var.bastion_subnets + bastion_ssh_keys = [for name in var.ssh_keys : data.ibm_is_ssh_key.bastion[name].id] + ldap_server = var.enable_ldap == true && var.ldap_server == "null" ? length(module.ldap_vsi) > 0 ? var.ldap_primary_ip[0] : null : var.ldap_server + ldap_instance_image_id = var.enable_ldap == true && var.ldap_server == "null" ? data.ibm_is_image.ldap_vsi_image[0].id : "null" + + # The below logic is needed to point the API endpoints for the dynanic host creation + us_east = "https://api.us-east.codeengine.cloud.ibm.com/v2beta" + eu_de = "https://api.eu-de.codeengine.cloud.ibm.com/v2beta" + us_south = "https://api.us-south.codeengine.cloud.ibm.com/v2beta" + + # ip/names of vsis + management_vsi_data = flatten(module.management_vsi[*]["list"]) + management_private_ip = local.management_vsi_data[0]["ipv4_address"] + management_hostname = local.management_vsi_data[0]["name"] + + management_candidate_vsi_data = flatten(module.management_candidate_vsi[*]["list"]) + management_candidate_private_ips = local.management_candidate_vsi_data[*]["ipv4_address"] + management_candidate_hostnames = local.management_candidate_vsi_data[*]["name"] + + login_vsi_data = flatten(module.login_vsi[*]["list"]) + login_private_ips = local.login_vsi_data[*]["ipv4_address"] + login_hostnames = local.login_vsi_data[*]["name"] + + ldap_vsi_data = flatten(module.ldap_vsi[*]["list"]) + #ldap_private_ips = local.ldap_vsi_data[*]["ipv4_address"] + ldap_hostnames = local.ldap_vsi_data[*]["name"] + +} + +########################################################################### +# IBM Cloud Dababase for MySQL database local variables +########################################################################### +locals { + db_name = "pac" + db_user = "pacuser" +} + +## Differentiating VPC File Share and NFS share +locals { + nfs_file_share = [ + for share in var.mount_path : + { + mount_path = share.mount_path + nfs_share = share.nfs_share + } + if share.nfs_share != null && share.nfs_share != "" + ] - # Security Groups - protocol_secondary_security_group = [for subnet in local.protocol_subnets : + vpc_file_share = [ + for share in var.mount_path : { - security_group_id = one(module.compute_sg[*].security_group_id) - interface_name = subnet["name"] + mount_path = share.mount_path + size = share.size + iops = share.iops } + if share.size != null && share.iops != null ] } diff --git a/modules/landing_zone_vsi/main.tf b/modules/landing_zone_vsi/main.tf index a9b64894..39667e96 100644 --- a/modules/landing_zone_vsi/main.tf +++ b/modules/landing_zone_vsi/main.tf @@ -1,174 +1,238 @@ module "compute_key" { - count = local.enable_compute ? 1 : 0 - source = "./../key" - private_key_path = "compute_id_rsa" #checkov:skip=CKV_SECRET_6 -} - -module "storage_key" { - count = local.enable_storage ? 1 : 0 - source = "./../key" - private_key_path = "storage_id_rsa" #checkov:skip=CKV_SECRET_6 -} - -module "login_sg" { - count = local.enable_login ? 1 : 0 - source = "terraform-ibm-modules/security-group/ibm" - version = "1.0.1" - add_ibm_cloud_internal_rules = true - resource_group = local.resource_group_id - security_group_name = format("%s-login-sg", local.prefix) - security_group_rules = local.login_security_group_rules - vpc_id = var.vpc_id + count = local.enable_compute ? 1 : 0 + source = "./../key" + # private_key_path = "compute_id_rsa" #checkov:skip=CKV_SECRET_6 } module "compute_sg" { count = local.enable_compute ? 1 : 0 source = "terraform-ibm-modules/security-group/ibm" - version = "1.0.1" + version = "2.6.1" add_ibm_cloud_internal_rules = true - resource_group = local.resource_group_id - security_group_name = format("%s-comp-sg", local.prefix) + resource_group = var.resource_group + security_group_name = format("%s-cluster-sg", local.prefix) security_group_rules = local.compute_security_group_rules vpc_id = var.vpc_id + tags = local.tags } -module "storage_sg" { - count = local.enable_storage ? 1 : 0 - source = "terraform-ibm-modules/security-group/ibm" - version = "1.0.1" - add_ibm_cloud_internal_rules = true - resource_group = local.resource_group_id - security_group_name = format("%s-strg-sg", local.prefix) - security_group_rules = local.storage_security_group_rules - vpc_id = var.vpc_id +module "compute_sg_with_ldap_connection" { + count = var.ldap_server == "null" ? 0 : 1 + source = "terraform-ibm-modules/security-group/ibm" + version = "2.6.1" + resource_group = var.resource_group + add_ibm_cloud_internal_rules = true + use_existing_security_group_id = true + existing_security_group_id = module.compute_sg[0].security_group_id + security_group_rules = local.ldap_security_group_rule_for_cluster + vpc_id = var.vpc_id + depends_on = [module.compute_sg] } +module "ssh_connection_to_login_node_via_cluster_nodes" { + count = var.bastion_instance_name != null ? 1 : 0 + source = "terraform-ibm-modules/security-group/ibm" + version = "2.6.1" + resource_group = var.resource_group + add_ibm_cloud_internal_rules = true + use_existing_security_group_id = true + existing_security_group_id = var.bastion_security_group_id + security_group_rules = local.ssh_connection_to_login_node_via_cluster_nodes + vpc_id = var.vpc_id + depends_on = [module.compute_sg] +} -module "login_vsi" { - count = length(var.login_instances) - source = "terraform-ibm-modules/landing-zone-vsi/ibm" - version = "2.13.0" - vsi_per_subnet = var.login_instances[count.index]["count"] - create_security_group = false - security_group = null - image_id = local.login_image_id - machine_type = var.login_instances[count.index]["profile"] - prefix = count.index == 0 ? local.login_node_name : format("%s-%s", local.login_node_name, count.index) - resource_group_id = local.resource_group_id - enable_floating_ip = false - security_group_ids = module.login_sg[*].security_group_id - ssh_key_ids = local.login_ssh_keys - subnets = local.login_subnets - tags = local.tags - user_data = data.template_file.login_user_data.rendered - vpc_id = var.vpc_id - kms_encryption_enabled = var.kms_encryption_enabled - skip_iam_authorization_policy = local.skip_iam_authorization_policy - boot_volume_encryption_key = var.boot_volume_encryption_key +module "nfs_storage_sg" { + count = var.storage_security_group_id != null ? 1 : 0 + source = "terraform-ibm-modules/security-group/ibm" + version = "2.6.1" + resource_group = var.resource_group + add_ibm_cloud_internal_rules = true + use_existing_security_group_id = true + existing_security_group_id = var.storage_security_group_id + security_group_rules = local.storage_nfs_security_group_rules + vpc_id = var.vpc_id } module "management_vsi" { - count = length(var.management_instances) - source = "terraform-ibm-modules/landing-zone-vsi/ibm" - version = "2.13.0" - vsi_per_subnet = var.management_instances[count.index]["count"] + count = 1 + # count = length(var.management_instances) + source = "terraform-ibm-modules/landing-zone-vsi/ibm" + version = "4.0.0" + vsi_per_subnet = 1 + # vsi_per_subnet = var.management_instances[count.index]["count"] create_security_group = false security_group = null - image_id = local.management_image_id - machine_type = var.management_instances[count.index]["profile"] - prefix = count.index == 0 ? local.management_node_name : format("%s-%s", local.management_node_name, count.index) - resource_group_id = local.resource_group_id + image_id = local.image_mapping_entry_found ? local.new_image_id : data.ibm_is_image.management[0].id + machine_type = data.ibm_is_instance_profile.management_node.name + prefix = format("%s-%s", local.management_node_name, count.index + 1) + resource_group_id = var.resource_group enable_floating_ip = false security_group_ids = module.compute_sg[*].security_group_id ssh_key_ids = local.management_ssh_keys - subnets = local.compute_subnets + subnets = [local.compute_subnets[0]] tags = local.tags - user_data = data.template_file.management_user_data.rendered + user_data = "${data.template_file.management_user_data.rendered} ${file("${path.module}/templates/lsf_management.sh")}" vpc_id = var.vpc_id kms_encryption_enabled = var.kms_encryption_enabled skip_iam_authorization_policy = local.skip_iam_authorization_policy boot_volume_encryption_key = var.boot_volume_encryption_key - placement_group_id = var.placement_group_ids - #placement_group_id = var.placement_group_ids[(var.management_instances[count.index]["count"])%(length(var.placement_group_ids))] + } -module "compute_vsi" { - count = length(var.static_compute_instances) +module "management_candidate_vsi" { + count = var.management_node_count - 1 source = "terraform-ibm-modules/landing-zone-vsi/ibm" - version = "2.13.0" - vsi_per_subnet = var.static_compute_instances[count.index]["count"] + version = "4.0.0" create_security_group = false security_group = null - image_id = local.compute_image_id - machine_type = var.static_compute_instances[count.index]["profile"] - prefix = count.index == 0 ? local.compute_node_name : format("%s-%s", local.compute_node_name, count.index) - resource_group_id = local.resource_group_id - enable_floating_ip = false security_group_ids = module.compute_sg[*].security_group_id - ssh_key_ids = local.compute_ssh_keys - subnets = local.compute_subnets - tags = local.tags - user_data = data.template_file.compute_user_data.rendered vpc_id = var.vpc_id + ssh_key_ids = local.management_ssh_keys + subnets = [local.compute_subnets[0]] + resource_group_id = var.resource_group + enable_floating_ip = false + user_data = "${data.template_file.management_user_data.rendered} ${file("${path.module}/templates/lsf_management.sh")}" kms_encryption_enabled = var.kms_encryption_enabled skip_iam_authorization_policy = local.skip_iam_authorization_policy boot_volume_encryption_key = var.boot_volume_encryption_key - placement_group_id = var.placement_group_ids - #placement_group_id = var.placement_group_ids[(var.static_compute_instances[count.index]["count"])%(length(var.placement_group_ids))] + image_id = local.image_mapping_entry_found ? local.new_image_id : data.ibm_is_image.management[0].id + prefix = format("%s-%s", local.management_node_name, count.index + 2) + machine_type = data.ibm_is_instance_profile.management_node.name + vsi_per_subnet = 1 + tags = local.tags } -module "storage_vsi" { - count = length(var.storage_instances) +module "login_vsi" { + count = 1 source = "terraform-ibm-modules/landing-zone-vsi/ibm" - version = "2.13.0" - vsi_per_subnet = var.storage_instances[count.index]["count"] + version = "4.0.0" + vsi_per_subnet = 1 create_security_group = false security_group = null - image_id = local.storage_image_id - machine_type = var.storage_instances[count.index]["profile"] - prefix = count.index == 0 ? local.storage_node_name : format("%s-%s", local.storage_node_name, count.index) - resource_group_id = local.resource_group_id + image_id = local.login_image_mapping_entry_found ? local.new_login_image_id : data.ibm_is_image.login[0].id + machine_type = var.login_node_instance_type + prefix = local.login_node_name + resource_group_id = var.resource_group enable_floating_ip = false - security_group_ids = module.storage_sg[*].security_group_id - ssh_key_ids = local.storage_ssh_keys - subnets = local.storage_subnets + security_group_ids = [var.bastion_security_group_id] + ssh_key_ids = local.bastion_ssh_keys + subnets = length(var.bastion_subnets) == 2 ? [local.bastion_subnets[1]] : [local.bastion_subnets[0]] tags = local.tags - user_data = data.template_file.storage_user_data.rendered + user_data = "${data.template_file.login_user_data.rendered} ${file("${path.module}/templates/login_vsi.sh")}" vpc_id = var.vpc_id - block_storage_volumes = local.enable_block_storage ? local.block_storage_volumes : [] kms_encryption_enabled = var.kms_encryption_enabled - skip_iam_authorization_policy = local.skip_iam_authorization_policy boot_volume_encryption_key = var.boot_volume_encryption_key - placement_group_id = var.placement_group_ids - #placement_group_id = var.placement_group_ids[(var.storage_instances[count.index]["count"])%(length(var.placement_group_ids))] + skip_iam_authorization_policy = local.skip_iam_authorization_policy } -module "protocol_vsi" { - count = length(var.protocol_instances) - source = "terraform-ibm-modules/landing-zone-vsi/ibm" - version = "2.13.0" - vsi_per_subnet = var.protocol_instances[count.index]["count"] - create_security_group = false - security_group = null - image_id = local.protocol_image_id - machine_type = var.protocol_instances[count.index]["profile"] - prefix = count.index == 0 ? local.protocol_node_name : format("%s-%s", local.protocol_node_name, count.index) - resource_group_id = local.resource_group_id +module "ldap_vsi" { + count = local.ldap_enable + # count = length(var.management_instances) + source = "terraform-ibm-modules/landing-zone-vsi/ibm" + version = "4.0.0" + vsi_per_subnet = 1 + create_security_group = false + security_group = null + image_id = local.ldap_instance_image_id + machine_type = var.ldap_vsi_profile + prefix = local.ldap_node_name + # prefix = count.index == 0 ? local.management_node_name : format("%s-%s", local.management_node_name, count.index) + resource_group_id = var.resource_group enable_floating_ip = false - security_group_ids = module.storage_sg[*].security_group_id - ssh_key_ids = local.protocol_ssh_keys - subnets = local.storage_subnets + security_group_ids = module.compute_sg[*].security_group_id + ssh_key_ids = local.management_ssh_keys + subnets = [local.compute_subnets[0]] tags = local.tags - user_data = data.template_file.protocol_user_data.rendered + user_data = var.enable_ldap == true && var.ldap_server == "null" ? "${data.template_file.ldap_user_data[0].rendered} ${file("${path.module}/templates/ldap_user_data.sh")}" : "" vpc_id = var.vpc_id kms_encryption_enabled = var.kms_encryption_enabled skip_iam_authorization_policy = local.skip_iam_authorization_policy boot_volume_encryption_key = var.boot_volume_encryption_key - # Bug: 5847 - LB profile & subnets are not configurable - # load_balancers = local.enable_load_balancer ? local.load_balancers : [] - secondary_allow_ip_spoofing = true - secondary_security_groups = local.protocol_secondary_security_group - secondary_subnets = local.protocol_subnets - placement_group_id = var.placement_group_ids - #placement_group_id = var.placement_group_ids[(var.protocol_instances[count.index]["count"])%(length(var.placement_group_ids))] + #placement_group_id = var.placement_group_ids[(var.management_instances[count.index]["count"])%(length(var.placement_group_ids))] +} + +module "generate_db_password" { + count = var.enable_app_center && var.app_center_high_availability ? 1 : 0 + source = "../../modules/security/password" + length = 15 + special = true + override_special = "-_" + min_numeric = 1 +} + +module "ssh_key" { + source = "./../key" +} + +module "wait_management_vsi_booted" { + source = "./../../modules/null/remote_exec" + cluster_host = concat([local.management_private_ip]) + cluster_user = var.cluster_user #"root" + cluster_private_key = var.compute_private_key_content + login_host = var.bastion_fip + login_user = "ubuntu" + login_private_key = var.bastion_private_key_content + command = ["cloud-init status --wait;hostname;date;df;id"] + timeout = "15m" # let's be patient, the VSI may need time to boot completely + depends_on = [ + module.management_vsi + ] +} + +module "wait_management_candidate_vsi_booted" { + source = "./../../modules/null/remote_exec" + cluster_host = concat(local.management_candidate_private_ips) + cluster_user = var.cluster_user #"root" + cluster_private_key = var.compute_private_key_content + login_host = var.bastion_fip + login_user = "ubuntu" + login_private_key = var.bastion_private_key_content + command = ["cloud-init status --wait;hostname;date;df;id"] + timeout = "15m" # let's be patient, the VSI may need time to boot completely + depends_on = [ + module.management_candidate_vsi + ] +} + +module "do_management_vsi_configuration" { + source = "./../../modules/null/remote_exec_script" + cluster_host = concat([local.management_private_ip]) + cluster_user = var.cluster_user #"root" + cluster_private_key = var.compute_private_key_content + login_host = var.bastion_fip + login_user = "ubuntu" + login_private_key = var.bastion_private_key_content + payload_files = ["${path.module}/configuration_steps/configure_management_vsi.sh", "${path.module}/configuration_steps/compute_user_data_fragment.sh"] + payload_dirs = [] + new_file_name = "management_values" + new_file_content = data.template_file.management_values.rendered + script_to_run = "configure_management_vsi.sh" + sudo_user = "root" + with_bash = true + depends_on = [ + module.wait_management_vsi_booted + ] + trigger_string = join(",", module.management_vsi[0].ids) +} + +module "do_management_candidate_vsi_configuration" { + source = "./../../modules/null/remote_exec_script" + cluster_host = concat(local.management_candidate_private_ips) + cluster_user = var.cluster_user #"root" + cluster_private_key = var.compute_private_key_content + login_host = var.bastion_fip + login_user = "ubuntu" + login_private_key = var.bastion_private_key_content + payload_files = ["${path.module}/configuration_steps/configure_management_vsi.sh"] + payload_dirs = [] + new_file_name = "management_values" + new_file_content = data.template_file.management_values.rendered + script_to_run = "configure_management_vsi.sh" + sudo_user = "root" + with_bash = true + depends_on = [ + module.wait_management_candidate_vsi_booted + ] + trigger_string = join(",", flatten(module.management_candidate_vsi[*].ids)) } diff --git a/modules/landing_zone_vsi/outputs.tf b/modules/landing_zone_vsi/outputs.tf index f41d0923..76e8ffe2 100644 --- a/modules/landing_zone_vsi/outputs.tf +++ b/modules/landing_zone_vsi/outputs.tf @@ -1,29 +1,46 @@ -output "login_vsi_data" { - description = "Login VSI data" - value = module.login_vsi[*]["list"] -} - output "management_vsi_data" { description = "Management VSI data" value = module.management_vsi[*]["list"] } -output "compute_vsi_data" { - description = "Compute VSI data" - value = module.compute_vsi[*]["list"] +output "management_candidate_vsi_data" { + description = "Management candidate VSI data" + value = module.management_candidate_vsi[*]["list"] } -output "storage_vsi_data" { - description = "Storage VSI data" - value = module.storage_vsi[*]["list"] +output "login_vsi_data" { + description = "Login VSI data" + value = module.login_vsi[*]["list"] } -output "protocol_vsi_data" { - description = "Protocol VSI data" - value = module.protocol_vsi[*]["list"] +output "ldap_vsi_data" { + description = "Login VSI data" + value = module.ldap_vsi[*]["list"] +} + +output "image_map_entry_found" { + description = "Available if the image name provided is located within the image map" + value = "${local.image_mapping_entry_found} -- - ${var.management_image_name}" +} + +output "ldap_server" { + description = "LDAP server IP" + value = local.ldap_server } output "compute_sg_id" { description = "Compute SG id" value = module.compute_sg[*].security_group_id } + +output "compute_public_key_content" { + description = "Compute public key content" + value = one(module.compute_key[*].private_key_content) + sensitive = true +} + +output "compute_private_key_content" { + description = "Compute private key content" + value = one(module.compute_key[*].private_key_content) + sensitive = true +} diff --git a/modules/landing_zone_vsi/template_files.tf b/modules/landing_zone_vsi/template_files.tf index f1f7cb30..a5aadd84 100644 --- a/modules/landing_zone_vsi/template_files.tf +++ b/modules/landing_zone_vsi/template_files.tf @@ -1,58 +1,115 @@ -data "template_file" "login_user_data" { - template = file("${path.module}/templates/login_user_data.tpl") - vars = { - bastion_public_key_content = var.bastion_public_key_content != null ? var.bastion_public_key_content : "" - login_public_key_content = local.enable_login ? module.compute_key[0].public_key_content : "" - login_private_key_content = local.enable_login ? module.compute_key[0].private_key_content : "" - login_interfaces = var.storage_type == "scratch" ? local.vsi_interfaces[0] : local.bms_interfaces[0] - login_dns_domain = var.dns_domain_names["compute"] - } -} - data "template_file" "management_user_data" { template = file("${path.module}/templates/management_user_data.tpl") vars = { - bastion_public_key_content = var.bastion_public_key_content != null ? var.bastion_public_key_content : "" - management_public_key_content = local.enable_management ? module.compute_key[0].public_key_content : "" - management_private_key_content = local.enable_management ? module.compute_key[0].private_key_content : "" - management_interfaces = var.storage_type == "scratch" ? local.vsi_interfaces[0] : local.bms_interfaces[0] - management_dns_domain = var.dns_domain_names["compute"] + management_node_count = var.management_node_count + rc_cidr_block = local.compute_subnets[0].cidr + cluster_prefix = var.prefix + cluster_private_key_content = local.enable_management ? module.compute_key[0].private_key_content : "" + cluster_public_key_content = local.enable_management ? module.compute_key[0].public_key_content : "" + hyperthreading = var.hyperthreading_enabled + network_interface = local.vsi_interfaces[0] + dns_domain = var.dns_domain_names["compute"] + mount_path = var.share_path + enable_ldap = var.enable_ldap + ldap_server_ip = local.ldap_server + ldap_basedns = var.enable_ldap == true ? var.ldap_basedns : "null" + login_ip_address = var.login_private_ips } } -data "template_file" "compute_user_data" { - template = file("${path.module}/templates/compute_user_data.tpl") +data "template_file" "login_user_data" { + template = file("${path.module}/templates/login_user_data.tpl") vars = { - bastion_public_key_content = var.bastion_public_key_content != null ? var.bastion_public_key_content : "" - compute_public_key_content = local.enable_compute ? module.compute_key[0].public_key_content : "" - compute_private_key_content = local.enable_compute ? module.compute_key[0].private_key_content : "" - compute_interfaces = var.storage_type == "scratch" ? local.vsi_interfaces[0] : local.bms_interfaces[0] - compute_dns_domain = var.dns_domain_names["compute"] - # TODO: Fix me - dynamic_compute_instances = var.dynamic_compute_instances == null ? "" : "" + network_interface = local.vsi_interfaces[0] + dns_domain = var.dns_domain_names["compute"] + cluster_private_key_content = local.enable_management ? module.compute_key[0].private_key_content : "" + cluster_public_key_content = local.enable_management ? module.compute_key[0].public_key_content : "" + mount_path = var.share_path + enable_ldap = var.enable_ldap + rc_cidr_block = local.bastion_subnets[0].cidr + cluster_prefix = var.prefix + rc_cidr_block_1 = local.compute_subnets[0].cidr + hyperthreading = var.hyperthreading_enabled + ldap_server_ip = local.ldap_server + ldap_basedns = var.enable_ldap == true ? var.ldap_basedns : "null" } } -data "template_file" "storage_user_data" { - template = file("${path.module}/templates/storage_user_data.tpl") +data "template_file" "ldap_user_data" { + count = var.enable_ldap == true ? 1 : 0 + template = file("${path.module}/templates/ldap_user_data.tpl") vars = { - bastion_public_key_content = var.bastion_public_key_content != null ? var.bastion_public_key_content : "" - storage_public_key_content = local.enable_storage ? module.storage_key[0].public_key_content : "" - storage_private_key_content = local.enable_storage ? module.storage_key[0].private_key_content : "" - storage_interfaces = var.storage_type == "scratch" ? local.vsi_interfaces[0] : local.bms_interfaces[0] - storage_dns_domain = var.dns_domain_names["storage"] + ssh_public_key_content = local.enable_management ? module.compute_key[0].public_key_content : "" + ldap_basedns = var.ldap_basedns + ldap_admin_password = var.ldap_admin_password + cluster_prefix = var.prefix + ldap_user = var.ldap_user_name + ldap_user_password = var.ldap_user_password + dns_domain = var.dns_domain_names["compute"] } } -data "template_file" "protocol_user_data" { - template = file("${path.module}/templates/protocol_user_data.tpl") +data "template_file" "management_values" { + template = file("${path.module}/configuration_steps/management_values.tpl") vars = { - bastion_public_key_content = var.bastion_public_key_content != null ? var.bastion_public_key_content : "" - storage_public_key_content = local.enable_protocol ? module.storage_key[0].public_key_content : "" - storage_private_key_content = local.enable_protocol ? module.storage_key[0].private_key_content : "" - storage_interfaces = var.storage_type == "scratch" ? local.vsi_interfaces[0] : local.bms_interfaces[0] - protocol_interfaces = var.storage_type == "scratch" ? local.vsi_interfaces[1] : local.bms_interfaces[1] - storage_dns_domain = var.dns_domain_names["storage"] - protocol_dns_domain = var.dns_domain_names["protocol"] + bastion_public_key_content = var.bastion_public_key_content != null ? var.bastion_public_key_content : "" + vpc_apikey_value = var.ibmcloud_api_key + resource_records_apikey_value = var.ibmcloud_api_key + management_node_count = var.management_node_count + api_endpoint_us_east = local.us_east + api_endpoint_eu_de = local.eu_de + api_endpoint_us_south = local.us_south + image_id = local.compute_image_from_data ? data.ibm_is_image.compute[0].id : local.new_compute_image_id + subnet_id = local.compute_subnets[0].crn + security_group_id = module.compute_sg[0].security_group_id + sshkey_id = join(",", local.compute_ssh_keys) + region_name = data.ibm_is_region.region.name + zone_name = var.zones[0] + vpc_id = var.vpc_id + rc_cidr_block = local.compute_subnets[0].cidr + rc_max_num = local.rc_max_num + rc_rg = var.resource_group + cluster_name = var.cluster_id + ce_project_guid = file("${abspath("${path.module}/../../solutions/hpc")}/assets/hpcaas-ce-project-guid.cfg") + cluster_prefix = var.prefix + cluster_private_key_content = local.enable_management ? module.compute_key[0].private_key_content : "" + cluster_public_key_content = local.enable_management ? module.compute_key[0].public_key_content : "" + hyperthreading = var.hyperthreading_enabled + network_interface = local.vsi_interfaces[0] + dns_domain = var.dns_domain_names["compute"] + mount_path = var.share_path + custom_mount_paths = join(" ", concat(local.vpc_file_share[*]["mount_path"], local.nfs_file_share[*]["mount_path"])) + custom_file_shares = join(" ", concat([for file_share in var.file_share : file_share], local.nfs_file_share[*]["nfs_share"])) + contract_id = var.contract_id + enable_app_center = var.enable_app_center + app_center_gui_pwd = var.app_center_gui_pwd + enable_ldap = var.enable_ldap + ldap_server_ip = local.ldap_server + ldap_server_hostname = length(local.ldap_hostnames) > 0 ? local.ldap_hostnames[0] : "null" + ldap_basedns = var.enable_ldap == true ? var.ldap_basedns : "null" + bootdrive_crn = var.boot_volume_encryption_key == null ? "" : var.boot_volume_encryption_key + management_ip = local.management_private_ip + management_hostname = local.management_hostname + management_cand_ips = join(",", local.management_candidate_private_ips) + management_cand_hostnames = join(",", local.management_candidate_hostnames) + login_ip = local.login_private_ips[0] + login_hostname = local.login_hostnames[0] + # PAC High Availability + app_center_high_availability = var.app_center_high_availability + db_adminuser = var.enable_app_center && var.app_center_high_availability ? var.db_instance_info.adminuser : "" + db_adminpassword = var.enable_app_center && var.app_center_high_availability ? var.db_instance_info.adminpassword : "" + db_hostname = var.enable_app_center && var.app_center_high_availability ? var.db_instance_info.hostname : "" + db_port = var.enable_app_center && var.app_center_high_availability ? var.db_instance_info.port : "" + db_certificate = var.enable_app_center && var.app_center_high_availability ? var.db_instance_info.certificate : "" + db_name = var.enable_app_center && var.app_center_high_availability ? local.db_name : "" + db_user = var.enable_app_center && var.app_center_high_availability ? local.db_user : "" + db_password = var.enable_app_center && var.app_center_high_availability ? module.generate_db_password[0].password : "" + # Observability + observability_monitoring_enable = var.observability_monitoring_enable + observability_monitoring_on_compute_nodes_enable = var.observability_monitoring_on_compute_nodes_enable + cloud_monitoring_access_key = var.cloud_monitoring_access_key + cloud_monitoring_ingestion_url = var.cloud_monitoring_ingestion_url + cloud_monitoring_prws_key = var.cloud_monitoring_prws_key + cloud_monitoring_prws_url = var.cloud_monitoring_prws_url } } diff --git a/modules/landing_zone_vsi/templates/ldap_user_data.sh b/modules/landing_zone_vsi/templates/ldap_user_data.sh new file mode 100644 index 00000000..932ea1e5 --- /dev/null +++ b/modules/landing_zone_vsi/templates/ldap_user_data.sh @@ -0,0 +1,170 @@ +#!/usr/bin/bash +# shellcheck disable=all + +################################################### +# Copyright (C) IBM Corp. 2023 All Rights Reserved. +# Licensed under the Apache License v2.0 +################################################### + +#!/usr/bin/env bash + +USER=ubuntu +BASE_DN="${ldap_basedns}" +LDAP_DIR="/opt" +LDAP_ADMIN_PASSWORD="${ldap_admin_password}" +LDAP_GROUP="${cluster_prefix}" +LDAP_USER="${ldap_user}" +LDAP_USER_PASSWORD="${ldap_user_password}" + +if grep -E -q "CentOS|Red Hat" /etc/os-release +then + USER=vpcuser +elif grep -q "Ubuntu" /etc/os-release +then + USER=ubuntu +fi +sed -i -e "s/^/no-port-forwarding,no-agent-forwarding,no-X11-forwarding,command=\"echo \'Please login as the user \\\\\"$USER\\\\\" rather than the user \\\\\"root\\\\\".\';echo;sleep 5; exit 142\" /" /root/.ssh/authorized_keys + +#input parameters +ssh_public_key_content="${ssh_public_key_content}" +echo "${ssh_public_key_content}" >> home/$USER/.ssh/authorized_keys +echo "StrictHostKeyChecking no" >> /home/$USER/.ssh/config + +# Setup Network configuration +# Change the MTU setting as this is required for setting mtu as 9000 for communication to happen between clusters +if grep -q "NAME=\"Red Hat Enterprise Linux\"" /etc/os-release; then + # Replace the MTU value in the Netplan configuration + echo "MTU=9000" >> "/etc/sysconfig/network-scripts/ifcfg-${network_interface}" + echo "DOMAIN=\"${dns_domain}\"" >> "/etc/sysconfig/network-scripts/ifcfg-${network_interface}" + # Change the MTU setting as 9000 at router level. + gateway_ip=$(ip route | grep default | awk '{print $3}' | head -n 1) + echo "${rc_cidr_block} via $gateway_ip dev ${network_interface} metric 0 mtu 9000" >> /etc/sysconfig/network-scripts/route-eth0 + systemctl restart NetworkManager +elif grep -q "NAME=\"Ubuntu\"" /etc/os-release; then + net_int=$(basename /sys/class/net/en*) + netplan_config="/etc/netplan/50-cloud-init.yaml" + gateway_ip=$(ip route | grep default | awk '{print $3}' | head -n 1) + cidr_range=$(ip route show | grep "kernel" | awk '{print $1}' | head -n 1) + usermod -s /bin/bash lsfadmin + # Replace the MTU value in the Netplan configuration + if ! grep -qE "^[[:space:]]*mtu: 9000" $netplan_config; then + echo "MTU 9000 Packages entries not found" + # Append the MTU configuration to the Netplan file + sudo sed -i '/'"$net_int"':/a\ mtu: 9000' $netplan_config + sudo sed -i '/dhcp4: true/a \ nameservers:\n search: ['"$dns_domain"']' $netplan_config + sudo sed -i '/'"$net_int"':/a\ routes:\n - to: '"$cidr_range"'\n via: '"$gateway_ip"'\n metric: 100\n mtu: 9000' $netplan_config + sudo netplan apply + echo "MTU set to 9000 on Netplan." + else + echo "MTU entry already exists in Netplan. Skipping." + fi +fi + +#Installing LDAP +apt-get update -y +export DEBIAN_FRONTEND='non-interactive' +echo -e "slapd slapd/root_password password ${LDAP_ADMIN_PASSWORD}" |debconf-set-selections +echo -e "slapd slapd/root_password_again password ${LDAP_ADMIN_PASSWORD}" |debconf-set-selections +apt-get install -y slapd ldap-utils + +echo -e "slapd slapd/internal/adminpw password ${LDAP_ADMIN_PASSWORD}" |debconf-set-selections +echo -e "slapd slapd/internal/generated_adminpw password ${LDAP_ADMIN_PASSWORD}" |debconf-set-selections +echo -e "slapd slapd/password2 password ${LDAP_ADMIN_PASSWORD}" |debconf-set-selections +echo -e "slapd slapd/password1 password ${LDAP_ADMIN_PASSWORD}" |debconf-set-selections +echo -e "slapd slapd/domain string ${BASE_DN}" |debconf-set-selections +echo -e "slapd shared/organization string ${BASE_DN}" |debconf-set-selections +echo -e "slapd slapd/purge_database boolean false" |debconf-set-selections +echo -e "slapd slapd/move_old_database boolean true" |debconf-set-selections +echo -e "slapd slapd/no_configuration boolean false" |debconf-set-selections +dpkg-reconfigure slapd +echo "BASE dc=${BASE_DN%%.*},dc=${BASE_DN#*.}" >> /etc/ldap/ldap.conf +echo "URI ldap://localhost" >> /etc/ldap/ldap.conf +systemctl restart slapd +systemctl enable slapd + +#LDAP Operations + +check_and_create_ldap_ou() { + local ou_name="$1" + local ldif_file="${LDAP_DIR}/ou${ou_name}.ldif" + local search_result="" + + echo "dn: ou=${ou_name},dc=${BASE_DN%%.*},dc=${BASE_DN#*.} +objectClass: organizationalUnit +ou: ${ou_name}" > "${ldif_file}" + + ldapsearch -x -D "cn=admin,dc=${BASE_DN%%.*},dc=${BASE_DN#*.}" -w "${LDAP_ADMIN_PASSWORD}" -b "ou=${ou_name},dc=${BASE_DN%%.*},dc=${BASE_DN#*.}" "objectClass=organizationalUnit" > /dev/null 2>&1 + search_result=$? + + [ ${search_result} -eq 32 ] && echo "${ou_name}OUNotFound" || echo "${ou_name}OUFound" +} + +# LDAP | Server People OU Check and Create +ldap_people_ou_search=$(check_and_create_ldap_ou People) +[ "${ldap_people_ou_search}" == "PeopleOUNotFound" ] && ldapadd -x -D "cn=admin,dc=${BASE_DN%%.*},dc=${BASE_DN#*.}" -w "${LDAP_ADMIN_PASSWORD}" -f "${LDAP_DIR}/ouPeople.ldif" +[ "${ldap_people_ou_search}" == "PeopleOUFound" ] && echo "LDAP OU 'People' already exists. Skipping." + +# LDAP | Server Groups OU Check and Create +ldap_groups_ou_search=$(check_and_create_ldap_ou Groups) +[ "${ldap_groups_ou_search}" == "GroupsOUNotFound" ] && ldapadd -x -D "cn=admin,dc=${BASE_DN%%.*},dc=${BASE_DN#*.}" -w "${LDAP_ADMIN_PASSWORD}" -f "${LDAP_DIR}/ouGroups.ldif" +[ "${ldap_groups_ou_search}" == "GroupsOUFound" ] && echo "LDAP OU 'Groups' already exists. Skipping." + +# Creating LDAP Group on the LDAP Server + +# LDAP | Group File +echo "dn: cn=${LDAP_GROUP},ou=Groups,dc=${BASE_DN%%.*},dc=${BASE_DN#*.} +objectClass: posixGroup +cn: ${LDAP_GROUP} +gidNumber: 5000" > "${LDAP_DIR}/group.ldif" + +# LDAP Group Search +ldap_group_dn="cn=${LDAP_GROUP},ou=Groups,dc=${BASE_DN%%.*},dc=${BASE_DN#*.}" +ldap_group_search_result=$(ldapsearch -x -D "cn=admin,dc=${BASE_DN%%.*},dc=${BASE_DN#*.}" -w "${LDAP_ADMIN_PASSWORD}" -b "${ldap_group_dn}" "(cn=${LDAP_GROUP})" 2>&1) + +# Check if LDAP Group exists +if echo "${ldap_group_search_result}" | grep -q "dn: ${ldap_group_dn}," +then + echo "LDAP Group '${LDAP_GROUP}' already exists. Skipping." + ldap_group_search="GroupFound" +else + echo "LDAP Group '${LDAP_GROUP}' not found. Creating..." + ldapadd -x -D "cn=admin,dc=${BASE_DN%%.*},dc=${BASE_DN#*.}" -w "${LDAP_ADMIN_PASSWORD}" -f "${LDAP_DIR}/group.ldif" + ldap_group_search="GroupNotFound" +fi + +# Creating LDAP User on the LDAP Server + +# Generate LDAP Password Hash +ldap_hashed_password=$(slappasswd -s "${LDAP_USER_PASSWORD}") + +# LDAP | User File +echo "dn: uid=${LDAP_USER},ou=People,dc=${BASE_DN%%.*},dc=${BASE_DN#*.} +objectClass: inetOrgPerson +objectClass: posixAccount +objectClass: shadowAccount +uid: ${LDAP_USER} +sn: ${LDAP_USER} +givenName: ${LDAP_USER} +cn: ${LDAP_USER} +displayName: ${LDAP_USER} +uidNumber: 10000 +gidNumber: 5000 +userPassword: ${ldap_hashed_password} +gecos: ${LDAP_USER} +loginShell: /bin/bash +homeDirectory: /home/${LDAP_USER}" > "${LDAP_DIR}/users.ldif" + +# LDAP User Search +ldap_user_dn="uid=${LDAP_USER},ou=People,dc=${BASE_DN%%.*},dc=${BASE_DN#*.}" +ldap_user_search_result=$(ldapsearch -x -D "cn=admin,dc=${BASE_DN%%.*},dc=${BASE_DN#*.}" -w "${LDAP_ADMIN_PASSWORD}" -b "${ldap_user_dn}" uid cn 2>&1) + +# Check if LDAP User exists +if echo "${ldap_user_search_result}" | grep -q "dn: ${ldap_user_dn}," +then + echo "LDAP User '${LDAP_USER}' already exists. Skipping." + ldap_user_search="UserFound" +else + echo "LDAP User '${LDAP_USER}' not found. Creating..." + ldapadd -x -D "cn=admin,dc=${BASE_DN%%.*},dc=${BASE_DN#*.}" -w "${LDAP_ADMIN_PASSWORD}" -f "${LDAP_DIR}/users.ldif" + ldap_user_search="UserNotFound" +fi diff --git a/modules/landing_zone_vsi/templates/ldap_user_data.tpl b/modules/landing_zone_vsi/templates/ldap_user_data.tpl new file mode 100644 index 00000000..fa9412ca --- /dev/null +++ b/modules/landing_zone_vsi/templates/ldap_user_data.tpl @@ -0,0 +1,21 @@ +#!/usr/bin/bash + +################################################### +# Copyright (C) IBM Corp. 2023 All Rights Reserved. +# Licensed under the Apache License v2.0 +################################################### + +logfile=/tmp/user_data.log +echo "Export LDAP user data (variable values)" +echo "START $(date '+%Y-%m-%d %H:%M:%S')" >> $logfile + +%EXPORT_USER_DATA% +#input parameters +ldap_basedns="${ldap_basedns}" +ldap_admin_password="${ldap_admin_password}" +cluster_prefix="${cluster_prefix}" +ldap_user="${ldap_user}" +ldap_user_password="${ldap_user_password}" +dns_domain="${dns_domain}" + +echo "END $(date '+%Y-%m-%d %H:%M:%S')" >> $logfile diff --git a/modules/landing_zone_vsi/templates/login_user_data.tpl b/modules/landing_zone_vsi/templates/login_user_data.tpl index b6e391e1..30f63d28 100644 --- a/modules/landing_zone_vsi/templates/login_user_data.tpl +++ b/modules/landing_zone_vsi/templates/login_user_data.tpl @@ -1,29 +1,26 @@ #!/usr/bin/bash - ################################################### # Copyright (C) IBM Corp. 2023 All Rights Reserved. # Licensed under the Apache License v2.0 ################################################### -#!/usr/bin/env bash -if grep -E -q "CentOS|Red Hat" /etc/os-release -then - USER=vpcuser -elif grep -q "Ubuntu" /etc/os-release -then - USER=ubuntu -fi -sed -i -e "s/^/no-port-forwarding,no-agent-forwarding,no-X11-forwarding,command=\"echo \'Please login as the user \\\\\"$USER\\\\\" rather than the user \\\\\"root\\\\\".\';echo;sleep 5; exit 142\" /" /root/.ssh/authorized_keys - -# input parameters -echo "${bastion_public_key_content}" >> /~/.ssh/authorized_keys -echo "${login_public_key_content}" >> ~/.ssh/authorized_keys -echo "StrictHostKeyChecking no" >> ~/.ssh/config -echo "${login_private_key_content}" > ~/.ssh/id_rsa -chmod 600 ~/.ssh/id_rsa +logfile=/tmp/user_data.log +echo "Export user data (variable values)" +echo "START $(date '+%Y-%m-%d %H:%M:%S')" >> $logfile -# network setup -echo "DOMAIN=${login_dns_domain}" >> "/etc/sysconfig/network-scripts/ifcfg-${login_interfaces}" -echo "MTU=9000" >> "/etc/sysconfig/network-scripts/ifcfg-${login_interfaces}" -chage -I -1 -m 0 -M 99999 -E -1 -W 14 vpcuser -systemctl restart NetworkManager +%EXPORT_USER_DATA% +#input parameters +network_interface=${network_interface} +dns_domain="${dns_domain}" +cluster_private_key_content="${cluster_private_key_content}" +cluster_public_key_content="${cluster_public_key_content}" +mount_path="${mount_path}" +enable_ldap="${enable_ldap}" +network_interface=""${network_interface}"" +rc_cidr_block="${rc_cidr_block}" +rc_cidr_block_1="${rc_cidr_block_1}" +cluster_prefix="${cluster_prefix}" +ldap_server_ip="${ldap_server_ip}" +ldap_basedns="${ldap_basedns}" +hyperthreading="${hyperthreading}" +echo "END $(date '+%Y-%m-%d %H:%M:%S')" >> $logfile diff --git a/modules/landing_zone_vsi/templates/login_vsi.sh b/modules/landing_zone_vsi/templates/login_vsi.sh new file mode 100644 index 00000000..c2b82e0a --- /dev/null +++ b/modules/landing_zone_vsi/templates/login_vsi.sh @@ -0,0 +1,327 @@ +#!/bin/sh +# shellcheck disable=all + +################################################### +# Copyright (C) IBM Corp. 2023 All Rights Reserved. +# Licensed under the Apache License v2.0 +################################################### + +#variables + +logfile="/tmp/user_data.log" + +LSF_TOP="/opt/ibm/lsf" +LSF_CONF=$LSF_TOP/conf +LSF_HOSTS_FILE="/etc/hosts" + +nfs_server_with_mount_path=${mount_path} + + +# Setup logs for user data +echo "START $(date '+%Y-%m-%d %H:%M:%S')" >> $logfile + +# Disallow root login +sed -i -e "s/^/no-port-forwarding,no-agent-forwarding,no-X11-forwarding,command=\"echo \'Please login as the user \\\\\"lsfadmin or vpcuser\\\\\" rather than the user \\\\\"root\\\\\".\';echo;sleep 5; exit 142\" /" /root/.ssh/authorized_keys + +# echo "DOMAIN=\"$dns_domain\"" >> "/etc/sysconfig/network-scripts/ifcfg-eth0" +echo "DOMAIN=\"$dns_domain\"" >> "/etc/sysconfig/network-scripts/ifcfg-${network_interface}" + + +# Setup lsfadmin user +# Updates the lsfadmin user as never expire +chage -I -1 -m 0 -M 99999 -E -1 -W 14 lsfadmin +# Setup ssh +lsfadmin_home_dir="/home/lsfadmin" +lsfadmin_ssh_dir="${lsfadmin_home_dir}/.ssh" +mkdir -p ${lsfadmin_ssh_dir} + +# Change for RHEL / Ubuntu compute image. +if grep -q "NAME=\"Red Hat Enterprise Linux\"" /etc/os-release; then + cp /home/vpcuser/.ssh/authorized_keys "${lsfadmin_ssh_dir}/authorized_keys" +elif grep -q "NAME=\"Ubuntu\"" /etc/os-release; then + cp /home/ubuntu/.ssh/authorized_keys "${lsfadmin_ssh_dir}/authorized_keys" + sudo cp /home/ubuntu/.profile "{$lsfadmin_home_dir}" +else + echo "Provided OS distribution not match, provide either RHEL or Ubuntu" >> $logfile +fi + +# Setup Network configuration +# Change the MTU setting as this is required for setting mtu as 9000 for communication to happen between clusters +if grep -q "NAME=\"Red Hat Enterprise Linux\"" /etc/os-release; then + # Replace the MTU value in the Netplan configuration + echo "MTU=9000" >> "/etc/sysconfig/network-scripts/ifcfg-${network_interface}" + echo "DOMAIN=\"${dns_domain}\"" >> "/etc/sysconfig/network-scripts/ifcfg-${network_interface}" + # Change the MTU setting as 9000 at router level. + gateway_ip=$(ip route | grep default | awk '{print $3}' | head -n 1) + echo "${rc_cidr_block} via $gateway_ip dev ${network_interface} metric 0 mtu 9000" >> /etc/sysconfig/network-scripts/route-eth0 + systemctl restart NetworkManager +elif grep -q "NAME=\"Ubuntu\"" /etc/os-release; then + net_int=$(basename /sys/class/net/en*) + netplan_config="/etc/netplan/50-cloud-init.yaml" + gateway_ip=$(ip route | grep default | awk '{print $3}' | head -n 1) + cidr_range=$(ip route show | grep "kernel" | awk '{print $1}' | head -n 1) + usermod -s /bin/bash lsfadmin + # Replace the MTU value in the Netplan configuration + if ! grep -qE "^[[:space:]]*mtu: 9000" $netplan_config; then + echo "MTU 9000 Packages entries not found" + # Append the MTU configuration to the Netplan file + sudo sed -i '/'"$net_int"':/a\ mtu: 9000' $netplan_config + sudo sed -i '/dhcp4: true/a \ nameservers:\n search: ['"$dns_domain"']' $netplan_config + sudo sed -i '/'"$net_int"':/a\ routes:\n - to: '"$cidr_range"'\n via: '"$gateway_ip"'\n metric: 100\n mtu: 9000' $netplan_config + sudo netplan apply + echo "MTU set to 9000 on Netplan." + else + echo "MTU entry already exists in Netplan. Skipping." + fi +fi + +echo "${cluster_public_key_content}" >> "${lsfadmin_ssh_dir}/authorized_keys" +echo "${cluster_private_key_content}" >> "${lsfadmin_ssh_dir}/id_rsa" +echo "StrictHostKeyChecking no" >> "${lsfadmin_ssh_dir}/config" +chmod 600 "${lsfadmin_ssh_dir}/authorized_keys" +chmod 600 "${lsfadmin_ssh_dir}/id_rsa" +chmod 700 ${lsfadmin_ssh_dir} +chown -R lsfadmin:lsfadmin ${lsfadmin_ssh_dir} +echo "SSH key setup for lsfadmin user is completed" >> $logfile + + +# Setup root user +root_ssh_dir="/root/.ssh" +echo "${cluster_public_key_content}" >> $root_ssh_dir/authorized_keys +echo "StrictHostKeyChecking no" >> $root_ssh_dir/config +echo "cluster ssh key has been added to root user" >> $logfile + +echo "$hyperthreading" +if [ "$hyperthreading" == true ]; then + ego_define_ncpus="threads" +else + ego_define_ncpus="cores" + cat << 'EOT' > /root/lsf_hyperthreading +#!/bin/sh +for vcpu in $(cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list | cut -s -d- -f2 | cut -d- -f2 | uniq); do + echo "0" > "/sys/devices/system/cpu/cpu"$vcpu"/online" +done +EOT + chmod 755 /root/lsf_hyperthreading + command="/root/lsf_hyperthreading" + sh $command && (crontab -l 2>/dev/null; echo "@reboot $command") | crontab - +fi + +# Setup LSF +echo "Setting LSF share." >> $logfile +# Setup file share +if [ -n "${nfs_server_with_mount_path}" ]; then + echo "File share ${nfs_server_with_mount_path} found" >> $logfile + nfs_client_mount_path="/mnt/lsf" + rm -rf "${nfs_client_mount_path}" + rm -rf /opt/ibm/lsf/conf/ + rm -rf /opt/ibm/lsf/work/ + mkdir -p "${nfs_client_mount_path}" + # Mount LSF TOP + mount -t nfs4 -o sec=sys,vers=4.1 "$nfs_server_with_mount_path" "$nfs_client_mount_path" >> $logfile + # Verify mount + if mount | grep "$nfs_client_mount_path"; then + echo "Mount found" >> $logfile + else + echo "No mount found, exiting!" >> $logfile + exit 1 + fi + # Update mount to fstab for automount + echo "$nfs_server_with_mount_path $nfs_client_mount_path nfs rw,sec=sys,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,_netdev 0 0 " >> /etc/fstab + for dir in conf work; do + mv "${LSF_TOP}/$dir" "${nfs_client_mount_path}" + ln -fs "${nfs_client_mount_path}/$dir" "${LSF_TOP}" + chown -R lsfadmin:root "${LSF_TOP}" + done +else + echo "No mount point value found, exiting!" >> $logfile + exit 1 +fi +echo "Setting LSF share is completed." >> $logfile + +echo "source ${LSF_CONF}/profile.lsf" >> "${lsfadmin_home_dir}"/.bashrc +echo "source ${LSF_CONF}/profile.lsf" >> /root/.bashrc +echo "profile setup copy complete" >> $logfile + +# Pause execution for 30 seconds +sleep 30 + +# Display the contents of /etc/resolv.conf before changes +echo "Contents of /etc/resolv.conf before changes:" +cat /etc/resolv.conf + +# Restart the NetworkManager service +sudo systemctl restart NetworkManager + +# Toggle networking off and on using nmcli +sudo nmcli networking off +sudo nmcli networking on + +# Display the updated contents of /etc/resolv.conf +echo "Contents of /etc/resolv.conf after changes:" +cat /etc/resolv.conf +#python3 -c "import ipaddress; print('\n'.join([str(ip) + ' ${cluster_prefix}-' + str(ip).replace('.', '-') for ip in ipaddress.IPv4Network('${rc_cidr_block_1}')]) + '\n' + '\n'.join([str(ip) + ' ${cluster_prefix}-' + str(ip).replace('.', '-') for ip in ipaddress.IPv4Network('${rc_cidr_block_2}')]))" >> "$LSF_HOSTS_FILE" + +#Hostname resolution - login node to management nodes +sleep 300 +ls /mnt/lsf +ls -ltr /mnt/lsf +cp /mnt/lsf/conf/hosts /etc/hosts + +# Ldap Configuration: +enable_ldap="${enable_ldap}" +ldap_server_ip="${ldap_server_ip}" +base_dn="${ldap_basedns}" + +# Setting up the LDAP configuration +if [ "$enable_ldap" = "true" ]; then + + # Detect the operating system + if grep -q "NAME=\"Red Hat Enterprise Linux\"" /etc/os-release; then + + # Detect RHEL version + rhel_version=$(grep -oE 'release [0-9]+' /etc/redhat-release | awk '{print $2}') + + if [ "$rhel_version" == "8" ]; then + echo "Detected RHEL 8. Proceeding with LDAP client configuration...." >> $logfile + + # Allow Password authentication + sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config + systemctl restart sshd + + # Configure LDAP authentication + authconfig --enableldap --enableldapauth \ + --ldapserver=ldap://"${ldap_server_ip}" \ + --ldapbasedn="dc=${base_dn%%.*},dc=${base_dn#*.}" \ + --enablemkhomedir --update + + # Check the exit status of the authconfig command + if [ $? -eq 0 ]; then + echo "LDAP Authentication enabled successfully." >> $logfile + else + echo "Failed to enable LDAP and LDAP Authentication." >> $logfile + exit 1 + fi + + # Update LDAP Client configurations in nsswitch.conf + sed -i -e 's/^passwd:.*$/passwd: files ldap/' -e 's/^shadow:.*$/shadow: files ldap/' -e 's/^group:.*$/group: files ldap/' /etc/nsswitch.conf # pragma: allowlist secret + + # Update PAM configuration files + sed -i -e '/^auth/d' /etc/pam.d/password-auth + sed -i -e '/^auth/d' /etc/pam.d/system-auth + + auth_line="\nauth required pam_env.so\n\ +auth sufficient pam_unix.so nullok try_first_pass\n\ +auth requisite pam_succeed_if.so uid >= 1000 quiet_success\n\ +auth sufficient pam_ldap.so use_first_pass\n\ +auth required pam_deny.so" + + echo -e "$auth_line" | tee -a /etc/pam.d/password-auth /etc/pam.d/system-auth + + # Copy 'password-auth' settings to 'sshd' + cat /etc/pam.d/password-auth > /etc/pam.d/sshd + + # Configure nslcd + cat < /etc/nslcd.conf +uid nslcd +gid ldap +uri ldap://${ldap_server_ip}/ +base dc=${base_dn%%.*},dc=${base_dn#*.} +EOF + + # Restart nslcd and nscd service + systemctl restart nslcd + systemctl restart nscd + + # Enable nslcd and nscd service + systemctl enable nslcd + systemctl enable nscd + + # Validate the LDAP configuration + if ldapsearch -x -H ldap://"${ldap_server_ip}"/ -b "dc=${base_dn%%.*},dc=${base_dn#*.}" > /dev/null; then + echo "LDAP configuration completed successfully !!" >> $logfile + else + echo "LDAP configuration failed !!" >> $logfile + exit 1 + fi + + # Make LSF commands available for every user. + echo ". ${LSF_CONF}/profile.lsf" >> /etc/bashrc + source /etc/bashrc + else + echo "This script is designed for RHEL 8. Detected RHEL version: $rhel_version. Exiting." >> $logfile + exit 1 + fi + elif grep -q "NAME=\"Ubuntu\"" /etc/os-release; then + + echo "Detected as Ubuntu. Proceeding with LDAP client configuration..." >> $logfile + + # Update package repositories + sudo apt-get update -y + + # Update SSH configuration to allow password authentication + sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config + sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloudimg-settings.conf + sudo systemctl restart ssh + + # Create preseed file for LDAP configuration + cat > debconf-ldap-preseed.txt <> /etc/bash.bashrc + source /etc/bash.bashrc + + # Restart and enable the service + systemctl restart nscd + systemctl restart nslcd + + # Enable nslcd and nscd service + systemctl enable nslcd + systemctl enable nscd + + sleep 5 + + # Validate the LDAP client service status + if sudo systemctl is-active --quiet nscd; then + echo "LDAP client configuration completed successfully !!" >> $logfile + else + echo "LDAP client configuration failed. nscd service is not running." >> $logfile + exit 1 + fi + else + echo -e "debconf-ldap-preseed.txt Not found. Skipping LDAP client configuration." >> $logfile + fi + else + echo "This script is designed for Ubuntu 22 and installation is not supporting. Exiting." >> $logfile + fi +fi diff --git a/modules/landing_zone_vsi/templates/lsf_management.sh b/modules/landing_zone_vsi/templates/lsf_management.sh new file mode 100644 index 00000000..40b072b7 --- /dev/null +++ b/modules/landing_zone_vsi/templates/lsf_management.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# shellcheck disable=all + +################################################### +# Copyright (C) IBM Corp. 2023 All Rights Reserved. +# Licensed under the Apache License v2.0 +################################################### + +# Local variable declaration +logfile="/tmp/user_data.log" + +# Setup logs for user data +echo "START $(date '+%Y-%m-%d %H:%M:%S')" >> $logfile + +echo "umask=$(umask)" >> $logfile + +# Disallow root login +sed -i -e "s/^/no-port-forwarding,no-agent-forwarding,no-X11-forwarding,command=\"echo \'Please login as the user \\\\\"lsfadmin or vpcuser\\\\\" rather than the user \\\\\"root\\\\\".\';echo;sleep 5; exit 142\" /" /root/.ssh/authorized_keys + +# Setup Network configuration +# Change the MTU setting as this is required for setting mtu as 9000 for communication to happen between clusters +echo "MTU=9000" >> "/etc/sysconfig/network-scripts/ifcfg-${network_interface}" +echo "DOMAIN=\"$dns_domain\"" >> "/etc/sysconfig/network-scripts/ifcfg-${network_interface}" + +# Change the MTU setting as 9000 at router level. +gateway_ip=$(ip route | grep default | awk '{print $3}' | head -n 1) +echo "${rc_cidr_block} via $gateway_ip dev ${network_interface} metric 0 mtu 9000" >> /etc/sysconfig/network-scripts/route-"${network_interface}" + +systemctl restart NetworkManager + +echo 1 > /proc/sys/vm/overcommit_memory # tt reports many failures of memory allocation at fork(). why? +{ + echo 'vm.overcommit_memory=1' + echo 'net.core.rmem_max=26214400' + echo 'net.core.rmem_default=26214400' + echo 'net.core.wmem_max=26214400' + echo 'net.core.wmem_default=26214400' + echo 'net.ipv4.tcp_fin_timeout = 5' + echo 'net.core.somaxconn = 8000' +} > /etc/sysctl.conf +sysctl -p /etc/sysctl.conf + +if [ ! "$hyperthreading" == true ]; then + for vcpu in $(cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list | cut -s -d- -f2 | cut -d- -f2 | uniq); do + echo 0 > /sys/devices/system/cpu/cpu"$vcpu"/online + done +fi + +# Setup lsfadmin user +# Updates the lsfadmin user as never expire +chage -I -1 -m 0 -M 99999 -E -1 -W 14 lsfadmin +# Setup ssh +lsfadmin_home_dir="/home/lsfadmin" +lsfadmin_ssh_dir="${lsfadmin_home_dir}/.ssh" +mkdir -p ${lsfadmin_ssh_dir} +cp /home/vpcuser/.ssh/authorized_keys "${lsfadmin_ssh_dir}/authorized_keys" +echo "${cluster_public_key_content}" >> "${lsfadmin_ssh_dir}/authorized_keys" +echo "${cluster_private_key_content}" >> "${lsfadmin_ssh_dir}/id_rsa" +echo "StrictHostKeyChecking no" >> "${lsfadmin_ssh_dir}/config" +chmod 600 "${lsfadmin_ssh_dir}/authorized_keys" +chmod 600 "${lsfadmin_ssh_dir}/id_rsa" +chmod 700 ${lsfadmin_ssh_dir} +chown -R lsfadmin:lsfadmin ${lsfadmin_ssh_dir} +echo "SSH key setup for lsfadmin user is completed" >> $logfile + +# Setup root user +root_ssh_dir="/root/.ssh" +echo "${cluster_public_key_content}" >> $root_ssh_dir/authorized_keys +echo "StrictHostKeyChecking no" >> $root_ssh_dir/config +echo "cluster ssh key has been added to root user" >> $logfile diff --git a/modules/landing_zone_vsi/templates/management_user_data.tpl b/modules/landing_zone_vsi/templates/management_user_data.tpl index f0f172f1..3c441833 100644 --- a/modules/landing_zone_vsi/templates/management_user_data.tpl +++ b/modules/landing_zone_vsi/templates/management_user_data.tpl @@ -5,25 +5,18 @@ # Licensed under the Apache License v2.0 ################################################### -#!/usr/bin/env bash -if grep -E -q "CentOS|Red Hat" /etc/os-release -then - USER=vpcuser -elif grep -q "Ubuntu" /etc/os-release -then - USER=ubuntu -fi -sed -i -e "s/^/no-port-forwarding,no-agent-forwarding,no-X11-forwarding,command=\"echo \'Please login as the user \\\\\"$USER\\\\\" rather than the user \\\\\"root\\\\\".\';echo;sleep 5; exit 142\" /" /root/.ssh/authorized_keys +logfile=/tmp/user_data.log +echo "Export user data (variable values)" +echo "START $(date '+%Y-%m-%d %H:%M:%S')" >> $logfile -# input parameters -echo "${bastion_public_key_content}" >> ~/.ssh/authorized_keys -echo "${management_public_key_content}" >> ~/.ssh/authorized_keys -echo "StrictHostKeyChecking no" >> ~/.ssh/config -echo "${management_private_key_content}" > ~/.ssh/id_rsa -chmod 600 ~/.ssh/id_rsa +### EXPORT_USER_DATA ### -# network setup -echo "DOMAIN=${management_dns_domain}" >> "/etc/sysconfig/network-scripts/ifcfg-${management_interfaces}" -echo "MTU=9000" >> "/etc/sysconfig/network-scripts/ifcfg-${management_interfaces}" -chage -I -1 -m 0 -M 99999 -E -1 -W 14 vpcuser -systemctl restart NetworkManager +#input parameters +rc_cidr_block="${rc_cidr_block}" +cluster_private_key_content="${cluster_private_key_content}" +cluster_public_key_content="${cluster_public_key_content}" +hyperthreading="${hyperthreading}" +network_interface=${network_interface} +dns_domain="${dns_domain}" + +echo "END $(date '+%Y-%m-%d %H:%M:%S')" >> $logfile diff --git a/modules/landing_zone_vsi/variables.tf b/modules/landing_zone_vsi/variables.tf index 6efbb2ae..9c776a4b 100644 --- a/modules/landing_zone_vsi/variables.tf +++ b/modules/landing_zone_vsi/variables.tf @@ -1,11 +1,11 @@ ############################################################################## # Offering Variations ############################################################################## -variable "storage_type" { - type = string - default = "scratch" - description = "Select the required storage type(scratch/persistent/eval)." -} +# variable "storage_type" { +# type = string +# default = "scratch" +# description = "Select the required storage type(scratch/persistent/eval)." +# } ############################################################################## # Account Variables @@ -56,16 +56,15 @@ variable "vpc_id" { description = "ID of an existing VPC in which the cluster resources will be deployed." } -variable "placement_group_ids" { - type = string - default = null - description = "VPC placement group ids" -} - ############################################################################## # Access Variables ############################################################################## +variable "bastion_fip" { + type = string + description = "Bastion FIP." +} + variable "bastion_security_group_id" { type = string description = "Bastion security group id." @@ -78,45 +77,24 @@ variable "bastion_public_key_content" { description = "Bastion security group id." } -############################################################################## -# Compute Variables -############################################################################## - -variable "login_subnets" { - type = list(object({ - name = string - id = string - zone = string - cidr = string - })) - default = [] - description = "Subnets to launch the login hosts." +variable "cluster_user" { + type = string + description = "Linux user for cluster administration." } -variable "login_ssh_keys" { - type = list(string) - description = "The key pair to use to launch the login host." +variable "compute_private_key_content" { + type = string + description = "Compute private key content" } -variable "login_image_name" { +variable "bastion_private_key_content" { type = string - default = "ibm-redhat-8-6-minimal-amd64-5" - description = "Image name to use for provisioning the login instances." + description = "Bastion private key content" } -variable "login_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 1 - }] - description = "Number of instances to be launched for login." -} +############################################################################## +# Compute Variables +############################################################################## variable "compute_subnets" { type = list(object({ @@ -124,6 +102,7 @@ variable "compute_subnets" { id = string zone = string cidr = string + crn = string })) default = [] description = "Subnets to launch the compute host." @@ -136,99 +115,143 @@ variable "compute_ssh_keys" { variable "management_image_name" { type = string - default = "ibm-redhat-8-6-minimal-amd64-5" + default = "hpcaas-lsf10-rhel88-v3" description = "Image name to use for provisioning the management cluster instances." } -variable "management_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 3 - }] - description = "Number of instances to be launched for management." -} - -variable "static_compute_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 0 - }] - description = "Min Number of instances to be launched for compute cluster." -} - -variable "dynamic_compute_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 250 - }] - description = "MaxNumber of instances to be launched for compute cluster." -} - variable "compute_image_name" { type = string - default = "ibm-redhat-8-6-minimal-amd64-5" + default = "hpcaas-lsf10-rhel88-compute-v2" description = "Image name to use for provisioning the compute cluster instances." } +variable "login_image_name" { + type = string + default = "hpcaas-lsf10-rhel88-compute-v2" + description = "Image name to use for provisioning the login instance." +} + ############################################################################## -# Scale Storage Variables +# DNS Template Variables ############################################################################## -variable "storage_subnets" { +variable "dns_domain_names" { + type = object({ + compute = string + #storage = string + #protocol = string + }) + default = { + compute = "comp.com" + storage = "strg.com" + protocol = "ces.com" + } + description = "IBM Cloud HPC DNS domain names." +} + +############################################################################## +# Encryption Variables +############################################################################## + +# TODO: landing-zone-vsi limitation to opt out encryption +variable "kms_encryption_enabled" { + description = "Enable Key management" + type = bool + default = true +} + +variable "boot_volume_encryption_key" { + type = string + default = null + description = "CRN of boot volume encryption key" +} + +variable "cluster_id" { + type = string + description = "Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same contract. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment." +} + +variable "contract_id" { + type = string + sensitive = true + description = "Ensure that you have received the contract ID from IBM technical sales. Contract ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_)." +} + +variable "hyperthreading_enabled" { + type = bool + default = true + description = "Setting this to true will enable hyper-threading in the compute nodes of the cluster (default). Otherwise, hyper-threading will be disabled." +} + +variable "enable_app_center" { + type = bool + default = false + description = "Set to true to enable the IBM Spectrum LSF Application Center GUI (default: false). [System requirements](https://www.ibm.com/docs/en/slac/10.2.0?topic=requirements-system-102-fix-pack-14) for IBM Spectrum LSF Application Center Version 10.2 Fix Pack 14." +} + +variable "app_center_gui_pwd" { + type = string + sensitive = true + default = "" + description = "Password for IBM Spectrum LSF Application Center GUI. Note: Password should be at least 8 characters, must have one number, one lowercase letter, one uppercase letter, and at least one special character." +} + +variable "management_node_count" { + type = number + default = 3 + description = "Number of management nodes. This is the total number of management nodes. Enter a value between 1 and 10." + validation { + condition = 1 <= var.management_node_count && var.management_node_count <= 10 + error_message = "Input \"management_node_count\" must be must be greater than or equal to 1 and less than or equal to 10." + } +} + +variable "management_node_instance_type" { + type = string + default = "bx2-16x64" + description = "Specify the virtual server instance profile type to be used to create the management nodes for the IBM Cloud HPC cluster. For choices on profile types, see [Instance profiles](https://cloud.ibm.com/docs/vpc?topic=vpc-profiles)." + validation { + condition = can(regex("^[^\\s]+-[0-9]+x[0-9]+", var.management_node_instance_type)) + error_message = "The profile must be a valid profile name." + } +} + +variable "share_path" { + type = string + description = "Provide the exact path to where the VPC file share needs to be mounted" +} + +variable "mount_path" { type = list(object({ - name = string - id = string - zone = string - cidr = string + mount_path = string, + size = optional(number), + iops = optional(number), + nfs_share = optional(string) })) - default = [] - description = "Subnets to launch the storage host." + description = "Provide the path for the vpc file share to be mounted on to the HPC Cluster nodes" } -variable "storage_ssh_keys" { +variable "file_share" { type = list(string) - description = "The key pair to use to launch the storage cluster host." + description = "VPC file share mount points considering the ip address and the file share name" } -variable "storage_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "bx2-2x8" - count = 3 - }] - description = "Number of instances to be launched for storage cluster." +variable "login_private_ips" { + description = "Login private IPs" + type = string } -variable "storage_image_name" { +variable "login_node_instance_type" { type = string - default = "ibm-redhat-8-6-minimal-amd64-5" - description = "Image name to use for provisioning the storage cluster instances." + default = "bx2-2x8" + description = "Specify the virtual server instance profile type to be used to create the login node for the IBM Cloud HPC cluster. For choices on profile types, see [Instance profiles](https://cloud.ibm.com/docs/vpc?topic=vpc-profiles)." + validation { + condition = can(regex("^[^\\s]+-[0-9]+x[0-9]+", var.login_node_instance_type)) + error_message = "The profile must be a valid profile name." + } } -variable "protocol_subnets" { +variable "bastion_subnets" { type = list(object({ name = string id = string @@ -239,71 +262,141 @@ variable "protocol_subnets" { description = "Subnets to launch the bastion host." } -variable "protocol_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "bx2-2x8" - count = 2 - }] - description = "Number of instances to be launched for protocol hosts." -} - -variable "nsd_details" { - type = list( - object({ - profile = string - capacity = optional(number) - iops = optional(number) - }) - ) - default = [{ - profile = "custom" - size = 100 - iops = 100 - }] - description = "NSD details" +variable "ssh_keys" { + type = list(string) + description = "The key pair to use to access the host." +} + +variable "enable_ldap" { + type = bool + default = false + description = "Set this option to true to enable LDAP for IBM Cloud HPC, with the default value set to false." +} + +variable "ldap_basedns" { + type = string + default = "hpcaas.com" + description = "The dns domain name is used for configuring the LDAP server. If an LDAP server is already in existence, ensure to provide the associated DNS domain name." +} + +variable "ldap_server" { + type = string + default = "null" + description = "Provide the IP address for the existing LDAP server. If no address is given, a new LDAP server will be created." +} + +variable "ldap_admin_password" { + type = string + sensitive = true + default = "" + description = "The LDAP administrative password should be 8 to 20 characters long, with a mix of at least three alphabetic characters, including one uppercase and one lowercase letter. It must also include two numerical digits and at least one special character from (~@_+:) are required. It is important to avoid including the username in the password for enhanced security.[This value is ignored for an existing LDAP server]." +} + +variable "ldap_user_name" { + type = string + default = "" + description = "Custom LDAP User for performing cluster operations. Note: Username should be between 4 to 32 characters, (any combination of lowercase and uppercase letters).[This value is ignored for an existing LDAP server]" +} + +variable "ldap_user_password" { + type = string + sensitive = true + default = "" + description = "The LDAP user password should be 8 to 20 characters long, with a mix of at least three alphabetic characters, including one uppercase and one lowercase letter. It must also include two numerical digits and at least one special character from (~@_+:) are required.It is important to avoid including the username in the password for enhanced security.[This value is ignored for an existing LDAP server]." +} + +variable "ldap_vsi_profile" { + type = string + default = "cx2-2x4" + description = "Profile to be used for LDAP virtual server instance." +} + +variable "ldap_vsi_osimage_name" { + type = string + default = "ibm-ubuntu-22-04-3-minimal-amd64-1" + description = "Image name to be used for provisioning the LDAP instances." +} + +variable "ldap_primary_ip" { + type = list(string) + description = "List of LDAP primary IPs." } ############################################################################## -# DNS Template Variables +# High Availability ############################################################################## +variable "app_center_high_availability" { + type = bool + default = true + description = "Set to false to disable the IBM Spectrum LSF Application Center GUI High Availability (default: true) ." +} -variable "dns_domain_names" { +########################################################################### +# IBM Cloud Dababase for MySQL Instance variables +########################################################################### +variable "db_instance_info" { + description = "The IBM Cloud Database for MySQL information required to reference the PAC database." type = object({ - compute = string - storage = string - protocol = string + id = string + adminuser = string + adminpassword = string + hostname = string + port = number + certificate = string }) - default = { - compute = "comp.com" - storage = "strg.com" - protocol = "ces.com" - } - description = "IBM Cloud HPC DNS domain names." + default = null +} + +variable "storage_security_group_id" { + type = string + default = null + description = "Existing Scale storage security group id" } ############################################################################## -# Encryption Variables +# Observability Variables ############################################################################## -# TODO: landing-zone-vsi limitation to opt out encryption -variable "kms_encryption_enabled" { - description = "Enable Key management" +variable "observability_monitoring_enable" { + description = "Set true to enable IBM Cloud Monitoring instance provisioning." type = bool - default = true + default = false } -variable "boot_volume_encryption_key" { +variable "observability_monitoring_on_compute_nodes_enable" { + description = "Set true to enable IBM Cloud Monitoring on Compute Nodes." + type = bool + default = false +} + +variable "cloud_monitoring_access_key" { + description = "IBM Cloud Monitoring access key for agents to use" type = string - default = null - description = "CRN of boot volume encryption key" + sensitive = true } -############################################################################## -# TODO: Auth Server (LDAP/AD) Variables -############################################################################## +variable "cloud_monitoring_ingestion_url" { + description = "IBM Cloud Monitoring ingestion url for agents to use" + type = string +} + +variable "cloud_monitoring_prws_key" { + description = "IBM Cloud Monitoring Prometheus Remote Write ingestion key" + type = string + sensitive = true +} + +variable "cloud_monitoring_prws_url" { + description = "IBM Cloud Monitoring Prometheus Remote Write ingestion url" + type = string +} + +########################################################################### +# Existing Bastion Support variables +########################################################################### + +variable "bastion_instance_name" { + type = string + default = null + description = "Bastion instance name." +} diff --git a/modules/landing_zone_vsi/version.tf b/modules/landing_zone_vsi/version.tf index beb59d86..c917feb9 100644 --- a/modules/landing_zone_vsi/version.tf +++ b/modules/landing_zone_vsi/version.tf @@ -11,8 +11,3 @@ terraform { } } } - -provider "ibm" { - ibmcloud_api_key = var.ibmcloud_api_key - region = local.region -} diff --git a/modules/my_ip/README.md b/modules/my_ip/README.md new file mode 100644 index 00000000..dd08134b --- /dev/null +++ b/modules/my_ip/README.md @@ -0,0 +1,3 @@ +# This module will detect your current internet visible address. +# It can be useful if you want your IP to be added to the allowed_ips list, +# so you are sure you will be able to access the cluster. diff --git a/modules/my_ip/datasource.tf b/modules/my_ip/datasource.tf new file mode 100644 index 00000000..4e77519e --- /dev/null +++ b/modules/my_ip/datasource.tf @@ -0,0 +1,3 @@ +data "external" "my_ipv4" { + program = ["bash", "-c", "echo '{\"ip\":\"'\"$(curl -4 http://ifconfig.io)\"'\"}'"] +} diff --git a/modules/playbook/outputs.tf b/modules/my_ip/locals.tf similarity index 100% rename from modules/playbook/outputs.tf rename to modules/my_ip/locals.tf diff --git a/modules/my_ip/main.tf b/modules/my_ip/main.tf new file mode 100644 index 00000000..e69de29b diff --git a/modules/my_ip/outputs.tf b/modules/my_ip/outputs.tf new file mode 100644 index 00000000..c95e17f2 --- /dev/null +++ b/modules/my_ip/outputs.tf @@ -0,0 +1,4 @@ +output "my_cidr" { + value = data.external.my_ipv4.result.ip != "" ? ["${data.external.my_ipv4.result.ip}/32"] : [] + description = "The IPv4 in CIDR format (a '/32' is appended)" +} diff --git a/modules/my_ip/variables.tf b/modules/my_ip/variables.tf new file mode 100644 index 00000000..e69de29b diff --git a/modules/my_ip/version.tf b/modules/my_ip/version.tf new file mode 100644 index 00000000..d27453f4 --- /dev/null +++ b/modules/my_ip/version.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.3, < 1.7" + required_providers { + external = { + source = "hashicorp/external" + version = "2.3.3" + } + } +} diff --git a/modules/null/ldap_remote_exec/main.tf b/modules/null/ldap_remote_exec/main.tf new file mode 100644 index 00000000..e4b34c06 --- /dev/null +++ b/modules/null/ldap_remote_exec/main.tf @@ -0,0 +1,20 @@ +data "template_file" "ldap_connection_script" { + template = file("${path.module}/validate_ldap_connection.tpl") + vars = { + ldap_server = var.ldap_server + } +} + +# The resource is used to validate the existing LDAP server connection. +resource "null_resource" "validate_ldap_server_connection" { + count = var.enable_ldap == true && var.ldap_server != "null" ? 1 : 0 + connection { + type = "ssh" + user = var.login_user + private_key = var.login_private_key + host = var.login_host + } + provisioner "remote-exec" { + inline = [data.template_file.ldap_connection_script.rendered] + } +} diff --git a/modules/null/ldap_remote_exec/outputs.tf b/modules/null/ldap_remote_exec/outputs.tf new file mode 100644 index 00000000..e69de29b diff --git a/modules/null/ldap_remote_exec/validate_ldap_connection.tpl b/modules/null/ldap_remote_exec/validate_ldap_connection.tpl new file mode 100644 index 00000000..c765ac8b --- /dev/null +++ b/modules/null/ldap_remote_exec/validate_ldap_connection.tpl @@ -0,0 +1,9 @@ +#!/bin/bash +# shellcheck disable=SC2154 + +if openssl s_client -connect "${ldap_server}:389" /dev/null | grep -q 'CONNECTED'; then + echo "The connection to the existing LDAP server ${ldap_server} was successfully established." +else + echo "The connection to the existing LDAP server ${ldap_server} failed, please establish it." + exit 1 +fi diff --git a/modules/null/ldap_remote_exec/variables.tf b/modules/null/ldap_remote_exec/variables.tf new file mode 100644 index 00000000..e3863bde --- /dev/null +++ b/modules/null/ldap_remote_exec/variables.tf @@ -0,0 +1,24 @@ +variable "enable_ldap" { + type = bool + description = "Set this option to true to enable LDAP for IBM Cloud HPC, with the default value set to false." +} + +variable "ldap_server" { + type = string + description = "Provide the IP address for the existing LDAP server. If no address is given, a new LDAP server will be created." +} + +variable "login_host" { + description = "Login host to be used for ssh connectivity." + type = string +} + +variable "login_user" { + description = "Login user to be used for ssh connectivity." + type = string +} + +variable "login_private_key" { + description = "Login private key to be used for ssh connectivity." + type = string +} diff --git a/modules/null/ldap_remote_exec/version.tf b/modules/null/ldap_remote_exec/version.tf new file mode 100644 index 00000000..b6904c55 --- /dev/null +++ b/modules/null/ldap_remote_exec/version.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.3, < 1.7" + required_providers { + null = { + source = "hashicorp/null" + version = ">= 3.0.0" + } + template = { + source = "hashicorp/template" + version = "~> 2" + } + } +} diff --git a/modules/null/local_exec/main.tf b/modules/null/local_exec/main.tf new file mode 100644 index 00000000..b9351a60 --- /dev/null +++ b/modules/null/local_exec/main.tf @@ -0,0 +1,18 @@ +################################################### +# Copyright (C) IBM Corp. 2023 All Rights Reserved. +# Licensed under the Apache License v2.0 +################################################### + +/* + This module used to run null for IBM Cloud CLIs +*/ + +resource "null_resource" "local_exec" { + provisioner "local-exec" { + command = "ibmcloud login --apikey ${var.ibmcloud_api_key} -r ${var.region}; ${var.command}" + } + + triggers = { + value = var.trigger_resource_id + } +} diff --git a/modules/null/local_exec/outputs.tf b/modules/null/local_exec/outputs.tf new file mode 100644 index 00000000..e69de29b diff --git a/modules/null/local_exec/variables.tf b/modules/null/local_exec/variables.tf new file mode 100644 index 00000000..1cf7a665 --- /dev/null +++ b/modules/null/local_exec/variables.tf @@ -0,0 +1,20 @@ +variable "region" { + description = "The region with which IBM CLI login should happen." + type = string +} + +variable "ibmcloud_api_key" { + description = "IBM Cloud API key for the IBM Cloud account where the IBM Cloud HPC cluster needs to be deployed. For more information on how to create an API key, see [Managing user API keys](https://cloud.ibm.com/docs/account?topic=account-userapikey)." + type = string + sensitive = true +} + +variable "command" { + description = "This is the command to execute." + type = string +} + +variable "trigger_resource_id" { + description = "A map of arbitrary strings that, when changed, will force the null resource to be replaced, re-running any associated provisioners." + type = any +} diff --git a/modules/null/local_exec/version.tf b/modules/null/local_exec/version.tf new file mode 100644 index 00000000..2c8d6bef --- /dev/null +++ b/modules/null/local_exec/version.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.3, < 1.7" + required_providers { + null = { + source = "hashicorp/null" + version = ">= 3.0.0" + } + } +} diff --git a/modules/null/local_exec_script/main.tf b/modules/null/local_exec_script/main.tf new file mode 100644 index 00000000..40d21752 --- /dev/null +++ b/modules/null/local_exec_script/main.tf @@ -0,0 +1,6 @@ +resource "null_resource" "execute_local_script" { + provisioner "local-exec" { + command = "${var.script_path} ${var.script_arguments}" + environment = var.script_environment + } +} diff --git a/modules/null/local_exec_script/outputs.tf b/modules/null/local_exec_script/outputs.tf new file mode 100644 index 00000000..bdb2b8cd --- /dev/null +++ b/modules/null/local_exec_script/outputs.tf @@ -0,0 +1 @@ +# This empty file exists to suppress TFLint Warning on the terraform_standard_module_structure diff --git a/modules/null/local_exec_script/variables.tf b/modules/null/local_exec_script/variables.tf new file mode 100644 index 00000000..582ffe48 --- /dev/null +++ b/modules/null/local_exec_script/variables.tf @@ -0,0 +1,14 @@ +variable "script_path" { + description = "The path to the script to execute" + type = string +} + +variable "script_arguments" { + description = "The arguments to pass to the script" + type = string +} + +variable "script_environment" { + description = "The environment variables to pass to the script" + type = map(string) +} diff --git a/modules/null/local_exec_script/version.tf b/modules/null/local_exec_script/version.tf new file mode 100644 index 00000000..2c8d6bef --- /dev/null +++ b/modules/null/local_exec_script/version.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.3, < 1.7" + required_providers { + null = { + source = "hashicorp/null" + version = ">= 3.0.0" + } + } +} diff --git a/modules/null/remote_exec/main.tf b/modules/null/remote_exec/main.tf new file mode 100644 index 00000000..02147d34 --- /dev/null +++ b/modules/null/remote_exec/main.tf @@ -0,0 +1,30 @@ +################################################### +# Copyright (C) IBM Corp. 2023 All Rights Reserved. +# Licensed under the Apache License v2.0 +################################################### + +/* + This module used to run null for LSF utilities +*/ + +resource "null_resource" "remote_exec" { + count = length(var.cluster_host) + connection { + type = "ssh" + host = var.cluster_host[count.index] + user = var.cluster_user + private_key = var.cluster_private_key + bastion_host = var.login_host + bastion_user = var.login_user + bastion_private_key = var.login_private_key + timeout = var.timeout + } + + provisioner "remote-exec" { + inline = var.command + } + + triggers = { + build = timestamp() + } +} diff --git a/modules/null/remote_exec/outputs.tf b/modules/null/remote_exec/outputs.tf new file mode 100644 index 00000000..e69de29b diff --git a/modules/null/remote_exec/variables.tf b/modules/null/remote_exec/variables.tf new file mode 100644 index 00000000..aa5178e0 --- /dev/null +++ b/modules/null/remote_exec/variables.tf @@ -0,0 +1,40 @@ +variable "cluster_host" { + description = "Cluster hosts to be used for ssh connectivity." + type = list(string) +} + +variable "cluster_user" { + description = "Cluster user to be used for ssh connectivity." + type = string +} + +variable "cluster_private_key" { + description = "Cluster private key to be used for ssh connectivity." + type = string +} + +variable "login_host" { + description = "Login host to be used for ssh connectivity." + type = string +} + +variable "login_user" { + description = "Login user to be used for ssh connectivity." + type = string +} + +variable "login_private_key" { + description = "Login private key to be used for ssh connectivity." + type = string +} + +variable "command" { + description = "These are the list of commands to execute." + type = list(string) +} + +variable "timeout" { + description = "Timeout for connection attempts." + type = string + default = "5m" +} diff --git a/modules/null/remote_exec/version.tf b/modules/null/remote_exec/version.tf new file mode 100644 index 00000000..2c8d6bef --- /dev/null +++ b/modules/null/remote_exec/version.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.3, < 1.7" + required_providers { + null = { + source = "hashicorp/null" + version = ">= 3.0.0" + } + } +} diff --git a/modules/null/remote_exec_script/locals.tf b/modules/null/remote_exec_script/locals.tf new file mode 100644 index 00000000..a1ba863f --- /dev/null +++ b/modules/null/remote_exec_script/locals.tf @@ -0,0 +1,4 @@ +locals { + command_1 = "sh -c \"cd /tmp && ${var.with_bash ? "bash " : ""}${var.script_to_run}\"" + final_command = "${var.sudo_user != "" ? "sudo -i -u ${var.sudo_user} -- " : ""}${local.command_1}" +} diff --git a/modules/null/remote_exec_script/main.tf b/modules/null/remote_exec_script/main.tf new file mode 100644 index 00000000..52a3019e --- /dev/null +++ b/modules/null/remote_exec_script/main.tf @@ -0,0 +1,126 @@ +################################################### +# Copyright (C) IBM Corp. 2023 All Rights Reserved. +# Licensed under the Apache License v2.0 +################################################### + +/* + This module used to run null for LSF utilities +*/ + +resource "null_resource" "remote_exec_script_cp_files" { + count = length(var.cluster_host) * length(var.payload_files) + + provisioner "file" { + connection { + type = "ssh" + host = var.cluster_host[floor(count.index / length(var.payload_files))] + user = var.cluster_user + private_key = var.cluster_private_key + bastion_host = var.login_host + bastion_user = var.login_user + bastion_private_key = var.login_private_key + } + source = var.payload_files[count.index % length(var.payload_files)] + destination = "/tmp/${basename(var.payload_files[count.index % length(var.payload_files)])}" + } + + provisioner "local-exec" { + when = destroy + command = "true" + } + + triggers = { + trigger_string = var.trigger_string + } +} + +resource "null_resource" "remote_exec_script_cp_dirs" { + count = length(var.cluster_host) * length(var.payload_dirs) + + provisioner "file" { + connection { + type = "ssh" + host = var.cluster_host[floor(count.index / length(var.payload_dirs))] + user = var.cluster_user + private_key = var.cluster_private_key + bastion_host = var.login_host + bastion_user = var.login_user + bastion_private_key = var.login_private_key + } + source = var.payload_dirs[count.index % length(var.payload_dirs)] + destination = "/tmp/" + } + + provisioner "local-exec" { + when = destroy + command = "true" + } + + triggers = { + trigger_string = var.trigger_string + } +} + +resource "null_resource" "remote_exec_script_new_file" { + count = var.new_file_name != "" ? length(var.cluster_host) : 0 + + provisioner "file" { + connection { + type = "ssh" + host = var.cluster_host[count.index] + user = var.cluster_user + private_key = var.cluster_private_key + bastion_host = var.login_host + bastion_user = var.login_user + bastion_private_key = var.login_private_key + } + content = var.new_file_content + destination = "/tmp/${var.new_file_name}" + } + + provisioner "local-exec" { + when = destroy + command = "true" + } + + depends_on = [ + null_resource.remote_exec_script_cp_dirs # we may want to create the file in a subpath created with the cp_dirs + ] + triggers = { + trigger_string = var.trigger_string + } +} + +resource "null_resource" "remote_exec_script_run" { + count = length(var.cluster_host) + + provisioner "remote-exec" { + connection { + type = "ssh" + host = var.cluster_host[count.index] + user = var.cluster_user + private_key = var.cluster_private_key + bastion_host = var.login_host + bastion_user = var.login_user + bastion_private_key = var.login_private_key + } + inline = [ + "sh -c \"chmod +x /tmp/${var.script_to_run}\"", + "cd /tmp && ${local.final_command}" + ] + } + + provisioner "local-exec" { + when = destroy + command = "true" + } + + depends_on = [ + null_resource.remote_exec_script_cp_files, + null_resource.remote_exec_script_cp_dirs, + null_resource.remote_exec_script_new_file + ] + triggers = { + trigger_string = var.trigger_string + } +} diff --git a/modules/null/remote_exec_script/outputs.tf b/modules/null/remote_exec_script/outputs.tf new file mode 100644 index 00000000..21f87cdd --- /dev/null +++ b/modules/null/remote_exec_script/outputs.tf @@ -0,0 +1,9 @@ +output "command_1" { + description = "Command for reference" + value = local.command_1 +} + +output "final_command" { + description = "Command for reference" + value = local.final_command +} diff --git a/modules/null/remote_exec_script/variables.tf b/modules/null/remote_exec_script/variables.tf new file mode 100644 index 00000000..79d0e9e5 --- /dev/null +++ b/modules/null/remote_exec_script/variables.tf @@ -0,0 +1,76 @@ +variable "cluster_host" { + description = "Cluster hosts to be used for ssh connectivity." + type = list(string) +} + +variable "cluster_user" { + description = "Cluster user to be used for ssh connectivity." + type = string +} + +variable "cluster_private_key" { + description = "Cluster private key to be used for ssh connectivity." + type = string +} + +variable "login_host" { + description = "Login host to be used for ssh connectivity." + type = string +} + +variable "login_user" { + description = "Login user to be used for ssh connectivity." + type = string +} + +variable "login_private_key" { + description = "Login private key to be used for ssh connectivity." + type = string +} + +variable "payload_files" { + description = "List of files that are to be transferred." + type = list(string) + default = [] +} + +variable "payload_dirs" { + description = "List of directories that are to be transferred." + type = list(string) + default = [] +} + +variable "new_file_name" { + description = "File name to be created." + type = string + default = "" +} + +variable "new_file_content" { + description = "Content of file to be created." + type = string + default = "" +} + +variable "script_to_run" { + description = "Name of script to be run." + type = string +} + +variable "sudo_user" { + description = "User we want to sudo to (e.g. 'root')." + type = string + default = "" +} + +variable "with_bash" { + description = "If we want a 'bash -c' execution of the script." + type = bool + default = false +} + +variable "trigger_string" { + description = "Changing this string will trigger a re-run" + type = string + default = "" +} diff --git a/modules/null/remote_exec_script/version.tf b/modules/null/remote_exec_script/version.tf new file mode 100644 index 00000000..2c8d6bef --- /dev/null +++ b/modules/null/remote_exec_script/version.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.3, < 1.7" + required_providers { + null = { + source = "hashicorp/null" + version = ">= 3.0.0" + } + } +} diff --git a/modules/observability_instance/datasources.tf b/modules/observability_instance/datasources.tf new file mode 100644 index 00000000..d6ad0b90 --- /dev/null +++ b/modules/observability_instance/datasources.tf @@ -0,0 +1,12 @@ +data "ibm_iam_auth_token" "tokendata" {} + +data "http" "sysdig_prws_key" { + url = "https://${var.location}.monitoring.cloud.ibm.com/api/token" + + # Optional request headers + request_headers = { + Accept = "application/json" + Authorization = data.ibm_iam_auth_token.tokendata.iam_access_token + IBMInstanceID = var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_guid : "" + } +} diff --git a/modules/observability_instance/main.tf b/modules/observability_instance/main.tf new file mode 100644 index 00000000..fc216d8d --- /dev/null +++ b/modules/observability_instance/main.tf @@ -0,0 +1,48 @@ +# This module requires additional logdna provider configuration blocks +locals { + activity_tracker_instance_name = var.activity_tracker_instance_name + log_analysis_instance_name = var.log_analysis_instance_name + cloud_monitoring_instance_name = var.cloud_monitoring_instance_name + + logs_instance_endpoint = "https://api.${var.location}.logging.cloud.ibm.com" +} + +module "observability_instance" { + # Replace "master" with a GIT release version to lock into a specific release + source = "terraform-ibm-modules/observability-instances/ibm" + version = "2.12.2" + providers = { + logdna.at = logdna.at + logdna.ld = logdna.ld + } + region = var.location + ibmcloud_api_key = var.ibmcloud_api_key + resource_group_id = var.rg + + # Log Analysis + log_analysis_provision = var.log_analysis_provision + log_analysis_instance_name = local.log_analysis_instance_name + log_analysis_plan = var.log_analysis_plan + log_analysis_tags = var.tags + # IBM Cloud Monitoring + cloud_monitoring_provision = var.cloud_monitoring_provision + cloud_monitoring_instance_name = local.cloud_monitoring_instance_name + cloud_monitoring_plan = var.observability_monitoring_plan + cloud_monitoring_tags = var.tags + # Activity Tracker + activity_tracker_plan = var.activity_tracker_plan + activity_tracker_instance_name = local.activity_tracker_instance_name + activity_tracker_provision = var.activity_tracker_provision + activity_tracker_tags = var.tags + /* + # Event Routing + activity_tracker_routes = var.activity_tracker_routes + cos_targets = var.cos_targets + eventstreams_targets = var.eventstreams_targets + log_analysis_targets = var.log_analysis_targets + global_event_routing_settings = var.global_event_routing_settings + */ + enable_archive = var.enable_archive + enable_platform_logs = var.enable_platform_logs + enable_platform_metrics = var.enable_platform_metrics +} diff --git a/modules/observability_instance/outputs.tf b/modules/observability_instance/outputs.tf new file mode 100644 index 00000000..41313a5b --- /dev/null +++ b/modules/observability_instance/outputs.tf @@ -0,0 +1,27 @@ +output "cloud_monitoring_access_key" { + value = var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_access_key : null + description = "IBM Cloud Monitoring access key for agents to use" + sensitive = true +} + +output "cloud_monitoring_ingestion_url" { + value = var.cloud_monitoring_provision ? "ingest.${var.location}.monitoring.cloud.ibm.com" : null + description = "IBM Cloud Monitoring ingestion url for agents to use" +} + +output "log_analysis_ingestion_key" { + value = var.log_analysis_provision ? module.observability_instance.log_analysis_ingestion_key : null + description = "Log Analysis ingest key for agents to use" + sensitive = true +} + +output "cloud_monitoring_prws_key" { + value = var.cloud_monitoring_provision ? jsondecode(data.http.sysdig_prws_key.response_body).token.key : null + description = "IBM Cloud Monitoring Prometheus Remote Write ingestion key" + sensitive = true +} + +output "cloud_monitoring_prws_url" { + value = "https://ingest.prws.${var.location}.monitoring.cloud.ibm.com/prometheus/remote/write" + description = "IBM Cloud Monitoring Prometheus Remote Write ingestion url" +} diff --git a/modules/observability_instance/providers.tf b/modules/observability_instance/providers.tf new file mode 100644 index 00000000..550e085d --- /dev/null +++ b/modules/observability_instance/providers.tf @@ -0,0 +1,23 @@ +provider "logdna" { + alias = "ats" + servicekey = module.observability_instance.activity_tracker_ats_resource_key != null ? module.observability_instance.activity_tracker_ats_resource_key : "" + url = local.logs_instance_endpoint +} + +provider "logdna" { + alias = "sts" + servicekey = module.observability_instance.log_analysis_sts_resource_key != null ? module.observability_instance.log_analysis_sts_resource_key : "" + url = local.logs_instance_endpoint +} + +provider "logdna" { + alias = "at" + servicekey = module.observability_instance.activity_tracker_resource_key != null ? module.observability_instance.activity_tracker_resource_key : "" + url = local.logs_instance_endpoint +} + +provider "logdna" { + alias = "ld" + servicekey = module.observability_instance.log_analysis_resource_key != null ? module.observability_instance.log_analysis_resource_key : "" + url = local.logs_instance_endpoint +} diff --git a/modules/observability_instance/variables.tf b/modules/observability_instance/variables.tf new file mode 100644 index 00000000..9cf762ae --- /dev/null +++ b/modules/observability_instance/variables.tf @@ -0,0 +1,95 @@ +# Variable for IBM Cloud API Key +variable "ibmcloud_api_key" { + description = "IBM Cloud API Key" + type = string +} + +variable "activity_tracker_plan" { + description = "Type of service activity_tracker_plan." + type = string + default = "7-day" +} + +variable "log_analysis_plan" { + description = "Type of service log_analysis_plan." + type = string + default = "7-day" +} + +variable "observability_monitoring_plan" { + description = "Type of service observability_monitoring_plan." + type = string + default = "graduated-tier" +} + +variable "location" { + description = "Location where the resource is provisioned" + type = string + default = "us-south" +} + +variable "activity_tracker_instance_name" { + description = "Name of the resource activity_tracker_instance_name" + type = string + default = "demo-auditing-tf-instance" +} + +variable "log_analysis_instance_name" { + description = "Name of the resource log_analysis_instance_name" + type = string + default = "demo-auditing-tf-instance" +} + +variable "cloud_monitoring_instance_name" { + description = "Name of the resource cloud_monitoring_instance_name" + type = string + default = "demo-auditing-tf-instance" +} + +variable "rg" { + description = "Name of the resource group associated with the instance" + type = string + default = "default" +} + +variable "activity_tracker_provision" { + description = "Set true to provision Activity Tracker instance" + type = bool + default = "false" +} + +variable "log_analysis_provision" { + description = "Set true to provision log_analysis_provision instance" + type = bool + default = "false" +} + +variable "cloud_monitoring_provision" { + description = "Set true to provision cloud_monitoring_provision instance" + type = bool + default = "false" +} + +variable "tags" { + description = "Comma-separated list of tags" + type = list(string) + default = [] +} + +variable "enable_archive" { + description = "Set true to enable archive" + type = bool + default = "false" +} + +variable "enable_platform_logs" { + description = "Set true to enable platform logs" + type = bool + default = "false" +} + +variable "enable_platform_metrics" { + description = "Set true to enable platform metrics" + type = bool + default = "false" +} diff --git a/modules/observability_instance/versions.tf b/modules/observability_instance/versions.tf new file mode 100644 index 00000000..bb81e1ba --- /dev/null +++ b/modules/observability_instance/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.3, < 1.7" + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + version = ">= 1.56.2" + } + logdna = { + source = "logdna/logdna" + version = ">= 1.14.2" + configuration_aliases = [logdna.ats, logdna.sts, logdna.at, logdna.ld] + } + http = { + source = "hashicorp/http" + version = ">= 2.0.0" + } + } +} diff --git a/modules/playbook/main.tf b/modules/playbook/main.tf deleted file mode 100644 index 22c0eab6..00000000 --- a/modules/playbook/main.tf +++ /dev/null @@ -1,52 +0,0 @@ -resource "local_file" "create_playbook" { - count = var.inventory_path != null ? 1 : 0 - content = <- - -o ControlMaster=auto - -o ControlPersist=30m - -o UserKnownHostsFile=/dev/null - -o StrictHostKeyChecking=no - -o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${var.private_key_path} -J ubuntu@${var.bastion_fip} -W %h:%p root@{{ inventory_hostname }}" -EOT - filename = var.playbook_path -} - -/* -resource "null_resource" "run_playbook" { - count = var.inventory_path != null ? 1 : 0 - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - command = "ansible-playbook -i ${var.inventory_path} ${var.playbook_path}" - } - triggers = { - build = timestamp() - } - depends_on = [local_file.create_playbook] -} -*/ - -resource "ansible_playbook" "playbook" { - playbook = var.playbook_path - name = "localhost" - replayable = false - verbosity = 6 - extra_vars = { - ansible_python_interpreter = "auto" - } - depends_on = [local_file.create_playbook] -} diff --git a/modules/playbook/variables.tf b/modules/playbook/variables.tf deleted file mode 100644 index 72e75409..00000000 --- a/modules/playbook/variables.tf +++ /dev/null @@ -1,23 +0,0 @@ -variable "bastion_fip" { - type = string - default = null - description = "If Bastion is enabled, jump-host connection is required." -} - -variable "private_key_path" { - description = "Private key file path" - type = string - default = "id_rsa" -} - -variable "inventory_path" { - description = "Inventory file path" - type = string - default = "inventory.ini" -} - -variable "playbook_path" { - description = "Playbook path" - type = string - default = "ssh.yaml" -} diff --git a/modules/playbook/version.tf b/modules/playbook/version.tf deleted file mode 100644 index 6878f9cf..00000000 --- a/modules/playbook/version.tf +++ /dev/null @@ -1,23 +0,0 @@ -terraform { - required_version = ">= 1.3, < 1.7" - required_providers { - local = { - source = "hashicorp/local" - version = "~> 2" - } - /* - null = { - source = "hashicorp/null" - version = "~> 3" - } - http = { - source = "hashicorp/http" - version = "~> 3.4.0" - } - */ - ansible = { - version = "~> 1.1.0" - source = "ansible/ansible" - } - } -} diff --git a/modules/security/password/main.tf b/modules/security/password/main.tf new file mode 100644 index 00000000..11db2786 --- /dev/null +++ b/modules/security/password/main.tf @@ -0,0 +1,10 @@ +resource "random_password" "generate" { + length = var.length + special = var.special + numeric = var.numeric + upper = var.upper + min_lower = var.min_lower + min_upper = var.min_upper + override_special = var.override_special + min_numeric = var.min_numeric +} diff --git a/modules/security/password/outputs.tf b/modules/security/password/outputs.tf new file mode 100644 index 00000000..66b6ff88 --- /dev/null +++ b/modules/security/password/outputs.tf @@ -0,0 +1,5 @@ +output "password" { + description = "The generated random password" + sensitive = true + value = random_password.generate.result +} diff --git a/modules/security/password/variables.tf b/modules/security/password/variables.tf new file mode 100644 index 00000000..a0a35ab7 --- /dev/null +++ b/modules/security/password/variables.tf @@ -0,0 +1,53 @@ +variable "length" { + description = "The length of the password desired." + type = number + default = 12 +} + +variable "numeric" { + description = "Use numbers in the password" + type = bool + default = true +} + +variable "special" { + description = "Use special characters in the password" + type = bool + default = true +} + +# variable "lower" { +# description = "Include lowercase alphabet characters in the result." +# type = bool +# default = true +# } + +variable "upper" { + description = "Include uppercase alphabet characters in the result." + type = bool + default = true +} + +variable "min_lower" { + description = "Minimum number of lowercase alphabet characters in the result." + type = number + default = 0 +} + +variable "min_upper" { + description = "Minimum number of uppercase alphabet characters in the result." + type = number + default = 0 +} + +variable "override_special" { + description = "Supply your own list of special characters to use for string generation." + type = string + default = "!@#$%&*()-_=+[]{}<>:?" +} + +variable "min_numeric" { + description = "Minimum number of numeric characters in the result." + type = number + default = 0 +} diff --git a/modules/security/password/version.tf b/modules/security/password/version.tf new file mode 100644 index 00000000..63105e86 --- /dev/null +++ b/modules/security/password/version.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.3, < 1.7" + required_providers { + random = { + source = "hashicorp/random" + version = ">= 3.0.0" + } + } +} diff --git a/modules/security/scc/main.tf b/modules/security/scc/main.tf new file mode 100644 index 00000000..96e689b6 --- /dev/null +++ b/modules/security/scc/main.tf @@ -0,0 +1,58 @@ +# This module requires additional logdna provider configuration blocks +locals { + scc_region = var.location != "" ? var.location : "us-south" + scc_scope = [{ + environment = var.scc_scope_environment + properties = [ + { + name = "scope_id" + value = var.rg + }, + { + name = "scope_type" + value = "account.resource_group" + } + ] + }] +} + +module "event_notification" { + source = "terraform-ibm-modules/event-notifications/ibm" + version = "1.3.4" + resource_group_id = var.rg + name = "${var.prefix}-scc-event_notification" + plan = var.event_notification_plan + service_endpoints = var.event_notification_service_endpoints + region = local.scc_region + tags = var.tags +} + +module "create_scc_instance" { + source = "terraform-ibm-modules/scc/ibm" + version = "1.4.2" + instance_name = "${var.prefix}-scc-instance" + plan = var.scc_plan + region = local.scc_region + resource_group_id = var.rg + resource_tags = var.tags + cos_bucket = var.cos_bucket + cos_instance_crn = var.cos_instance_crn + en_instance_crn = module.event_notification.crn + skip_cos_iam_authorization_policy = false + attach_wp_to_scc_instance = false + skip_scc_wp_auth_policy = true + wp_instance_crn = null +} + +module "create_profile_attachment" { + count = var.scc_profile == null || var.scc_profile == "" ? 0 : 1 + source = "terraform-ibm-modules/scc/ibm//modules/attachment" + version = "1.4.2" + profile_name = var.scc_profile + profile_version = var.scc_profile_version + scc_instance_id = module.create_scc_instance.guid + attachment_name = "${var.prefix}-scc-attachment" + attachment_description = var.scc_attachment_description + attachment_schedule = var.scc_attachment_schedule + scope = local.scc_scope +} diff --git a/modules/security/scc/outputs.tf b/modules/security/scc/outputs.tf new file mode 100644 index 00000000..ba65df78 --- /dev/null +++ b/modules/security/scc/outputs.tf @@ -0,0 +1,14 @@ +############################################################### +# Outputs +############################################################### + +output "scc_crn" { + value = var.scc_provision ? module.create_scc_instance.crn : null + description = "The CRN of the SCC instance created by this module" + sensitive = true +} + +output "scc_en_crn" { + description = "The CRN of the event notification instance created in this module" + value = var.scc_provision ? module.event_notification.crn : null +} diff --git a/modules/security/scc/variables.tf b/modules/security/scc/variables.tf new file mode 100644 index 00000000..be987fe7 --- /dev/null +++ b/modules/security/scc/variables.tf @@ -0,0 +1,120 @@ +############################################################### +# Input variables +############################################################### + +# Prefix to append to resources name +variable "prefix" { + type = string + description = "Prefix to append to all resources created by this example" + default = "scc" +} + +# Region +variable "location" { + description = "Location where the resource is provisioned" + type = string + default = "us-south" +} + +# Resource Group Name +variable "rg" { + description = "Name of the resource group associated with the instance" + type = string + default = "default" +} + +# List of Resource Tags" +variable "tags" { + description = "Comma-separated list of tags" + type = list(string) + default = [] +} + +# Opt-In feature for SCC Instance +variable "scc_provision" { + type = bool + default = false + description = "Flag to enable SCC instance creation. If true, an instance of SCC (Security and Compliance Center) will be created." +} + +# SCC Instance Location +# variable "scc_location" { +# type = string +# default = "us-south" +# description = "SCC Instance region (possible choices 'us-south', 'eu-de', 'ca-tor', 'eu-es')" +# } + +# SCC Instance Plan +variable "scc_plan" { + type = string + default = "security-compliance-center-standard-plan" + description = "SCC Instance plan to be used" +} + +# SCC Instance Profile +variable "scc_profile" { + type = string + default = "CIS IBM Cloud Foundations Benchmark" + description = "Profile to be set on the SCC Instance (accepting empty, 'CIS IBM Cloud Foundations Benchmark' and 'IBM Cloud Framework for Financial Services')" +} + +# SCC Instance Profile Version +variable "scc_profile_version" { + type = string + default = "1.0.0" + description = "Version of Profile to be set on the SCC Instance" +} + +# SCC Scope Environment +variable "scc_scope_environment" { + type = string + default = "ibm-cloud" + description = "SCC Scope reference environment" +} + +# SCC Attachment Description +variable "scc_attachment_description" { + type = string + default = "Attachment automatically created by IBM Cloud HPC" + description = "Description of the SCC Attachment" +} + +# SCC Attachment Schedule +variable "scc_attachment_schedule" { + type = string + default = "daily" + description = "Schedule of the SCC Attachment" +} + +# SCC Attachment Status +# variable "scc_attachment_status" { +# type = string +# default = "enabled" +# description = "Status of the SCC Attachment" +# } + +# Event Notification Instance Plan +variable "event_notification_plan" { + type = string + default = "lite" + description = "Event Notifications Instance plan to be used" +} + +# Event Notification Instance Service Endpoints +variable "event_notification_service_endpoints" { + type = string + default = "public-and-private" + description = "Event Notifications Service Endpoints to be used" +} + +# COS Instance CRN +variable "cos_instance_crn" { + type = string + description = "CRN of the COS instance created by Landing Zone Module" +} + +# COS Bucket for SCC +variable "cos_bucket" { + type = string + description = "Name of the COS Bucket created for SCC Instance" +} diff --git a/modules/security/scc/versions.tf b/modules/security/scc/versions.tf new file mode 100644 index 00000000..01db83a5 --- /dev/null +++ b/modules/security/scc/versions.tf @@ -0,0 +1,3 @@ +terraform { + required_version = ">= 1.3, < 1.7" +} diff --git a/outputs.tf b/outputs.tf deleted file mode 100644 index 60e42680..00000000 --- a/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "ssh_command" { - description = "SSH command to connect to HPC cluster" - value = module.hpc.ssh_command -} diff --git a/reference-architectures/deploy-arch-ibm-hpc-lsf.md b/reference-architectures/deploy-arch-ibm-hpc-lsf.md deleted file mode 100644 index b2d7e6f2..00000000 --- a/reference-architectures/deploy-arch-ibm-hpc-lsf.md +++ /dev/null @@ -1,3 +0,0 @@ -# IBM Cloud HPC (Spectrum LSF) - -IBM Spectrum LSF allows you to deploy high-performance computing (HPC) clusters by using IBM Spectrum LSF as HPC scheduling software. This offering uses open source Terraform-based automation to provision and configure IBM Cloud resources. With simple steps to define configuration properties and use automated deployment, you can build your own HPC clusters in minutes. IBM Spectrum LSF also enables configuration for auto-scaling, so IBM Spectrum LSF clusters can automatically add and remove worker nodes based on workload specifications. This allows you to take full advantage of consumption-based pricing and pay for cloud resources only when they are needed. diff --git a/reference-architectures/deploy-arch-ibm-hpc-scale.md b/reference-architectures/deploy-arch-ibm-hpc-scale.md deleted file mode 100644 index 8382cdae..00000000 --- a/reference-architectures/deploy-arch-ibm-hpc-scale.md +++ /dev/null @@ -1,3 +0,0 @@ -# IBM Cloud HPC (Storage Scale) - -IBM Storage Scale is a high performance, highly available, clustered file system and associated management software, available on a variety of platforms. IBM Storage Scale can scale in several dimensions, including performance (bandwidth and IOPS), capacity and number of nodes* (instances) that can mount the file system. IBM Storage Scale addresses the needs of applications whose performance (or performance-to-capacity ratio) demands cannot be met by traditional scale-up storage systems; and IBM Storage Scale is therefore deployed for many I/O-demanding enterprise applications that require high performance or scale. IBM Storage Scale provides various configuration options, access methods (including traditional POSIX-based file access), and many features such as snapshots, compression, and encryption. Note that IBM Storage Scale is not itself an application in the traditional sense, but instead provides the storage infrastructure for applications. diff --git a/reference-architectures/deploy-arch-ibm-hpc-symphony.md b/reference-architectures/deploy-arch-ibm-hpc-symphony.md deleted file mode 100644 index 5ebc608a..00000000 --- a/reference-architectures/deploy-arch-ibm-hpc-symphony.md +++ /dev/null @@ -1,3 +0,0 @@ -# IBM Cloud HPC (Spectrum Symphony) - -IBM Spectrum Symphony allows you to deploy high-performance computing (HPC) clusters that use IBM Spectrum Symphony as the HPC scheduling software. This offering uses open source Terraform-based automation to provision and configure IBM Cloud resources. With simple steps to define configuration properties and use automated deployment, you can build your own HPC cluster in minutes. IBM Spectrum Symphony also supports auto-scaling for certain cluster configurations, so worker nodes can be automatically added and removed from a cluster based on workload requirements. This allows you to take full advantage of consumption-based pricing and pay for cloud resources only when they are needed. In addition, this offering supports use of IBM Spectrum Scale for applications requiring a high-performance file system for sharing of data. diff --git a/samples/configs/hpc_catalog_values.json b/samples/configs/hpc_catalog_values.json new file mode 100644 index 00000000..021f035c --- /dev/null +++ b/samples/configs/hpc_catalog_values.json @@ -0,0 +1,11 @@ +{ + "ibmcloud_api_key" : "Please fill here", + "resource_group" : "Default", + "zones" : "[\"us-east-1\"]", + "cluster_prefix" : "hpcaas", + "cluster_id" : "Please fill here", + "reservation_id" : "Please fill here", + "bastion_ssh_keys" : "[\"Please fill here\"]", + "compute_ssh_keys" : "[\"Please fill here\"]", + "remote_allowed_ips" : "[\"Please fill here\"]" +} diff --git a/samples/configs/hpc_schematics_values.json b/samples/configs/hpc_schematics_values.json new file mode 100644 index 00000000..9e76e252 --- /dev/null +++ b/samples/configs/hpc_schematics_values.json @@ -0,0 +1,116 @@ +{ + "name": "hpcaas-test", + "type": [ + "terraform_v1.5" + ], + "location": "eu-de", + "resource_group": "Default", + "description": "", + "tags": [], + "template_repo": { + "url": "https://github.com/terraform-ibm-modules/terraform-ibm-hpc", + "branch": "main" + }, + "template_data": [ + { + "folder": "solutions/hpc", + "type": "terraform_v1.5", + "env_values":[ + { + "TF_CLI_ARGS_apply": "-parallelism=250" + }, + { + "TF_CLI_ARGS_plan": "-parallelism=250" + }, + { + "TF_CLI_ARGS_destroy": "-parallelism=100" + }, + { + "VAR1":"" + }, + { + "VAR2":"" + } + ], + "variablestore": [ + { + "name": "TF_PARALLELISM", + "value": "250", + "type": "string", + "secure": false, + "description": "Parallelism concurrent operations limit. Valid values are between 1 and 256, both inclusive. [Learn more](https://www.terraform.io/docs/internals/graph.html#walking-the-graph)." + }, + { + "name": "TF_VERSION", + "value": "1.5", + "type": "string", + "secure": false, + "description": "The version of the Terraform engine that's used in the Schematics workspace." + }, + { + "name": "ibmcloud_api_key", + "value": "Please fill here", + "type": "string", + "secure": true, + "description": "IBM Cloud API key for the IBM Cloud account where the IBM Cloud HPC cluster needs to be deployed. For more information on how to create an API key, see [Managing user API keys](https://cloud.ibm.com/docs/account?topic=account-userapikey)." + }, + { + "name": "resource_group", + "value": "Default", + "type": "string", + "secure": false, + "description": "Resource group name from your IBM Cloud account where the VPC resources should be deployed. Note. If the resource group value is set as null, automation creates two different RG with the name (workload-rg and service-rg). For additional information on resource groups, see [Managing resource groups](https://cloud.ibm.com/docs/account?topic=account-rgs)." + }, + { + "name": "zones", + "value": "[\"us-east-1\"]", + "type": "list(string)", + "secure": false, + "description": "The IBM Cloud zone name within the selected region where the IBM Cloud HPC cluster should be deployed and requires a single input value. Supported zones are: eu-de-2 and eu-de-3 for eu-de, us-east-1 and us-east-3 for us-east, and us-south-1 for us-south. The management nodes, file storage shares, and compute nodes will be deployed in the same zone.[Learn more](https://cloud.ibm.com/docs/vpc?topic=vpc-creating-a-vpc-in-a-different-region#get-zones-using-the-cli)." + }, + { + "name": "cluster_prefix", + "value": "hpcaas", + "type": "string", + "secure": false, + "description": "Prefix that is used to name the IBM Cloud HPC cluster and IBM Cloud resources that are provisioned to build the IBM Cloud HPC cluster instance. You cannot create more than one instance of the IBM Cloud HPC cluster with the same name. Ensure that the name is unique. Prefix must start with a lowercase letter and contain only lowercase letters, digits, and hyphens in between. Hyphens must be followed by at least one lowercase letter or digit. There are no leading, trailing, or consecutive hyphens.Character length for cluster_prefix should be less than 16." + }, + { + "name": "cluster_id", + "value": "Please fill here", + "type": "string", + "secure": false, + "description": "Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservations. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment." + }, + { + "name": "reservation_id", + "value": "Please fill here", + "type": "string", + "secure": true, + "description": "Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_)." + }, + { + "name": "bastion_ssh_keys", + "value": "[\"Please fill here\"]", + "type": "list(string)", + "secure": false, + "description": "Provide the list of SSH key names configured in your IBM Cloud account to establish a connection to the IBM Cloud HPC bastion and login node. Ensure the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by following the provided instructions.[SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys)." + }, + { + "name": "compute_ssh_keys", + "value": "[\"Please fill here\"]", + "type": "list(string)", + "secure": false, + "description": "Provide the list of SSH key names configured in your IBM Cloud account to establish a connection to the IBM Cloud HPC cluster node. Ensure the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by following the provided instructions.[SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys)." + }, + { + "name": "remote_allowed_ips", + "value": "[\"Please fill here\"]", + "type": "list(string)", + "secure": false, + "description": "Comma-separated list of IP addresses that can access the IBM Cloud HPC cluster instance through an SSH interface. For security purposes, provide the public IP addresses assigned to the devices that are authorized to establish SSH connections (for example, [\"169.45.117.34\"]). To fetch the IP address of the device, use [https://ipv4.icanhazip.com/](https://ipv4.icanhazip.com/)." + } + ] + } + ] +} diff --git a/solutions/hpc/README.md b/solutions/hpc/README.md index 409ae50a..f6ca6eb6 100644 --- a/solutions/hpc/README.md +++ b/solutions/hpc/README.md @@ -2,100 +2,159 @@ | Name | Version | |------|---------| -| [ibm](#requirement\_ibm) | >= 1.56.2 | +| [terraform](#requirement\_terraform) | >= 1.3, < 1.7 | +| [http](#requirement\_http) | 3.4.2 | +| [ibm](#requirement\_ibm) | 1.65.1 | +| [null](#requirement\_null) | 3.2.2 | ## Providers | Name | Version | |------|---------| -| [ibm](#provider\_ibm) | >= 1.56.2 | +| [http](#provider\_http) | 3.4.2 | +| [ibm](#provider\_ibm) | 1.65.1 | +| [null](#provider\_null) | 3.2.2 | ## Modules | Name | Source | Version | |------|--------|---------| +| [alb](#module\_alb) | ./../../modules/alb | n/a | +| [alb\_api](#module\_alb\_api) | ./../../modules/alb_api | n/a | +| [bastion\_inventory](#module\_bastion\_inventory) | ./../../modules/inventory | n/a | | [bootstrap](#module\_bootstrap) | ./../../modules/bootstrap | n/a | +| [check\_cluster\_status](#module\_check\_cluster\_status) | ./../../modules/null/remote_exec | n/a | +| [check\_node\_status](#module\_check\_node\_status) | ./../../modules/null/remote_exec | n/a | +| [cloud\_monitoring\_instance\_creation](#module\_cloud\_monitoring\_instance\_creation) | ../../modules/observability_instance | n/a | +| [compute\_candidate\_dns\_records](#module\_compute\_candidate\_dns\_records) | ./../../modules/dns_record | n/a | | [compute\_dns\_records](#module\_compute\_dns\_records) | ./../../modules/dns_record | n/a | | [compute\_inventory](#module\_compute\_inventory) | ./../../modules/inventory | n/a | -| [compute\_playbook](#module\_compute\_playbook) | ./../../modules/playbook | n/a | +| [db](#module\_db) | ../../modules/database/mysql | n/a | | [dns](#module\_dns) | ./../../modules/dns | n/a | | [file\_storage](#module\_file\_storage) | ../../modules/file_storage | n/a | +| [generate\_db\_adminpassword](#module\_generate\_db\_adminpassword) | ../../modules/security/password | n/a | +| [ipvalidation\_cluster\_subnet](#module\_ipvalidation\_cluster\_subnet) | ../../modules/custom/subnet_cidr_check | n/a | +| [ipvalidation\_login\_subnet](#module\_ipvalidation\_login\_subnet) | ../../modules/custom/subnet_cidr_check | n/a | | [landing\_zone](#module\_landing\_zone) | ../../modules/landing_zone | n/a | | [landing\_zone\_vsi](#module\_landing\_zone\_vsi) | ../../modules/landing_zone_vsi | n/a | -| [protocol\_dns\_records](#module\_protocol\_dns\_records) | ./../../modules/dns_record | n/a | -| [storage\_dns\_records](#module\_storage\_dns\_records) | ./../../modules/dns_record | n/a | -| [storage\_inventory](#module\_storage\_inventory) | ./../../modules/inventory | n/a | -| [storage\_playbook](#module\_storage\_playbook) | ./../../modules/playbook | n/a | +| [ldap\_inventory](#module\_ldap\_inventory) | ./../../modules/inventory | n/a | +| [ldap\_vsi\_dns\_records](#module\_ldap\_vsi\_dns\_records) | ./../../modules/dns_record | n/a | +| [local\_exec\_script](#module\_local\_exec\_script) | ../../modules/null/local_exec_script | n/a | +| [login\_fip\_removal](#module\_login\_fip\_removal) | ./../../modules/null/local_exec | n/a | +| [login\_inventory](#module\_login\_inventory) | ./../../modules/inventory | n/a | +| [login\_vsi\_dns\_records](#module\_login\_vsi\_dns\_records) | ./../../modules/dns_record | n/a | +| [my\_ip](#module\_my\_ip) | ../../modules/my_ip | n/a | +| [scc\_instance\_and\_profile](#module\_scc\_instance\_and\_profile) | ./../../modules/security/scc | n/a | +| [validate\_ldap\_server\_connection](#module\_validate\_ldap\_server\_connection) | ./../../modules/null/ldap_remote_exec | n/a | +| [validation\_script\_executor](#module\_validation\_script\_executor) | ./../../modules/null/remote_exec | n/a | ## Resources | Name | Type | |------|------| -| [ibm_is_region.itself](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/is_region) | data source | -| [ibm_is_subnet.itself](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/is_subnet) | data source | -| [ibm_is_vpc.itself](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/is_vpc) | data source | -| [ibm_is_zone.itself](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/is_zone) | data source | +| [ibm_dns_resource_record.pac_cname](https://registry.terraform.io/providers/IBM-Cloud/ibm/1.65.1/docs/resources/dns_resource_record) | resource | +| [ibm_is_subnet_public_gateway_attachment.zone_1_attachment](https://registry.terraform.io/providers/IBM-Cloud/ibm/1.65.1/docs/resources/is_subnet_public_gateway_attachment) | resource | +| [null_resource.destroy_compute_resources](https://registry.terraform.io/providers/hashicorp/null/3.2.2/docs/resources/resource) | resource | +| [http_http.reservation_id_validation](https://registry.terraform.io/providers/hashicorp/http/3.4.2/docs/data-sources/http) | data source | +| [ibm_iam_auth_token.auth_token](https://registry.terraform.io/providers/IBM-Cloud/ibm/1.65.1/docs/data-sources/iam_auth_token) | data source | +| [ibm_is_public_gateways.public_gateways](https://registry.terraform.io/providers/IBM-Cloud/ibm/1.65.1/docs/data-sources/is_public_gateways) | data source | +| [ibm_is_region.region](https://registry.terraform.io/providers/IBM-Cloud/ibm/1.65.1/docs/data-sources/is_region) | data source | +| [ibm_is_subnet.existing_login_subnet](https://registry.terraform.io/providers/IBM-Cloud/ibm/1.65.1/docs/data-sources/is_subnet) | data source | +| [ibm_is_subnet.existing_subnet](https://registry.terraform.io/providers/IBM-Cloud/ibm/1.65.1/docs/data-sources/is_subnet) | data source | +| [ibm_is_vpc.existing_vpc](https://registry.terraform.io/providers/IBM-Cloud/ibm/1.65.1/docs/data-sources/is_vpc) | data source | +| [ibm_is_vpc.itself](https://registry.terraform.io/providers/IBM-Cloud/ibm/1.65.1/docs/data-sources/is_vpc) | data source | +| [ibm_is_vpc.vpc](https://registry.terraform.io/providers/IBM-Cloud/ibm/1.65.1/docs/data-sources/is_vpc) | data source | +| [ibm_is_vpc_address_prefixes.existing_vpc](https://registry.terraform.io/providers/IBM-Cloud/ibm/1.65.1/docs/data-sources/is_vpc_address_prefixes) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [allowed\_cidr](#input\_allowed\_cidr) | Network CIDR to access the VPC. This is used to manage network ACL rules for accessing the cluster. | `list(string)` | 
[
  "10.0.0.0/8"
]
| no | -| [bastion\_ssh\_keys](#input\_bastion\_ssh\_keys) | The key pair to use to access the bastion host. | `list(string)` | n/a | yes | -| [bastion\_subnets\_cidr](#input\_bastion\_subnets\_cidr) | Subnet CIDR block to launch the bastion host. | `list(string)` | 
[
  "10.0.0.0/24"
]
| no | -| [boot\_volume\_encryption\_enabled](#input\_boot\_volume\_encryption\_enabled) | Set to true when key management is set | `bool` | `true` | no | -| [bootstrap\_instance\_profile](#input\_bootstrap\_instance\_profile) | Bootstrap should be only used for better deployment performance | `string` | `"mx2-4x32"` | no | -| [compute\_gui\_password](#input\_compute\_gui\_password) | Password for compute cluster GUI | `string` | n/a | yes | -| [compute\_gui\_username](#input\_compute\_gui\_username) | GUI user to perform system management and monitoring tasks on compute cluster. | `string` | `"admin"` | no | -| [compute\_image\_name](#input\_compute\_image\_name) | Image name to use for provisioning the compute cluster instances. | `string` | `"ibm-redhat-8-6-minimal-amd64-5"` | no | -| [compute\_ssh\_keys](#input\_compute\_ssh\_keys) | The key pair to use to launch the compute host. | `list(string)` | n/a | yes | -| [compute\_subnets\_cidr](#input\_compute\_subnets\_cidr) | Subnet CIDR block to launch the compute cluster host. | `list(string)` | 
[
  "10.10.20.0/24",
  "10.20.20.0/24",
  "10.30.20.0/24"
]
| no | -| [cos\_instance\_name](#input\_cos\_instance\_name) | Exiting COS instance name | `string` | `null` | no | -| [dns\_custom\_resolver\_id](#input\_dns\_custom\_resolver\_id) | IBM Cloud DNS custom resolver id. | `string` | `null` | no | -| [dns\_domain\_names](#input\_dns\_domain\_names) | IBM Cloud HPC DNS domain names. | 
object({
    compute  = string
    storage  = string
    protocol = string
  })
| 
{
  "compute": "comp.com",
  "protocol": "ces.com",
  "storage": "strg.com"
}
| no | -| [dns\_instance\_id](#input\_dns\_instance\_id) | IBM Cloud HPC DNS service instance id. | `string` | `null` | no | -| [dynamic\_compute\_instances](#input\_dynamic\_compute\_instances) | MaxNumber of instances to be launched for compute cluster. | 
list(
    object({
      profile = string
      count   = number
    })
  )
| 
[
  {
    "count": 250,
    "profile": "cx2-2x4"
  }
]
| no | -| [enable\_atracker](#input\_enable\_atracker) | Enable Activity tracker | `bool` | `true` | no | -| [enable\_bastion](#input\_enable\_bastion) | The solution supports multiple ways to connect to your HPC cluster for example, using bastion node, via VPN or direct connection. If connecting to the HPC cluster via VPN or direct connection, set this value to false. | `bool` | `true` | no | -| [enable\_bootstrap](#input\_enable\_bootstrap) | Bootstrap should be only used for better deployment performance | `bool` | `false` | no | -| [enable\_cos\_integration](#input\_enable\_cos\_integration) | Integrate COS with HPC solution | `bool` | `true` | no | -| [enable\_vpc\_flow\_logs](#input\_enable\_vpc\_flow\_logs) | Enable Activity tracker | `bool` | `true` | no | -| [enable\_vpn](#input\_enable\_vpn) | The solution supports multiple ways to connect to your HPC cluster for example, using bastion node, via VPN or direct connection. If connecting to the HPC cluster via VPN, set this value to true. | `bool` | `false` | no | -| [file\_shares](#input\_file\_shares) | Custom file shares to access shared storage | 
list(
    object({
      mount_path = string,
      size       = number,
      iops       = number
    })
  )
| 
[
  {
    "iops": 1000,
    "mount_path": "/mnt/binaries",
    "size": 100
  },
  {
    "iops": 1000,
    "mount_path": "/mnt/data",
    "size": 100
  }
]
| no | -| [hpcs\_instance\_name](#input\_hpcs\_instance\_name) | Hyper Protect Crypto Service instance | `string` | `null` | no | -| [ibm\_customer\_number](#input\_ibm\_customer\_number) | Comma-separated list of the IBM Customer Number(s) (ICN) that is used for the Bring Your Own License (BYOL) entitlement check. For more information on how to find your ICN, see [What is my IBM Customer Number (ICN)?](https://www.ibm.com/support/pages/what-my-ibm-customer-number-icn). | `string` | `""` | no | -| [ibmcloud\_api\_key](#input\_ibmcloud\_api\_key) | IBM Cloud API Key that will be used for authentication in scripts run in this module. Only required if certain options are required. | `string` | `null` | no | -| [key\_management](#input\_key\_management) | null/key\_protect/hs\_crypto | `string` | `"key_protect"` | no | -| [login\_image\_name](#input\_login\_image\_name) | Image name to use for provisioning the login instances. | `string` | `"ibm-redhat-8-6-minimal-amd64-5"` | no | -| [login\_instances](#input\_login\_instances) | Number of instances to be launched for login. | 
list(
    object({
      profile = string
      count   = number
    })
  )
| 
[
  {
    "count": 1,
    "profile": "cx2-2x4"
  }
]
| no | -| [login\_ssh\_keys](#input\_login\_ssh\_keys) | The key pair to use to launch the login host. | `list(string)` | n/a | yes | -| [login\_subnets\_cidr](#input\_login\_subnets\_cidr) | Subnet CIDR block to launch the login host. | `list(string)` | 
[
  "10.10.10.0/24",
  "10.20.10.0/24",
  "10.30.10.0/24"
]
| no | -| [management\_image\_name](#input\_management\_image\_name) | Image name to use for provisioning the management cluster instances. | `string` | `"ibm-redhat-8-6-minimal-amd64-5"` | no | -| [management\_instances](#input\_management\_instances) | Number of instances to be launched for management. | 
list(
    object({
      profile = string
      count   = number
    })
  )
| 
[
  {
    "count": 3,
    "profile": "cx2-2x4"
  }
]
| no | -| [network\_cidr](#input\_network\_cidr) | Network CIDR for the VPC. This is used to manage network ACL rules for cluster provisioning. | `string` | `"10.0.0.0/8"` | no | -| [nsd\_details](#input\_nsd\_details) | Storage scale NSD details | 
list(
    object({
      profile  = string
      capacity = optional(number)
      iops     = optional(number)
    })
  )
| 
[
  {
    "iops": 100,
    "profile": "custom",
    "size": 100
  }
]
| no | -| [placement\_strategy](#input\_placement\_strategy) | VPC placement groups to create (null / host\_spread / power\_spread) | `string` | `null` | no | -| [prefix](#input\_prefix) | A unique identifier for resources. Must begin with a letter and end with a letter or number. This prefix will be prepended to any resources provisioned by this template. Prefixes must be 16 or fewer characters. | `string` | n/a | yes | -| [protocol\_instances](#input\_protocol\_instances) | Number of instances to be launched for protocol hosts. | 
list(
    object({
      profile = string
      count   = number
    })
  )
| 
[
  {
    "count": 2,
    "profile": "bx2-2x8"
  }
]
| no | -| [protocol\_subnets\_cidr](#input\_protocol\_subnets\_cidr) | Subnet CIDR block to launch the storage cluster host. | `list(string)` | 
[
  "10.10.40.0/24",
  "10.20.40.0/24",
  "10.30.40.0/24"
]
| no | -| [resource\_group](#input\_resource\_group) | String describing resource groups to create or reference | `string` | `null` | no | -| [scheduler](#input\_scheduler) | Select one of the scheduler (LSF/Symphony/Slurm/None) | `string` | `"LSF"` | no | -| [static\_compute\_instances](#input\_static\_compute\_instances) | Min Number of instances to be launched for compute cluster. | 
list(
    object({
      profile = string
      count   = number
    })
  )
| 
[
  {
    "count": 0,
    "profile": "cx2-2x4"
  }
]
| no | -| [storage\_gui\_password](#input\_storage\_gui\_password) | Password for storage cluster GUI | `string` | n/a | yes | -| [storage\_gui\_username](#input\_storage\_gui\_username) | GUI user to perform system management and monitoring tasks on storage cluster. | `string` | `"admin"` | no | -| [storage\_image\_name](#input\_storage\_image\_name) | Image name to use for provisioning the storage cluster instances. | `string` | `"ibm-redhat-8-6-minimal-amd64-5"` | no | -| [storage\_instances](#input\_storage\_instances) | Number of instances to be launched for storage cluster. | 
list(
    object({
      profile = string
      count   = number
    })
  )
| 
[
  {
    "count": 3,
    "profile": "bx2-2x8"
  }
]
| no | -| [storage\_ssh\_keys](#input\_storage\_ssh\_keys) | The key pair to use to launch the storage cluster host. | `list(string)` | n/a | yes | -| [storage\_subnets\_cidr](#input\_storage\_subnets\_cidr) | Subnet CIDR block to launch the storage cluster host. | `list(string)` | 
[
  "10.10.30.0/24",
  "10.20.30.0/24",
  "10.30.30.0/24"
]
| no | -| [storage\_type](#input\_storage\_type) | Select the required storage type(scratch/persistent/eval). | `string` | `"scratch"` | no | -| [vpc](#input\_vpc) | Name of an existing VPC in which the cluster resources will be deployed. If no value is given, then a new VPC will be provisioned for the cluster. [Learn more](https://cloud.ibm.com/docs/vpc) | `string` | `null` | no | -| [vpn\_peer\_address](#input\_vpn\_peer\_address) | The peer public IP address to which the VPN will be connected. | `string` | `null` | no | -| [vpn\_peer\_cidr](#input\_vpn\_peer\_cidr) | The peer CIDRs (e.g., 192.168.0.0/24) to which the VPN will be connected. | `list(string)` | `null` | no | -| [vpn\_preshared\_key](#input\_vpn\_preshared\_key) | The pre-shared key for the VPN. | `string` | `null` | no | -| [zones](#input\_zones) | Region where VPC will be created. To find your VPC region, use `ibmcloud is regions` command to find available regions. | `list(string)` | n/a | yes | +| [TF\_PARALLELISM](#input\_TF\_PARALLELISM) | Parallelism/ concurrent operations limit. Valid values are between 1 and 256, both inclusive. [Learn more](https://www.terraform.io/docs/internals/graph.html#walking-the-graph). | `string` | `"250"` | no | +| [TF\_VALIDATION\_SCRIPT\_FILES](#input\_TF\_VALIDATION\_SCRIPT\_FILES) | List of script file names used by validation test suites. If provided, these scripts will be executed as part of validation test suites execution. | `list(string)` | `[]` | no | +| [TF\_VERSION](#input\_TF\_VERSION) | The version of the Terraform engine that's used in the Schematics workspace. | `string` | `"1.5"` | no | +| [app\_center\_gui\_pwd](#input\_app\_center\_gui\_pwd) | Password for IBM Spectrum LSF Application Center GUI. Note: Password should be at least 8 characters, must have one number, one lowercase letter, one uppercase letter, and at least one special character. | `string` | `""` | no | +| [app\_center\_high\_availability](#input\_app\_center\_high\_availability) | Set to false to disable the IBM Spectrum LSF Application Center GUI High Availability (default: true). | `bool` | `true` | no | +| [bastion\_instance\_name](#input\_bastion\_instance\_name) | Bastion instance name. If none given then new bastion will be created. | `string` | `null` | no | +| [bastion\_instance\_public\_ip](#input\_bastion\_instance\_public\_ip) | Bastion instance public ip address. | `string` | `null` | no | +| [bastion\_security\_group\_id](#input\_bastion\_security\_group\_id) | Bastion security group id. | `string` | `null` | no | +| [bastion\_ssh\_keys](#input\_bastion\_ssh\_keys) | Provide the list of SSH key names configured in your IBM Cloud account to establish a connection to the IBM Cloud HPC bastion and login node. Ensure the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by following the provided instructions.[SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys). | `list(string)` | n/a | yes | +| [bastion\_ssh\_private\_key](#input\_bastion\_ssh\_private\_key) | Bastion SSH private key path, which will be used to login to bastion host. | `string` | `null` | no | +| [cluster\_id](#input\_cluster\_id) | Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservations. This can be up to 39 alphanumeric characters including the underscore (\_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment. | `string` | n/a | yes | +| [cluster\_prefix](#input\_cluster\_prefix) | Prefix that is used to name the IBM Cloud HPC cluster and IBM Cloud resources that are provisioned to build the IBM Cloud HPC cluster instance. You cannot create more than one instance of the IBM Cloud HPC cluster with the same name. Ensure that the name is unique. Prefix must start with a lowercase letter and contain only lowercase letters, digits, and hyphens in between. Hyphens must be followed by at least one lowercase letter or digit. There are no leading, trailing, or consecutive hyphens.Character length for cluster\_prefix should be less than 16. | `string` | `"hpcaas"` | no | +| [cluster\_subnet\_ids](#input\_cluster\_subnet\_ids) | Provide the list of existing subnet ID under the existing VPC where the cluster will be provisioned. One subnet ID is required as input value. Supported zones are: eu-de-2 and eu-de-3 for eu-de, us-east-1 and us-east-3 for us-east, and us-south-1 for us-south. The management nodes, file storage shares, and compute nodes will be deployed in the same zone. | `list(string)` | `[]` | no | +| [compute\_image\_name](#input\_compute\_image\_name) | Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-8 base OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v4). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. | `string` | `"hpcaas-lsf10-rhel88-compute-v5"` | no | +| [compute\_ssh\_keys](#input\_compute\_ssh\_keys) | Provide the list of SSH key names configured in your IBM Cloud account to establish a connection to the IBM Cloud HPC cluster node. Ensure the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by following the provided instructions.[SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys). | `list(string)` | n/a | yes | +| [cos\_instance\_name](#input\_cos\_instance\_name) | Provide the name of the existing cos instance to store vpc flow logs. | `string` | `null` | no | +| [custom\_file\_shares](#input\_custom\_file\_shares) | Mount points and sizes in GB and IOPS range of file shares that can be used to customize shared file storage layout. Provide the details for up to 5 shares. Each file share size in GB supports different range of IOPS. For more information, see [file share IOPS value](https://cloud.ibm.com/docs/vpc?topic=vpc-file-storage-profiles&interface=ui). | 
list(object({
    mount_path = string,
    size       = optional(number),
    iops       = optional(number),
    nfs_share  = optional(string)
  }))
| 
[
  {
    "iops": 2000,
    "mount_path": "/mnt/vpcstorage/tools",
    "size": 100
  },
  {
    "iops": 6000,
    "mount_path": "/mnt/vpcstorage/data",
    "size": 100
  },
  {
    "mount_path": "/mnt/scale/tools",
    "nfs_share": ""
  }
]
| no | +| [dns\_custom\_resolver\_id](#input\_dns\_custom\_resolver\_id) | Provide the id of existing IBM Cloud DNS custom resolver to skip creating a new custom resolver. Note: A VPC can be associated only to a single custom resolver, please provide the id of custom resolver if it is already associated to the VPC. | `string` | `null` | no | +| [dns\_domain\_name](#input\_dns\_domain\_name) | IBM Cloud DNS Services domain name to be used for the IBM Cloud HPC cluster. | 
object({
    compute = string
    #storage  = string
    #protocol = string
  })
| 
{
  "compute": "hpcaas.com"
}
| no | +| [dns\_instance\_id](#input\_dns\_instance\_id) | Provide the id of existing IBM Cloud DNS services domain to skip creating a new DNS service instance name. Note: If dns\_instance\_id is not equal to null, a new dns zone will be created under the existing dns service instance. | `string` | `null` | no | +| [enable\_app\_center](#input\_enable\_app\_center) | Set to true to enable the IBM Spectrum LSF Application Center GUI (default: false). [System requirements](https://www.ibm.com/docs/en/slac/10.2.0?topic=requirements-system-102-fix-pack-14) for IBM Spectrum LSF Application Center Version 10.2 Fix Pack 14. | `bool` | `false` | no | +| [enable\_cos\_integration](#input\_enable\_cos\_integration) | Set to true to create an extra cos bucket to integrate with HPC cluster deployment. | `bool` | `false` | no | +| [enable\_fip](#input\_enable\_fip) | The solution supports multiple ways to connect to your IBM Cloud HPC cluster for example, using a login node, or using VPN or direct connection. If connecting to the IBM Cloud HPC cluster using VPN or direct connection, set this value to false. | `bool` | `true` | no | +| [enable\_ldap](#input\_enable\_ldap) | Set this option to true to enable LDAP for IBM Cloud HPC, with the default value set to false. | `bool` | `false` | no | +| [enable\_vpc\_flow\_logs](#input\_enable\_vpc\_flow\_logs) | Flag to enable VPC flow logs. If true, a flow log collector will be created. | `bool` | `false` | no | +| [existing\_certificate\_instance](#input\_existing\_certificate\_instance) | When app\_center\_high\_availability is enable/set as true, The Application Center will be configured for high availability and requires a Application Load Balancer Front End listener to use a certificate CRN value stored in the Secret Manager. Provide the valid 'existing\_certificate\_instance' to configure the Application load balancer. | `string` | `""` | no | +| [hyperthreading\_enabled](#input\_hyperthreading\_enabled) | Setting this to true will enable hyper-threading in the compute nodes of the cluster (default). Otherwise, hyper-threading will be disabled. | `bool` | `true` | no | +| [ibmcloud\_api\_key](#input\_ibmcloud\_api\_key) | IBM Cloud API key for the IBM Cloud account where the IBM Cloud HPC cluster needs to be deployed. For more information on how to create an API key, see [Managing user API keys](https://cloud.ibm.com/docs/account?topic=account-userapikey). | `string` | n/a | yes | +| [key\_management](#input\_key\_management) | Set the value as key\_protect to enable customer managed encryption for boot volume and file share. If the key\_management is set as null, encryption will be always provider managed. | `string` | `"key_protect"` | no | +| [kms\_instance\_name](#input\_kms\_instance\_name) | Provide the name of the existing Key Protect instance associated with the Key Management Service. Note: To use existing kms\_instance\_name shall be considered only if key\_management value is set as key\_protect under key\_management variable. The name can be found under the details of the KMS, see [View key-protect ID](https://cloud.ibm.com/docs/key-protect?topic=key-protect-retrieve-instance-ID&interface=ui). | `string` | `null` | no | +| [kms\_key\_name](#input\_kms\_key\_name) | Provide the existing KMS encryption key name that you want to use for the IBM Cloud HPC cluster. Note: kms\_key\_name to be considered only if key\_management value is set as key\_protect under key\_management variable.(for example kms\_key\_name: my-encryption-key). | `string` | `null` | no | +| [ldap\_admin\_password](#input\_ldap\_admin\_password) | The LDAP administrative password should be 8 to 20 characters long, with a mix of at least three alphabetic characters, including one uppercase and one lowercase letter. It must also include two numerical digits and at least one special character from (~@\_+:) are required. It is important to avoid including the username in the password for enhanced security.[This value is ignored for an existing LDAP server]. | `string` | `""` | no | +| [ldap\_basedns](#input\_ldap\_basedns) | The dns domain name is used for configuring the LDAP server. If an LDAP server is already in existence, ensure to provide the associated DNS domain name. | `string` | `"hpcaas.com"` | no | +| [ldap\_server](#input\_ldap\_server) | Provide the IP address for the existing LDAP server. If no address is given, a new LDAP server will be created. | `string` | `"null"` | no | +| [ldap\_user\_name](#input\_ldap\_user\_name) | Custom LDAP User for performing cluster operations. Note: Username should be between 4 to 32 characters, (any combination of lowercase and uppercase letters).[This value is ignored for an existing LDAP server] | `string` | `""` | no | +| [ldap\_user\_password](#input\_ldap\_user\_password) | The LDAP user password should be 8 to 20 characters long, with a mix of at least three alphabetic characters, including one uppercase and one lowercase letter. It must also include two numerical digits and at least one special character from (~@\_+:) are required.It is important to avoid including the username in the password for enhanced security.[This value is ignored for an existing LDAP server]. | `string` | `""` | no | +| [ldap\_vsi\_osimage\_name](#input\_ldap\_vsi\_osimage\_name) | Image name to be used for provisioning the LDAP instances. By default ldap server are created on Ubuntu based OS flavour. | `string` | `"ibm-ubuntu-22-04-3-minimal-amd64-1"` | no | +| [ldap\_vsi\_profile](#input\_ldap\_vsi\_profile) | Specify the virtual server instance profile type to be used to create the ldap node for the IBM Cloud HPC cluster. For choices on profile types, see [Instance profiles](https://cloud.ibm.com/docs/vpc?topic=vpc-profiles). | `string` | `"cx2-2x4"` | no | +| [login\_image\_name](#input\_login\_image\_name) | Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-8 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v4). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. | `string` | `"hpcaas-lsf10-rhel88-compute-v5"` | no | +| [login\_node\_instance\_type](#input\_login\_node\_instance\_type) | Specify the virtual server instance profile type to be used to create the login node for the IBM Cloud HPC cluster. For choices on profile types, see [Instance profiles](https://cloud.ibm.com/docs/vpc?topic=vpc-profiles). | `string` | `"bx2-2x8"` | no | +| [login\_subnet\_id](#input\_login\_subnet\_id) | Provide the list of existing subnet ID under the existing VPC, where the login/bastion server will be provisioned. One subnet id is required as input value for the creation of login node and bastion in the same zone as the management nodes. Note: Provide a different subnet id for login\_subnet\_id, do not overlap or provide the same subnet id that was already provided for cluster\_subnet\_ids. | `string` | `null` | no | +| [management\_image\_name](#input\_management\_image\_name) | Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster management nodes. By default, the solution uses a RHEL88 base image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering. | `string` | `"hpcaas-lsf10-rhel88-v6"` | no | +| [management\_node\_count](#input\_management\_node\_count) | Number of management nodes. This is the total number of management nodes. Enter a value between 1 and 10. | `number` | `3` | no | +| [management\_node\_instance\_type](#input\_management\_node\_instance\_type) | Specify the virtual server instance profile type to be used to create the management nodes for the IBM Cloud HPC cluster. For choices on profile types, see [Instance profiles](https://cloud.ibm.com/docs/vpc?topic=vpc-profiles). | `string` | `"bx2-16x64"` | no | +| [observability\_atracker\_on\_cos\_enable](#input\_observability\_atracker\_on\_cos\_enable) | Enable Activity tracker service instance connected to Cloud Object Storage (COS). All the events will be stored into COS so that customers can connect to it and read those events or ingest them in their system. | `bool` | `true` | no | +| [observability\_monitoring\_enable](#input\_observability\_monitoring\_enable) | Set false to disable IBM Cloud Monitoring integration. If enabled, infrastructure and LSF application metrics from Management Nodes will be ingested. | `bool` | `false` | no | +| [observability\_monitoring\_on\_compute\_nodes\_enable](#input\_observability\_monitoring\_on\_compute\_nodes\_enable) | Set false to disable IBM Cloud Monitoring integration. If enabled, infrastructure metrics from Compute Nodes will be ingested. | `bool` | `false` | no | +| [observability\_monitoring\_plan](#input\_observability\_monitoring\_plan) | Type of service plan for IBM Cloud Monitoring instance. You can choose one of the following: lite, graduated-tier. For all details visit [IBM Cloud Monitoring Service Plans](https://cloud.ibm.com/docs/monitoring?topic=monitoring-service_plans). | `string` | `"graduated-tier"` | no | +| [remote\_allowed\_ips](#input\_remote\_allowed\_ips) | Comma-separated list of IP addresses that can access the IBM Cloud HPC cluster instance through an SSH interface. For security purposes, provide the public IP addresses assigned to the devices that are authorized to establish SSH connections (for example, ["169.45.117.34"]). To fetch the IP address of the device, use [https://ipv4.icanhazip.com/](https://ipv4.icanhazip.com/). | `list(string)` | n/a | yes | +| [reservation\_id](#input\_reservation\_id) | Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (\_). | `string` | n/a | yes | +| [resource\_group](#input\_resource\_group) | Resource group name from your IBM Cloud account where the VPC resources should be deployed. Note. If the resource group value is set as null, automation creates two different RG with the name (workload-rg and service-rg). For additional information on resource groups, see [Managing resource groups](https://cloud.ibm.com/docs/account?topic=account-rgs). | `string` | `"Default"` | no | +| [scc\_enable](#input\_scc\_enable) | Flag to enable SCC instance creation. If true, an instance of SCC (Security and Compliance Center) will be created. | `bool` | `false` | no | +| [scc\_event\_notification\_plan](#input\_scc\_event\_notification\_plan) | Event Notifications Instance plan to be used (it's used with S.C.C. instance), possible values 'lite' and 'standard'. | `string` | `"lite"` | no | +| [scc\_location](#input\_scc\_location) | Location where the SCC instance is provisioned (possible choices 'us-south', 'eu-de', 'ca-tor', 'eu-es') | `string` | `"us-south"` | no | +| [scc\_profile](#input\_scc\_profile) | Profile to be set on the SCC Instance (accepting empty, 'CIS IBM Cloud Foundations Benchmark' and 'IBM Cloud Framework for Financial Services') | `string` | `"CIS IBM Cloud Foundations Benchmark"` | no | +| [scc\_profile\_version](#input\_scc\_profile\_version) | Version of the Profile to be set on the SCC Instance (accepting empty, CIS and Financial Services profiles versions) | `string` | `"1.0.0"` | no | +| [skip\_iam\_authorization\_policy](#input\_skip\_iam\_authorization\_policy) | Set it to false if authorization policy is required for VPC block storage volumes to access kms. This can be set to true if authorization policy already exists. For more information on how to create authorization policy manually, see [creating authorization policies for block storage volume](https://cloud.ibm.com/docs/vpc?topic=vpc-block-s2s-auth&interface=ui). | `string` | `false` | no | +| [skip\_iam\_share\_authorization\_policy](#input\_skip\_iam\_share\_authorization\_policy) | Set it to false if authorization policy is required for VPC file share to access kms. This can be set to true if authorization policy already exists. For more information on how to create authorization policy manually, see [creating authorization policies for VPC file share](https://cloud.ibm.com/docs/vpc?topic=vpc-file-s2s-auth&interface=ui). | `string` | `false` | no | +| [storage\_security\_group\_id](#input\_storage\_security\_group\_id) | Provide the storage security group ID created from the Spectrum Scale storage cluster if the nfs\_share value is updated to use the scale fileset mountpoints under the cluster\_file\_share variable. | `string` | `null` | no | +| [vpc\_cidr](#input\_vpc\_cidr) | Creates the address prefix for the new VPC, when the vpc\_name variable is empty. The VPC requires an address prefix for creation of subnet in a single zone. The subnet are created with the specified CIDR blocks. For more information, see [Setting IP ranges](https://cloud.ibm.com/docs/vpc?topic=vpc-vpc-addressing-plan-design). | `string` | `"10.241.0.0/18"` | no | +| [vpc\_cluster\_login\_private\_subnets\_cidr\_blocks](#input\_vpc\_cluster\_login\_private\_subnets\_cidr\_blocks) | Provide the CIDR block required for the creation of the login cluster's private subnet. Only one CIDR block is needed. If using a hybrid environment, modify the CIDR block to avoid conflicts with any on-premises CIDR blocks. Since the login subnet is used only for the creation of login virtual server instances, provide a CIDR range of /28. | `list(string)` | 
[
  "10.241.16.0/28"
]
| no | +| [vpc\_cluster\_private\_subnets\_cidr\_blocks](#input\_vpc\_cluster\_private\_subnets\_cidr\_blocks) | Provide the CIDR block required for the creation of the compute cluster's private subnet. One CIDR block is required. If using a hybrid environment, modify the CIDR block to avoid conflicts with any on-premises CIDR blocks. Ensure the selected CIDR block size can accommodate the maximum number of management and dynamic compute nodes expected in your cluster. For more information on CIDR block size selection, refer to the documentation, see [Choosing IP ranges for your VPC](https://cloud.ibm.com/docs/vpc?topic=vpc-choosing-ip-ranges-for-your-vpc). | `list(string)` | 
[
  "10.241.0.0/20"
]
| no | +| [vpc\_name](#input\_vpc\_name) | Name of an existing VPC in which the cluster resources will be deployed. If no value is given, then a new VPC will be provisioned for the cluster. [Learn more](https://cloud.ibm.com/docs/vpc) | `string` | `null` | no | +| [vpn\_enabled](#input\_vpn\_enabled) | Set the value as true to deploy a VPN gateway for VPC in the cluster. | `bool` | `false` | no | +| [zones](#input\_zones) | The IBM Cloud zone name within the selected region where the IBM Cloud HPC cluster should be deployed and requires a single zone input value. Supported zones are: eu-de-2 and eu-de-3 for eu-de, us-east-1 and us-east-3 for us-east, and us-south-1 for us-south. The management nodes, file storage shares, and compute nodes will be deployed in the same zone.[Learn more](https://cloud.ibm.com/docs/vpc?topic=vpc-creating-a-vpc-in-a-different-region#get-zones-using-the-cli). | `list(string)` | 
[
  "us-east-1"
]
| no | ## Outputs | Name | Description | |------|-------------| -| [ssh\_command](#output\_ssh\_command) | SSH command to connect to HPC cluster | +| [application\_center\_tunnel](#output\_application\_center\_tunnel) | Available if IBM Spectrum LSF Application Center GUI is installed | +| [application\_center\_url](#output\_application\_center\_url) | Available if IBM Spectrum LSF Application Center GUI is installed | +| [application\_center\_url\_note](#output\_application\_center\_url\_note) | Available if IBM Spectrum LSF Application Center GUI is installed in High Availability | +| [image\_entry\_found](#output\_image\_entry\_found) | Available if the image name provided is located within the image map | +| [ldap\_hostnames](#output\_ldap\_hostnames) | LDAP nodes have these hostnames: | +| [ldap\_ips](#output\_ldap\_ips) | LDAP nodes have these IPs: | +| [login\_hostnames](#output\_login\_hostnames) | Login nodes have these hostnames: | +| [login\_ips](#output\_login\_ips) | Login nodes have these IPs: | +| [management\_candidate\_hostnames](#output\_management\_candidate\_hostnames) | Management candidate nodes have these hostnames: | +| [management\_candidate\_ips](#output\_management\_candidate\_ips) | Management candidate nodes have these IPs: | +| [management\_hostname](#output\_management\_hostname) | Management node has this hostname: | +| [management\_ip](#output\_management\_ip) | Management node has this IP: | +| [region\_name](#output\_region\_name) | The region name in which the cluster resources have been deployed | +| [remote\_allowed\_cidr](#output\_remote\_allowed\_cidr) | The following IPs/networks are allow-listed for incoming connections | +| [ssh\_to\_ldap\_node](#output\_ssh\_to\_ldap\_node) | SSH command to connect to LDAP node | +| [ssh\_to\_login\_node](#output\_ssh\_to\_login\_node) | SSH command to connect to Login node | +| [ssh\_to\_management\_node\_1](#output\_ssh\_to\_management\_node\_1) | SSH command to connect to HPC cluster | +| [vpc\_name](#output\_vpc\_name) | The VPC name in which the cluster resources have been deployed | diff --git a/solutions/hpc/assets/hpcaas-ce-project-guid.cfg b/solutions/hpc/assets/hpcaas-ce-project-guid.cfg new file mode 100644 index 00000000..2e177188 --- /dev/null +++ b/solutions/hpc/assets/hpcaas-ce-project-guid.cfg @@ -0,0 +1 @@ + diff --git a/solutions/hpc/catalogValidationValues.json.template b/solutions/hpc/catalogValidationValues.json.template deleted file mode 100644 index 1b47cadd..00000000 --- a/solutions/hpc/catalogValidationValues.json.template +++ /dev/null @@ -1,10 +0,0 @@ -{ - "ibmcloud_api_key": $VALIDATION_APIKEY, - "prefix": $PREFIX, - "zones": "[\"ca-tor-1\"]", - "resource_group": "geretain-hpc-rg", - "bastion_ssh_keys": "[\"geretain-hpc-ssh-key\"]", - "login_ssh_keys": "[\"geretain-hpc-ssh-key\"]", - "compute_ssh_keys": "[\"geretain-hpc-ssh-key\"]", - "storage_ssh_keys": "[\"geretain-hpc-ssh-key\"]" -} diff --git a/solutions/hpc/datasource.tf b/solutions/hpc/datasource.tf index f6fada1e..fac7dc67 100644 --- a/solutions/hpc/datasource.tf +++ b/solutions/hpc/datasource.tf @@ -1,22 +1,86 @@ -# Future use -/* -data "ibm_is_region" "itself" { +data "ibm_is_region" "region" { name = local.region } -data "ibm_is_zone" "itself" { - name = var.zones[0] - region = data.ibm_is_region.itself.name +data "ibm_is_vpc" "itself" { + count = var.vpc_name == null ? 0 : 1 + name = var.vpc_name } -*/ -data "ibm_is_vpc" "itself" { - count = var.vpc == null ? 0 : 1 - name = var.vpc +locals { + vpc_name = var.vpc_name == null ? one(module.landing_zone.vpc_name) : var.vpc_name + # region_name = [for zone in var.zones : join("-", slice(split("-", zone), 0, 2))][0] + api_endpoint_region_map = { + "us-east" = "https://api.us-east.codeengine.cloud.ibm.com/v2beta" + "eu-de" = "https://api.eu-de.codeengine.cloud.ibm.com/v2beta" + "us-south" = "https://api.us-south.codeengine.cloud.ibm.com/v2beta" + } + ldap_server_status = var.enable_ldap == true && var.ldap_server == "null" ? false : true + + # Decode the JSON reply got from the Code Engine API + # https://hpc-api..codeengine.cloud.ibm.com/v3/capacity_reservations + reservation_data = jsondecode(data.http.reservation_id_validation.response_body) + # Verify if in the capacity_reservations list there is one with the name equal to the Contract ID. + reservation_id_found = try(length([for res in local.reservation_data.capacity_reservations : res if res.name == var.reservation_id]), 0) > 0 + # Verify if the status code is 200 + valid_status_code = contains(["200"], tostring(data.http.reservation_id_validation.status_code)) + +} + +data "ibm_is_vpc" "existing_vpc" { + # Lookup for this VPC resource only if var.vpc_name is not empty + count = var.vpc_name != null ? 1 : 0 + name = var.vpc_name +} + +data "ibm_is_vpc" "vpc" { + name = local.vpc_name + # Depends on creation of new VPC or look up of existing VPC based on value of var.vpc_name, + depends_on = [module.landing_zone.vpc_name, data.ibm_is_vpc.existing_vpc] +} + +data "ibm_is_vpc_address_prefixes" "existing_vpc" { + #count = var.vpc_name != "" ? 1 : 0 + vpc = data.ibm_is_vpc.vpc.id +} + +data "ibm_is_subnet" "existing_subnet" { + # Lookup for this Subnet resources only if var.cluster_subnet_ids is not empty + count = (length(var.cluster_subnet_ids) == 1 && var.vpc_name != null) ? length(var.cluster_subnet_ids) : 0 + identifier = var.cluster_subnet_ids[count.index] } -/* -data "ibm_is_subnet" "itself" { - count = length(local.subnets) - identifier = local.subnets[count.index]["id"] + +data "ibm_is_subnet" "existing_login_subnet" { + # Lookup for this Subnet resources only if var.login_subnet_id is not empty + count = (var.login_subnet_id != null && var.vpc_name != null) ? 1 : 0 + identifier = var.login_subnet_id +} + +# Validating Contract ID +data "ibm_iam_auth_token" "auth_token" {} + +data "http" "reservation_id_validation" { + url = "${local.api_endpoint_region_map[local.region]}/capacity_reservations" + method = "GET" + request_headers = { + Accept = "application/json" + Authorization = data.ibm_iam_auth_token.auth_token.iam_access_token + # Content-Type = "application/json" + } +} + +# Code for Public Gateway attachment for the existing vpc and new subnets scenario + +data "ibm_is_public_gateways" "public_gateways" { +} + +locals { + public_gateways_list = data.ibm_is_public_gateways.public_gateways.public_gateways + zone_1_pgw_ids = var.vpc_name != null ? [for gateway in local.public_gateways_list : gateway.id if gateway.vpc == local.vpc_id && gateway.zone == var.zones[0]] : [] +} + +resource "ibm_is_subnet_public_gateway_attachment" "zone_1_attachment" { + count = (var.vpc_name != null && length(var.cluster_subnet_ids) == 0) ? 1 : 0 + subnet = local.compute_subnets[0].id + public_gateway = length(local.zone_1_pgw_ids) > 0 ? local.zone_1_pgw_ids[0] : "" } -*/ diff --git a/solutions/hpc/input_validation.tf b/solutions/hpc/input_validation.tf new file mode 100644 index 00000000..decf2e94 --- /dev/null +++ b/solutions/hpc/input_validation.tf @@ -0,0 +1,236 @@ +################################################### +# Copyright (C) IBM Corp. 2023 All Rights Reserved. +# Licensed under the Apache License v2.0 +################################################### + +# This file contains the complete information on all the validations performed from the code during the generate plan process +# Validations are performed to make sure, the appropriate error messages are displayed to user in-order to provide required input parameter + +# Module for the private cluster_subnet and login subnet cidr validation. +module "ipvalidation_cluster_subnet" { + count = length(var.vpc_cluster_private_subnets_cidr_blocks) + source = "../../modules/custom/subnet_cidr_check" + subnet_cidr = var.vpc_cluster_private_subnets_cidr_blocks[count.index] + vpc_address_prefix = [local.prefixes_in_given_zone_1][count.index] +} + +module "ipvalidation_login_subnet" { + source = "../../modules/custom/subnet_cidr_check" + subnet_cidr = var.vpc_cluster_login_private_subnets_cidr_blocks[0] + vpc_address_prefix = local.prefixes_in_given_zone_login +} + +locals { + # Copy address prefixes and CIDR of given zone into a new tuple + prefixes_in_given_zone_login = [ + for prefix in data.ibm_is_vpc_address_prefixes.existing_vpc[*].address_prefixes[0] : + prefix.cidr if prefix.zone[0].name == var.zones[0]] + + # To get the address prefix of zone1 + prefixes_in_given_zone_1 = [ + for prefix in data.ibm_is_vpc_address_prefixes.existing_vpc[*].address_prefixes[0] : + prefix.cidr if var.zones[0] == prefix.zone[0].name] + + # validation for the boot volume encryption toggling. + validate_enable_customer_managed_encryption = anytrue([alltrue([var.kms_key_name != null, var.kms_instance_name != null]), (var.kms_key_name == null), (var.key_management != "key_protect")]) + validate_enable_customer_managed_encryption_msg = "Please make sure you are passing the kms_instance_name if you are passing kms_key_name." + # tflint-ignore: terraform_unused_declarations + validate_enable_customer_managed_encryption_chk = regex( + "^${local.validate_enable_customer_managed_encryption_msg}$", + (local.validate_enable_customer_managed_encryption ? local.validate_enable_customer_managed_encryption_msg : "")) + + # validation for the boot volume encryption toggling. + validate_null_customer_managed_encryption = anytrue([alltrue([var.kms_instance_name == null, var.key_management != "key_protect"]), (var.key_management == "key_protect")]) + validate_null_customer_managed_encryption_msg = "Please make sure you are setting key_management as key_protect if you are passing kms_instance_name, kms_key_name." + # tflint-ignore: terraform_unused_declarations + validate_null_customer_managed_encryption_chk = regex( + "^${local.validate_null_customer_managed_encryption_msg}$", + (local.validate_null_customer_managed_encryption ? local.validate_null_customer_managed_encryption_msg : "")) + + # validate application center gui password + password_msg = "Password should be at least 8 characters, must have one number, one lowercase letter, and one uppercase letter, at least one unique character. Password Should not contain username" + validate_app_center_gui_pwd = (var.enable_app_center && can(regex("^.{8,}$", var.app_center_gui_pwd) != "") && can(regex("[0-9]{1,}", var.app_center_gui_pwd) != "") && can(regex("[a-z]{1,}", var.app_center_gui_pwd) != "") && can(regex("[A-Z]{1,}", var.app_center_gui_pwd) != "") && can(regex("[!@#$%^&*()_+=-]{1,}", var.app_center_gui_pwd) != "") && trimspace(var.app_center_gui_pwd) != "") || !var.enable_app_center + # tflint-ignore: terraform_unused_declarations + validate_app_center_gui_pwd_chk = regex( + "^${local.password_msg}$", + (local.validate_app_center_gui_pwd ? local.password_msg : "")) + + # Validate existing cluster subnet should be the subset of vpc_name entered + validate_subnet_id_vpc_msg = "Provided cluster subnets should be within the vpc entered." + validate_subnet_id_vpc = anytrue([length(var.cluster_subnet_ids) == 0, length(var.cluster_subnet_ids) == 1 && var.vpc_name != null ? alltrue([for subnet_id in var.cluster_subnet_ids : contains(data.ibm_is_vpc.existing_vpc[0].subnets[*].id, subnet_id)]) : false]) + # tflint-ignore: terraform_unused_declarations + validate_subnet_id_vpc_chk = regex("^${local.validate_subnet_id_vpc_msg}$", + (local.validate_subnet_id_vpc ? local.validate_subnet_id_vpc_msg : "")) + + # Validate existing cluster subnet should be in the appropriate zone. + validate_subnet_id_zone_msg = "Provided cluster subnets should be in appropriate zone." + validate_subnet_id_zone = anytrue([length(var.cluster_subnet_ids) == 0, length(var.cluster_subnet_ids) == 1 && var.vpc_name != null ? alltrue([data.ibm_is_subnet.existing_subnet[0].zone == var.zones[0]]) : false]) + # tflint-ignore: terraform_unused_declarations + validate_subnet_id_zone_chk = regex("^${local.validate_subnet_id_zone_msg}$", + (local.validate_subnet_id_zone ? local.validate_subnet_id_zone_msg : "")) + + # Validate existing login subnet should be the subset of vpc_name entered + validate_login_subnet_id_vpc_msg = "Provided login subnet should be within the vpc entered." + validate_login_subnet_id_vpc = anytrue([var.login_subnet_id == null, var.login_subnet_id != null && var.vpc_name != null ? alltrue([for subnet_id in [var.login_subnet_id] : contains(data.ibm_is_vpc.existing_vpc[0].subnets[*].id, subnet_id)]) : false]) + # tflint-ignore: terraform_unused_declarations + validate_login_subnet_id_vpc_chk = regex("^${local.validate_login_subnet_id_vpc_msg}$", + (local.validate_login_subnet_id_vpc ? local.validate_login_subnet_id_vpc_msg : "")) + + # Validate existing login subnet should be in the appropriate zone. + validate_login_subnet_id_zone_msg = "Provided login subnet should be in appropriate zone." + validate_login_subnet_id_zone = anytrue([var.login_subnet_id == null, var.login_subnet_id != null && var.vpc_name != null ? alltrue([data.ibm_is_subnet.existing_login_subnet[0].zone == var.zones[0]]) : false]) + # tflint-ignore: terraform_unused_declarations + validate_login_subnet_id_zone_chk = regex("^${local.validate_login_subnet_id_zone_msg}$", + (local.validate_login_subnet_id_zone ? local.validate_login_subnet_id_zone_msg : "")) + + # Contract ID validation + validate_reservation_id = length("${var.cluster_id}${var.reservation_id}") > 129 ? false : true + validate_reservation_id_msg = "The length of reservation_id and cluster_id combination should not exceed 128 characters." + # tflint-ignore: terraform_unused_declarations + validate_reservation_id_chk = regex( + "^${local.validate_reservation_id_msg}$", + (local.validate_reservation_id ? local.validate_reservation_id_msg : "")) + + validate_reservation_id_api = local.valid_status_code && local.reservation_id_found + validate_reservation_id_api_msg = "The provided reservation id doesn't have a valid reservation or the reservation id is not on the same account as HPC deployment." + # tflint-ignore: terraform_unused_declarations + validate_reservation_id_api_chk = regex( + "^${local.validate_reservation_id_api_msg}$", + (local.validate_reservation_id_api ? local.validate_reservation_id_api_msg : "")) + + # Validate custom fileshare + # Construct a list of Share size(GB) and IOPS range(IOPS)from values provided in https://cloud.ibm.com/docs/vpc?topic=vpc-file-storage-profiles&interface=ui#dp2-profile + # List values [[sharesize_start,sharesize_end,min_iops,max_iops], [..]....] + custom_fileshare_iops_range = [[10, 39, 100, 1000], [40, 79, 100, 2000], [80, 99, 100, 4000], [100, 499, 100, 6000], [500, 999, 100, 10000], [1000, 1999, 100, 20000], [2000, 3999, 200, 40000], [4000, 7999, 300, 40000], [8000, 15999, 500, 64000], [16000, 32000, 2000, 96000]] + # List with input iops value, min and max iops for the input share size. + size_iops_lst = [for values in var.custom_file_shares : [for list_val in local.custom_fileshare_iops_range : [values.size != null ? (values.iops != null ? (values.size >= list_val[0] && values.size <= list_val[1] ? values.iops : null) : null) : null, list_val[2], list_val[3]] if values.size != null]] + validate_custom_file_share = alltrue([for iops in local.size_iops_lst : (length(iops) > 0 ? (iops[0][0] != null ? (iops[0][0] >= iops[0][1] && iops[0][0] <= iops[0][2]) : true) : true)]) + # Validate the input iops falls inside the range. + # validate_custom_file_share = alltrue([for iops in local.size_iops_lst : iops[0][0] >= iops[0][1] && iops[0][0] <= iops[0][2]]) + validate_custom_file_share_msg = "Provided iops value is not valid for given file share size. Please refer 'File Storage for VPC profiles' page in ibm cloud docs for a valid iops and file share size combination." + # tflint-ignore: terraform_unused_declarations + validate_custom_file_share_chk = regex( + "^${local.validate_custom_file_share_msg}$", + (local.validate_custom_file_share ? local.validate_custom_file_share_msg : "")) + + # LDAP base DNS Validation + validate_ldap_basedns = (var.enable_ldap && trimspace(var.ldap_basedns) != "") || !var.enable_ldap + ldap_basedns_msg = "If LDAP is enabled, then the base DNS should not be empty or null. Need a valid domain name." + # tflint-ignore: terraform_unused_declarations + validate_ldap_basedns_chk = regex( + "^${local.ldap_basedns_msg}$", + (local.validate_ldap_basedns ? local.ldap_basedns_msg : "")) + + # LDAP base existing LDAP server + validate_ldap_server = (var.enable_ldap && trimspace(var.ldap_server) != "") || !var.enable_ldap + ldap_server_msg = "IP of existing LDAP server. If none given a new ldap server will be created. It should not be empty." + # tflint-ignore: terraform_unused_declarations + validate_ldap_server_chk = regex( + "^${local.ldap_server_msg}$", + (local.validate_ldap_server ? local.ldap_server_msg : "")) + + # LDAP Admin Password Validation + validate_ldap_adm_pwd = var.enable_ldap && var.ldap_server == "null" ? (length(var.ldap_admin_password) >= 8 && length(var.ldap_admin_password) <= 20 && can(regex("^(.*[0-9]){2}.*$", var.ldap_admin_password))) && can(regex("^(.*[A-Z]){1}.*$", var.ldap_admin_password)) && can(regex("^(.*[a-z]){1}.*$", var.ldap_admin_password)) && can(regex("^.*[~@_+:].*$", var.ldap_admin_password)) && can(regex("^[^!#$%^&*()=}{\\[\\]|\\\"';?.<,>-]+$", var.ldap_admin_password)) : local.ldap_server_status + ldap_adm_password_msg = "Password that is used for LDAP admin.The password must contain at least 8 characters and at most 20 characters. For a strong password, at least three alphabetic characters are required, with at least one uppercase and one lowercase letter. Two numbers, and at least one special character. Make sure that the password doesn't include the username." + # tflint-ignore: terraform_unused_declarations + validate_ldap_adm_pwd_chk = regex( + "^${local.ldap_adm_password_msg}$", + (local.validate_ldap_adm_pwd ? local.ldap_adm_password_msg : "")) + + # LDAP User Validation + validate_ldap_usr = var.enable_ldap && var.ldap_server == "null" ? (length(var.ldap_user_name) >= 4 && length(var.ldap_user_name) <= 32 && var.ldap_user_name != "" && can(regex("^[a-zA-Z0-9_-]*$", var.ldap_user_name)) && trimspace(var.ldap_user_name) != "") : local.ldap_server_status + ldap_usr_msg = "The input for 'ldap_user_name' is considered invalid. The username must be within the range of 4 to 32 characters and may only include letters, numbers, hyphens, and underscores. Spaces are not permitted." + # tflint-ignore: terraform_unused_declarations + validate_ldap_usr_chk = regex( + "^${local.ldap_usr_msg}$", + (local.validate_ldap_usr ? local.ldap_usr_msg : "")) + + # LDAP User Password Validation + validate_ldap_usr_pwd = var.enable_ldap && var.ldap_server == "null" ? (length(var.ldap_user_password) >= 8 && length(var.ldap_user_password) <= 20 && can(regex("^(.*[0-9]){2}.*$", var.ldap_user_password))) && can(regex("^(.*[A-Z]){1}.*$", var.ldap_user_password)) && can(regex("^(.*[a-z]){1}.*$", var.ldap_user_password)) && can(regex("^.*[~@_+:].*$", var.ldap_user_password)) && can(regex("^[^!#$%^&*()=}{\\[\\]|\\\"';?.<,>-]+$", var.ldap_user_password)) : local.ldap_server_status + ldap_usr_password_msg = "Password that is used for LDAP user.The password must contain at least 8 characters and at most 20 characters. For a strong password, at least three alphabetic characters are required, with at least one uppercase and one lowercase letter. Two numbers, and at least one special character. Make sure that the password doesn't include the username." + # tflint-ignore: terraform_unused_declarations + validate_ldap_usr_pwd_chk = regex( + "^${local.ldap_usr_password_msg}$", + (local.validate_ldap_usr_pwd ? local.ldap_usr_password_msg : "")) + + # Validate existing subnet public gateways + validate_subnet_name_pg_msg = "Provided existing cluster_subnet_ids should have public gateway attached." + validate_subnet_name_pg = anytrue([length(var.cluster_subnet_ids) == 0, length(var.cluster_subnet_ids) == 1 && var.vpc_name != null ? (data.ibm_is_subnet.existing_subnet[0].public_gateway != "") : false]) + # tflint-ignore: terraform_unused_declarations + validate_subnet_name_pg_chk = regex("^${local.validate_subnet_name_pg_msg}$", + (local.validate_subnet_name_pg ? local.validate_subnet_name_pg_msg : "")) + + # Validate existing vpc public gateways + validate_existing_vpc_pgw_msg = "Provided existing vpc should have the public gateways created in the provided zones." + validate_existing_vpc_pgw = anytrue([(var.vpc_name == null), alltrue([var.vpc_name != null, length(var.cluster_subnet_ids) == 1]), alltrue([var.vpc_name != null, length(var.cluster_subnet_ids) == 0, var.login_subnet_id == null, length(local.zone_1_pgw_ids) > 0])]) + # tflint-ignore: terraform_unused_declarations + validate_existing_vpc_pgw_chk = regex("^${local.validate_existing_vpc_pgw_msg}$", + (local.validate_existing_vpc_pgw ? local.validate_existing_vpc_pgw_msg : "")) + + # Validate in case of existing subnets provide both login_subnet_id and cluster_subnet_ids. + validate_login_subnet_id_msg = "In case of existing subnets provide both login_subnet_id and cluster_subnet_ids." + validate_login_subnet_id = anytrue([alltrue([length(var.cluster_subnet_ids) == 0, var.login_subnet_id == null]), alltrue([length(var.cluster_subnet_ids) != 0, var.login_subnet_id != null])]) + # tflint-ignore: terraform_unused_declarations + validate_login_subnet_id_chk = regex("^${local.validate_login_subnet_id_msg}$", + (local.validate_login_subnet_id ? local.validate_login_subnet_id_msg : "")) + + # Validate the subnet_id user input value + validate_subnet_id_msg = "If the cluster_subnet_ids are provided, the user should also provide the vpc_name." + validate_subnet_id = anytrue([var.vpc_name != null && length(var.cluster_subnet_ids) > 0, length(var.cluster_subnet_ids) == 0]) + # tflint-ignore: terraform_unused_declarations + validate_subnet_id_chk = regex("^${local.validate_subnet_id_msg}$", + (local.validate_subnet_id ? local.validate_subnet_id_msg : "")) + + # Management node count validation when Application Center is in High Availability + validate_management_node_count = (var.enable_app_center && var.app_center_high_availability && var.management_node_count >= 2) || !var.app_center_high_availability || !var.enable_app_center + management_node_count_msg = "When the Application Center is installed in High Availability, at least two management nodes must be installed." + # tflint-ignore: terraform_unused_declarations + validate_management_node_count_chk = regex( + "^${local.management_node_count_msg}$", + (local.validate_management_node_count ? local.management_node_count_msg : "")) + + # IBM Cloud Application load Balancer CRN validation + validate_alb_crn = (var.enable_app_center && var.app_center_high_availability) && can(regex("^crn:v1:bluemix:public:secrets-manager:[a-zA-Z\\-]+:[a-zA-Z0-9\\-]+\\/[a-zA-Z0-9\\-]+:[a-fA-F0-9\\-]+:secret:[a-fA-F0-9\\-]+$", var.existing_certificate_instance)) || !var.app_center_high_availability || !var.enable_app_center + alb_crn_template_msg = "When app_center_high_availability is enable/set as true, The Application Center will be configured for high availability and requires a Application Load Balancer Front End listener to use a certificate CRN value stored in the Secret Manager. Provide the valid 'existing_certificate_instance' to configure the Application load balancer." + # tflint-ignore: terraform_unused_declarations + validate_alb_crn_chk = regex( + "^${local.alb_crn_template_msg}$", + (local.validate_alb_crn ? local.alb_crn_template_msg : "")) + + # Validate the dns_custom_resolver_id should not be given in case of new vpc case + validate_custom_resolver_id_msg = "If it is the new vpc deployment, do not provide existing dns_custom_resolver_id as that will impact the name resolution of the cluster." + validate_custom_resolver_id = anytrue([var.vpc_name != null, var.vpc_name == null && var.dns_custom_resolver_id == null]) + # tflint-ignore: terraform_unused_declarations + validate_custom_resolver_id_chk = regex("^${local.validate_custom_resolver_id_msg}$", + (local.validate_custom_resolver_id ? local.validate_custom_resolver_id_msg : "")) + + validate_reservation_id_new_msg = "Provided cluster_id and reservation id cannot be set as empty if the provided region is eu-de and us-east and us-south." + validate_reservation_id_logic = local.region == "eu-de" || local.region == "us-east" || local.region == "us-south" ? var.reservation_id != "" && var.cluster_id != "" : true + # tflint-ignore: terraform_unused_declarations + validate_reservation_id_chk_new = regex("^${local.validate_reservation_id_new_msg}$", + (local.validate_reservation_id_logic ? local.validate_reservation_id_new_msg : "")) + + # IBM Cloud Monitoring validation + validate_observability_monitoring_enable_compute_nodes = (var.observability_monitoring_enable && var.observability_monitoring_on_compute_nodes_enable) || (var.observability_monitoring_enable && var.observability_monitoring_on_compute_nodes_enable == false) || (var.observability_monitoring_enable == false && var.observability_monitoring_on_compute_nodes_enable == false) + observability_monitoring_enable_compute_nodes_msg = "Please enable also IBM Cloud Monitoring to ingest metrics from Compute nodes" + # tflint-ignore: terraform_unused_declarations + observability_monitoring_enable_compute_nodes_chk = regex( + "^${local.observability_monitoring_enable_compute_nodes_msg}$", + (local.validate_observability_monitoring_enable_compute_nodes ? local.observability_monitoring_enable_compute_nodes_msg : "")) + + # Existing Bastion validation + validate_existing_bastion = var.bastion_instance_name != null ? (var.bastion_instance_public_ip != null && var.bastion_security_group_id != null && var.bastion_ssh_private_key != null) : local.bastion_instance_status + validate_existing_bastion_msg = "If bastion_instance_name is not null, then bastion_instance_public_ip, bastion_security_group_id, and bastion_ssh_private_key should not be null." + # tflint-ignore: terraform_unused_declarations + validate_existing_bastion_chk = regex( + "^${local.validate_existing_bastion_msg}$", + (local.validate_existing_bastion ? local.validate_existing_bastion_msg : "")) + + # Existing Storage security group validation + validate_existing_storage_sg = length([for share in var.custom_file_shares : { mount_path = share.mount_path, nfs_share = share.nfs_share } if share.nfs_share != null && share.nfs_share != ""]) > 0 ? var.storage_security_group_id != null ? true : false : true + validate_existing_storage_sg_msg = "Storage security group ID cannot be null when NFS share mount path is provided under cluster_file_shares variable." + # tflint-ignore: terraform_unused_declarations + validate_existing_storage_sg_chk = regex( + "^${local.validate_existing_storage_sg_msg}$", + (local.validate_existing_storage_sg ? local.validate_existing_storage_sg_msg : "")) +} diff --git a/solutions/hpc/locals.tf b/solutions/hpc/locals.tf index 8aa73a49..24d5a575 100644 --- a/solutions/hpc/locals.tf +++ b/solutions/hpc/locals.tf @@ -1,3 +1,29 @@ +########################################################################### +locals { + # (overridable) switch to enable extra outputs (debugging) + print_extra_outputs = false + + # (overridable) switch to add the current (plan execution) IP to allowed CIDR list + add_current_ip_to_allowed_cidr = false + + # (overridable) list of extra entries for allowed CIDR list + remote_allowed_ips_extra = [] +} + +########################################################################### +# Local tweaks support +########################################################################### +# You can enable local tweaks files to customize your local deployment with things +# never intended to be included in the standard code. +# You can use override files to override some values of switches (see above) +# or you can force other values defined in the plan or include extra resources. +# +# See the directory "localtweak_examples" for more. + + +########################################################################### +########################################################################### +########################################################################### # locals needed for landing_zone locals { # Region and Zone calculations @@ -7,44 +33,45 @@ locals { # locals needed for bootstrap locals { # dependency: landing_zone -> bootstrap - vpc_id = var.vpc == null ? one(module.landing_zone.vpc_id) : var.vpc - bastion_subnets = module.landing_zone.bastion_subnets - kms_encryption_enabled = var.key_management != null ? true : false - boot_volume_encryption_key = var.key_management != null ? one(module.landing_zone.boot_volume_encryption_key)["crn"] : null - existing_kms_instance_guid = var.key_management != null ? module.landing_zone.key_management_guid : null - # Future use - # skip_iam_authorization_policy = true + vpc_id = var.vpc_name == null ? one(module.landing_zone.vpc_id) : data.ibm_is_vpc.itself[0].id + bastion_subnets = length(var.cluster_subnet_ids) == 0 ? module.landing_zone.bastion_subnets : local.sorted_subnets + kms_encryption_enabled = var.key_management == "key_protect" ? true : false + boot_volume_encryption_key = var.key_management == "key_protect" ? one(module.landing_zone.boot_volume_encryption_key)["crn"] : null + existing_kms_instance_guid = var.key_management == "key_protect" ? module.landing_zone.key_management_guid : null + cluster_id = local.region == "eu-de" || local.region == "us-east" || local.region == "us-south" ? var.cluster_id : "HPC-LSF-1" } # locals needed for landing_zone_vsi locals { - # dependency: landing_zone -> bootstrap -> landing_zone_vsi - bastion_security_group_id = module.bootstrap.bastion_security_group_id - bastion_public_key_content = module.bootstrap.bastion_public_key_content + bastion_security_group_id = module.bootstrap.bastion_security_group_id + bastion_public_key_content = module.bootstrap.bastion_public_key_content + bastion_private_key_content = module.bootstrap.bastion_private_key_content + compute_private_key_content = module.landing_zone_vsi.compute_private_key_content # dependency: landing_zone -> landing_zone_vsi - login_subnets = module.landing_zone.login_subnets - compute_subnets = module.landing_zone.compute_subnets - storage_subnets = module.landing_zone.storage_subnets - protocol_subnets = module.landing_zone.protocol_subnets - #boot_volume_encryption_key = var.key_management != null ? one(module.landing_zone.boot_volume_encryption_key)["crn"] : null - #skip_iam_authorization_policy = true - #resource_group_id = data.ibm_resource_group.itself.id - #vpc_id = var.vpc == null ? module.landing_zone.vpc_id[0] : data.ibm_is_vpc.itself[0].id - #vpc_crn = var.vpc == null ? module.landing_zone.vpc_crn[0] : data.ibm_is_vpc.itself[0].crn + subnets_output = module.landing_zone.subnets + + sorted_subnets = length(var.cluster_subnet_ids) != 0 ? [ + element(local.subnets_output, index(local.subnets_output[*].id, var.cluster_subnet_ids[0])), + element(local.subnets_output, index(local.subnets_output[*].id, var.login_subnet_id)) + ] : [] + + sorted_compute_subnets = length(var.cluster_subnet_ids) == 0 ? [ + element(module.landing_zone.compute_subnets, index(module.landing_zone.compute_subnets[*].zone, var.zones[0])) + ] : [] + + + compute_subnets = length(var.cluster_subnet_ids) == 0 ? local.sorted_compute_subnets : local.sorted_subnets } # locals needed for file-storage locals { - # dependency: landing_zone -> file-storage - #vpc_id = var.vpc == null ? one(module.landing_zone.vpc_id) : var.vpc - #boot_volume_encryption_key = var.key_management != null ? one(module.landing_zone.boot_volume_encryption_key)["crn"] : null # dependency: landing_zone_vsi -> file-share compute_subnet_id = local.compute_subnets[0].id compute_security_group_id = module.landing_zone_vsi.compute_sg_id - management_instance_count = sum(var.management_instances[*]["count"]) + management_instance_count = var.management_node_count default_share = local.management_instance_count > 0 ? [ { mount_path = "/mnt/lsf" @@ -52,12 +79,24 @@ locals { iops = 1000 } ] : [] - storage_instance_count = sum(var.storage_instances[*]["count"]) - total_shares = local.storage_instance_count > 0 ? [] : concat(local.default_share, var.file_shares) + + vpc_file_share = [ + for share in var.custom_file_shares : + { + mount_path = share.mount_path + size = share.size + iops = share.iops + } + if share.size != null && share.iops != null + ] + + total_shares = concat(local.default_share, local.vpc_file_share) + + # total_shares = 10 file_shares = [ for count in range(length(local.total_shares)) : { - name = format("%s-%s", var.prefix, element(split("/", local.total_shares[count]["mount_path"]), length(split("/", local.total_shares[count]["mount_path"])) - 1)) + name = format("%s-%s", var.cluster_prefix, element(split("/", local.total_shares[count]["mount_path"]), length(split("/", local.total_shares[count]["mount_path"])) - 1)) size = local.total_shares[count]["size"] iops = local.total_shares[count]["iops"] } @@ -67,16 +106,14 @@ locals { # locals needed for DNS locals { # dependency: landing_zone -> DNS - resource_group_id = one(values(one(module.landing_zone.resource_group_id))) - vpc_crn = var.vpc == null ? one(module.landing_zone.vpc_crn) : one(data.ibm_is_vpc.itself[*].crn) + resource_groups = { + service_rg = var.resource_group == "null" ? module.landing_zone.resource_group_id[0]["service-rg"] : one(values(one(module.landing_zone.resource_group_id))) + workload_rg = var.resource_group == "null" ? module.landing_zone.resource_group_id[0]["workload-rg"] : one(values(one(module.landing_zone.resource_group_id))) + } + vpc_crn = var.vpc_name == null ? one(module.landing_zone.vpc_crn) : one(data.ibm_is_vpc.itself[*].crn) # TODO: Fix existing subnet logic - #subnets_crn = var.vpc == null ? module.landing_zone.subnets_crn : ### - #subnets = flatten([local.compute_subnets, local.storage_subnets, local.protocol_subnets]) - #subnets_crns = data.ibm_is_subnet.itself[*].crn - subnets_crn = module.landing_zone.subnets_crn - #boot_volume_encryption_key = var.key_management != null ? one(module.landing_zone.boot_volume_encryption_key)["crn"] : null - - # dependency: landing_zone_vsi -> file-share + # subnets_crn = module.landing_zone.subnets_crn + compute_subnets_crn = length(var.cluster_subnet_ids) == 0 ? local.sorted_compute_subnets[*].crn : local.sorted_subnets[*].crn } # locals needed for dns-records @@ -84,36 +121,50 @@ locals { # dependency: dns -> dns-records dns_instance_id = module.dns.dns_instance_id compute_dns_zone_id = one(flatten([ - for dns_zone in module.dns.dns_zone_maps : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["compute"] - ])) - storage_dns_zone_id = one(flatten([ - for dns_zone in module.dns.dns_zone_maps : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["storage"] - ])) - protocol_dns_zone_id = one(flatten([ - for dns_zone in module.dns.dns_zone_maps : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_names["protocol"] + for dns_zone in module.dns.dns_zone_maps : values(dns_zone) if one(keys(dns_zone)) == var.dns_domain_name["compute"] ])) + # dependency: landing_zone_vsi -> dns-records - compute_instances = flatten([module.landing_zone_vsi.management_vsi_data, module.landing_zone_vsi.compute_vsi_data]) - storage_instances = flatten([module.landing_zone_vsi.storage_vsi_data, module.landing_zone_vsi.protocol_vsi_data]) - protocol_instances = flatten([module.landing_zone_vsi.protocol_vsi_data]) + management_vsi_data = flatten(module.landing_zone_vsi.management_vsi_data) + management_private_ip = local.management_vsi_data[0]["ipv4_address"] + management_hostname = local.management_vsi_data[0]["name"] + + management_candidate_vsi_data = flatten(module.landing_zone_vsi.management_candidate_vsi_data) + management_candidate_private_ips = local.management_candidate_vsi_data[*]["ipv4_address"] + management_candidate_hostnames = local.management_candidate_vsi_data[*]["name"] + + login_vsi_data = flatten(module.landing_zone_vsi.login_vsi_data) + login_private_ips = local.login_vsi_data[*]["ipv4_address"] + login_hostnames = local.login_vsi_data[*]["name"] + + ldap_vsi_data = flatten(module.landing_zone_vsi.ldap_vsi_data) + ldap_private_ips = local.ldap_vsi_data[*]["ipv4_address"] + ldap_hostnames = local.ldap_vsi_data[*]["name"] compute_dns_records = [ - for instance in local.compute_instances : + for instance in local.management_vsi_data : + { + name = instance["name"] + rdata = instance["ipv4_address"] + } + ] + mgmt_candidate_dns_records = [ + for instance in local.management_candidate_vsi_data : { name = instance["name"] rdata = instance["ipv4_address"] } ] - storage_dns_records = [ - for instance in local.storage_instances : + login_vsi_dns_records = [ + for instance in local.login_vsi_data : { name = instance["name"] rdata = instance["ipv4_address"] } ] - protocol_dns_records = [ - for instance in local.protocol_instances : + ldap_vsi_dns_records = [ + for instance in local.ldap_vsi_data : { name = instance["name"] rdata = instance["ipv4_address"] @@ -123,17 +174,87 @@ locals { # locals needed for inventory locals { - compute_hosts = local.compute_instances[*]["ipv4_address"] - storage_hosts = local.storage_instances[*]["ipv4_address"] + compute_hosts = concat(local.management_vsi_data[*]["ipv4_address"], local.management_candidate_vsi_data[*]["ipv4_address"]) compute_inventory_path = "compute.ini" - storage_inventory_path = "storage.ini" + bastion_inventory_path = "bastion.ini" + login_inventory_path = "login.ini" + ldap_inventory_path = "ldap.ini" + + bastion_host = local.bastion_instance_public_ip != null ? [local.bastion_instance_public_ip] : var.enable_fip ? [local.bastion_fip, local.bastion_primary_ip] : [local.bastion_primary_ip, ""] + login_host = local.login_private_ips + ldap_host = local.ldap_private_ips } # locals needed for playbook locals { - bastion_fip = module.bootstrap.bastion_fip - compute_private_key_path = "compute_id_rsa" #checkov:skip=CKV_SECRET_6 - storage_private_key_path = "storage_id_rsa" #checkov:skip=CKV_SECRET_6 - compute_playbook_path = "compute_ssh.yaml" - storage_playbook_path = "storage_ssh.yaml" + cluster_user = "lsfadmin" + login_user = "ubuntu" + ldap_user = "ubuntu" + bastion_primary_ip = module.bootstrap.bastion_primary_ip + bastion_fip = module.bootstrap.bastion_fip[0] + bastion_fip_id = module.bootstrap.bastion_fip_id + no_addr_prefix = true +} + +locals { + share_path = module.file_storage.mount_path_1 +} + +########################################################################### +# IBM Cloud Dababase for MySQL local variables +########################################################################### +locals { + mysql_version = "8.0" + db_service_endpoints = "private" + db_template = [3, 12288, 122880, 3] +} + +########################################################################### +# IBM Application Load Balancer variables +########################################################################### + +locals { + # alb_created_by_api: + # - true -> use ibmcloud API + # - false -> use ibmcloud terraform provider (not recommended, dramatically slower) + alb_created_by_api = true + alb_hostname = local.alb_created_by_api ? module.alb_api.alb_hostname : module.alb.alb_hostname +} + +locals { + vsi_management_ids = [ + for instance in concat(local.management_vsi_data, local.management_candidate_vsi_data) : + { + id = instance["id"] + } + ] +} + +# locals needed for ssh connection +locals { + ssh_forward_host = (var.app_center_high_availability ? "pac.${var.dns_domain_name.compute}" : "localhost") + ssh_forwards = "-L 8443:${local.ssh_forward_host}:8443 -L 6080:${local.ssh_forward_host}:6080" + ssh_jump_host = local.bastion_instance_public_ip != null ? local.bastion_instance_public_ip : var.enable_fip ? module.bootstrap.bastion_fip[0] : module.bootstrap.bastion_primary_ip + ssh_jump_option = "-J ubuntu@${local.ssh_jump_host}" + ssh_host = var.app_center_high_availability ? local.login_private_ips[0] : local.management_private_ip + ssh_cmd = "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=5 -o ServerAliveCountMax=1 ${local.ssh_forwards} ${local.ssh_jump_option} lsfadmin@${local.ssh_host}" +} + +# Existing bastion Variables +locals { + # bastion_instance_name = var.bastion_instance_name != null ? var.bastion_instance_name : null + bastion_instance_public_ip = var.bastion_instance_name != null ? var.bastion_instance_public_ip : null + # bastion_security_group_id = var.bastion_instance_name != null ? var.bastion_security_group_id : module.bootstrap.bastion_security_group_id + bastion_ssh_private_key = var.bastion_instance_name != null ? var.bastion_ssh_private_key : null + bastion_instance_status = var.bastion_instance_name != null ? false : true + + existing_subnet_cidrs = var.vpc_name != null && length(var.cluster_subnet_ids) == 1 ? [data.ibm_is_subnet.existing_subnet[0].ipv4_cidr_block, data.ibm_is_subnet.existing_login_subnet[0].ipv4_cidr_block] : [] +} + +#################################################### +# The code below does some internal processing of variables and locals +# (e.g. concatenating lists). + +locals { + allowed_cidr = concat(var.remote_allowed_ips, local.remote_allowed_ips_extra, local.add_current_ip_to_allowed_cidr ? module.my_ip.my_cidr : []) } diff --git a/solutions/hpc/localtweak_examples/.gitignore b/solutions/hpc/localtweak_examples/.gitignore new file mode 100644 index 00000000..e094084d --- /dev/null +++ b/solutions/hpc/localtweak_examples/.gitignore @@ -0,0 +1 @@ +!localtweak__* diff --git a/solutions/hpc/localtweak_examples/README.md b/solutions/hpc/localtweak_examples/README.md new file mode 100644 index 00000000..85c66b91 --- /dev/null +++ b/solutions/hpc/localtweak_examples/README.md @@ -0,0 +1,10 @@ +The files in this directory are not normally included in the Terraform plan. +If you want to enable one of them in your deployment, just copy it to the base directory of the project (one level up from here), +or make a symlink if no customization is needed. Always be sure the new name ends with "*.tf". +For example: + cd solutions/hpc + ln -s localtweak_examples/localtweak___ALLOW_MY_IP_override.tf.txt localtweak___ALLOW_MY_IP_override.tf + +They add resources and/or use the "override" Terraform feature to modify the local deployment in ways that you do not want to commit to the standard code. +(note how the .gitignore file in the root directory mentions "localtweak__*.tf" files to avoid committing them accidentally) +In some cases additional configuration is required on your system (see "*_extras" directories). diff --git a/solutions/hpc/localtweak_examples/localtweak___ALLOW_MY_IP_override.tf.txt b/solutions/hpc/localtweak_examples/localtweak___ALLOW_MY_IP_override.tf.txt new file mode 100644 index 00000000..63aa9ce1 --- /dev/null +++ b/solutions/hpc/localtweak_examples/localtweak___ALLOW_MY_IP_override.tf.txt @@ -0,0 +1,6 @@ +# This localtweak will detect your current internet visible address and add it to the +# allowed_ips list, so you are sure you will be able to access the cluster. + +locals { + add_current_ip_to_allowed_cidr = true +} diff --git a/solutions/hpc/localtweak_examples/localtweak___FORCE_VALID_PASSWORD_override.tf.txt b/solutions/hpc/localtweak_examples/localtweak___FORCE_VALID_PASSWORD_override.tf.txt new file mode 100644 index 00000000..3f9245ba --- /dev/null +++ b/solutions/hpc/localtweak_examples/localtweak___FORCE_VALID_PASSWORD_override.tf.txt @@ -0,0 +1,6 @@ +# This localtweak will skip the enforcement of password policies for app center gui, +# so you can avoid involving complex passwords when in a testing environment. + +locals { + validate_app_center_gui_pwd = true +} diff --git a/solutions/hpc/localtweak_examples/localtweak___PRINT_EXTRA_OUTPUTS_override.tf.txt b/solutions/hpc/localtweak_examples/localtweak___PRINT_EXTRA_OUTPUTS_override.tf.txt new file mode 100644 index 00000000..cb2bbd30 --- /dev/null +++ b/solutions/hpc/localtweak_examples/localtweak___PRINT_EXTRA_OUTPUTS_override.tf.txt @@ -0,0 +1,5 @@ +# This localtweak will enable extra outputs (more debugging). + +locals { + print_extra_outputs = true +} diff --git a/solutions/hpc/localtweak_examples/localtweak___REMOTE_ALLOWED_IPS_EXTRA_override.tf.txt b/solutions/hpc/localtweak_examples/localtweak___REMOTE_ALLOWED_IPS_EXTRA_override.tf.txt new file mode 100644 index 00000000..af110bae --- /dev/null +++ b/solutions/hpc/localtweak_examples/localtweak___REMOTE_ALLOWED_IPS_EXTRA_override.tf.txt @@ -0,0 +1,6 @@ +# This localtweak will let you add IPs or networks to the allowed ips list, so you will be able +# to access the cluster from those IPs. + +locals { + remote_allowed_ips_extra = ["192.0.2.0/24"] +} diff --git a/solutions/hpc/localtweak_examples/localtweak___SSH_IP_ADDRESSES_IN_CONF.tf.txt b/solutions/hpc/localtweak_examples/localtweak___SSH_IP_ADDRESSES_IN_CONF.tf.txt new file mode 100644 index 00000000..16561b5d --- /dev/null +++ b/solutions/hpc/localtweak_examples/localtweak___SSH_IP_ADDRESSES_IN_CONF.tf.txt @@ -0,0 +1,43 @@ +# This localtweak will trigger some scripts (have a look in the extras directory too), +# which can fix the ip addresses in your ssh config files, so you will be able to use +# hostname aliases in ssh commands. + +resource "null_resource" "ssh_conf_fip" { + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + command = "[ -f ~/.ssh/config.d/hpcaas_ip_addr.sh ] && ~/.ssh/config.d/hpcaas_ip_addr.sh ${var.cluster_prefix} fip ${local.bastion_fip}" + } + triggers = { + build = timestamp() + } +} + +resource "null_resource" "ssh_conf_login" { + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + command = "[ -f ~/.ssh/config.d/hpcaas_ip_addr.sh ] && ~/.ssh/config.d/hpcaas_ip_addr.sh ${var.cluster_prefix} login ${local.login_private_ips[0]}" + } + triggers = { + build = timestamp() + } +} + +resource "null_resource" "ssh_conf_mgmt_ip" { + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + command = "[ -f ~/.ssh/config.d/hpcaas_ip_addr.sh ] && ~/.ssh/config.d/hpcaas_ip_addr.sh ${var.cluster_prefix} mgmt ${local.management_private_ip}" + } + triggers = { + build = timestamp() + } +} + +resource "null_resource" "ssh_conf_mgmt_candidate_ips" { + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + command = "[ -f ~/.ssh/config.d/hpcaas_ip_addr.sh ] && ~/.ssh/config.d/hpcaas_ip_addr.sh ${var.cluster_prefix} mgmt_candidate ${join(",", local.management_candidate_private_ips)}" + } + triggers = { + build = timestamp() + } +} diff --git a/solutions/hpc/localtweak_examples/localtweak___SSH_IP_ADDRESSES_IN_CONF_extras/hpcaas.conf b/solutions/hpc/localtweak_examples/localtweak___SSH_IP_ADDRESSES_IN_CONF_extras/hpcaas.conf new file mode 100644 index 00000000..3a663146 --- /dev/null +++ b/solutions/hpc/localtweak_examples/localtweak___SSH_IP_ADDRESSES_IN_CONF_extras/hpcaas.conf @@ -0,0 +1,33 @@ +# This directives assume a cluster prefix of "hpc-cluster-x", you may want to customize that by replacing the target string in the comments. + +### rules for all hosts ### +Host hpcx* + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + +### bastion host ### +Host hpcx + Hostname 150.239.223.53 # hpc-cluster-x fip + User ubuntu + IdentityFile ~/.ssh/keys/myprivatekey + +### login host ### +Host hpcxl + Hostname 10.241.16.5 # hpc-cluster-x iplogin + User lsfadmin + ProxyJump ubuntu@hpcx + +### rules for all management nodes ### +Host hpcx1 hpcx2 hpcx3 + User lsfadmin + ProxyJump ubuntu@hpcx + LocalForward 8443 pac.hpcaas.com:8443 + LocalForward 6080 pac.hpcaas.com:6080 + +### management hosts ### +Host hpcx1 + Hostname 10.241.0.9 # hpc-cluster-x ip1 +Host hpcx2 + Hostname 10.241.0.11 # hpc-cluster-x ip2 +Host hpcx3 + Hostname 10.241.0.10 # hpc-cluster-x ip3 diff --git a/solutions/hpc/localtweak_examples/localtweak___SSH_IP_ADDRESSES_IN_CONF_extras/hpcaas_ip_addr.sh b/solutions/hpc/localtweak_examples/localtweak___SSH_IP_ADDRESSES_IN_CONF_extras/hpcaas_ip_addr.sh new file mode 100755 index 00000000..d5b6e51a --- /dev/null +++ b/solutions/hpc/localtweak_examples/localtweak___SSH_IP_ADDRESSES_IN_CONF_extras/hpcaas_ip_addr.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +prefix="$1" +iptype="$2" +ipvalue="$3" + +if [ "$iptype" == "fip" ]; then + ip="$ipvalue" + sed -i -e "s/ Hostname .* # $prefix fip/ Hostname $ip # $prefix fip/g" ~/.ssh/config.d/hpcaas.conf +elif [ "$iptype" == "login" ]; then + ip="$ipvalue" + sed -i -e "s/ Hostname .* # $prefix iplogin/ Hostname $ip # $prefix iplogin/g" ~/.ssh/config.d/hpcaas.conf +elif [ "$iptype" == "mgmt" ]; then + ip1="$ipvalue" + sed -i -e "s/ Hostname .* # $prefix ip1/ Hostname $ip1 # $prefix ip1/g" ~/.ssh/config.d/hpcaas.conf +elif [ "$iptype" == "mgmt_candidate" ]; then + ips="$ipvalue" + ip2="${ips%,*}" + ip3="${ips#*,}" + sed -i -e "s/ Hostname .* # $prefix ip2/ Hostname $ip2 # $prefix ip2/g" ~/.ssh/config.d/hpcaas.conf + sed -i -e "s/ Hostname .* # $prefix ip3/ Hostname $ip3 # $prefix ip3/g" ~/.ssh/config.d/hpcaas.conf +fi diff --git a/solutions/hpc/main.tf b/solutions/hpc/main.tf index 6a110925..5049985d 100644 --- a/solutions/hpc/main.tf +++ b/solutions/hpc/main.tf @@ -1,158 +1,421 @@ +module "local_exec_script" { + source = "../../modules/null/local_exec_script" + script_path = "./scripts/check_reservation.sh" + script_arguments = "--region ${data.ibm_is_region.region.name} --resource-group-id ${local.resource_groups["workload_rg"]} --output /tmp/hpcaas-check-reservation.log" + script_environment = { + IBM_CLOUD_API_KEY = nonsensitive(var.ibmcloud_api_key) + RESERVATION_ID = nonsensitive(var.reservation_id) + } +} + module "landing_zone" { - source = "../../modules/landing_zone" - # TODO: Add logic - allowed_cidr = var.allowed_cidr - compute_subnets_cidr = var.compute_subnets_cidr + source = "../../modules/landing_zone" + compute_subnets_cidr = var.vpc_cluster_private_subnets_cidr_blocks cos_instance_name = var.cos_instance_name - enable_atracker = var.enable_atracker + enable_atracker = var.observability_atracker_on_cos_enable enable_cos_integration = var.enable_cos_integration enable_vpc_flow_logs = var.enable_vpc_flow_logs - enable_vpn = var.enable_vpn - hpcs_instance_name = var.hpcs_instance_name - ibmcloud_api_key = var.ibmcloud_api_key + enable_vpn = var.vpn_enabled key_management = var.key_management + kms_instance_name = var.kms_instance_name + kms_key_name = var.kms_key_name ssh_keys = var.bastion_ssh_keys - bastion_subnets_cidr = var.bastion_subnets_cidr - management_instances = var.management_instances - compute_instances = var.static_compute_instances - network_cidr = var.network_cidr - placement_strategy = var.placement_strategy - prefix = var.prefix - protocol_instances = var.protocol_instances - protocol_subnets_cidr = var.protocol_subnets_cidr + bastion_subnets_cidr = var.vpc_cluster_login_private_subnets_cidr_blocks + network_cidr = var.vpc_cidr + prefix = var.cluster_prefix resource_group = var.resource_group - storage_instances = var.storage_instances - storage_subnets_cidr = var.storage_subnets_cidr - vpc = var.vpc - vpn_peer_address = var.vpn_peer_address - vpn_peer_cidr = var.vpn_peer_cidr - vpn_preshared_key = var.vpn_preshared_key + vpc = var.vpc_name + subnet_id = var.cluster_subnet_ids + login_subnet_id = var.login_subnet_id zones = var.zones + no_addr_prefix = local.no_addr_prefix + scc_enable = var.scc_enable } module "bootstrap" { - source = "./../../modules/bootstrap" - ibmcloud_api_key = var.ibmcloud_api_key - resource_group = var.resource_group - prefix = var.prefix - zones = var.zones - vpc_id = local.vpc_id - network_cidr = var.network_cidr - enable_bastion = var.enable_bastion - bastion_subnets = local.bastion_subnets - enable_bootstrap = var.enable_bootstrap - bootstrap_instance_profile = var.bootstrap_instance_profile - ssh_keys = var.bastion_ssh_keys - allowed_cidr = var.allowed_cidr - kms_encryption_enabled = local.kms_encryption_enabled - boot_volume_encryption_key = local.boot_volume_encryption_key - existing_kms_instance_guid = local.existing_kms_instance_guid + source = "./../../modules/bootstrap" + resource_group = local.resource_groups["workload_rg"] + prefix = var.cluster_prefix + vpc_id = local.vpc_id + network_cidr = var.vpc_name != null && length(var.cluster_subnet_ids) > 0 ? local.existing_subnet_cidrs : split(",", var.vpc_cidr) + bastion_subnets = local.bastion_subnets + ssh_keys = var.bastion_ssh_keys + allowed_cidr = local.allowed_cidr + kms_encryption_enabled = local.kms_encryption_enabled + boot_volume_encryption_key = local.boot_volume_encryption_key + existing_kms_instance_guid = local.existing_kms_instance_guid + skip_iam_authorization_policy = var.skip_iam_authorization_policy + bastion_instance_name = var.bastion_instance_name + bastion_instance_public_ip = local.bastion_instance_public_ip + bastion_security_group_id = var.bastion_instance_name != null ? var.bastion_security_group_id : null + ldap_server = var.ldap_server +} + +module "generate_db_adminpassword" { + count = var.enable_app_center && var.app_center_high_availability ? 1 : 0 + source = "../../modules/security/password" + length = 15 + special = true + override_special = "-_" + min_numeric = 1 +} + +module "db" { + count = var.enable_app_center && var.app_center_high_availability ? 1 : 0 + source = "../../modules/database/mysql" + resource_group_id = local.resource_groups["service_rg"] + name = "${var.cluster_prefix}-database" + region = data.ibm_is_region.region.name + mysql_version = local.mysql_version + service_endpoints = local.db_service_endpoints + adminpassword = "db-${module.generate_db_adminpassword[0].password}" # with a prefix so we start with a letter + members = local.db_template[0] + memory = local.db_template[1] + disks = local.db_template[2] + vcpu = local.db_template[3] } module "landing_zone_vsi" { - source = "../../modules/landing_zone_vsi" - ibmcloud_api_key = var.ibmcloud_api_key - resource_group = var.resource_group - prefix = var.prefix - zones = var.zones - vpc_id = local.vpc_id - bastion_security_group_id = local.bastion_security_group_id - bastion_public_key_content = local.bastion_public_key_content - login_subnets = local.login_subnets - login_ssh_keys = var.login_ssh_keys - login_image_name = var.login_image_name - login_instances = var.login_instances - compute_subnets = local.compute_subnets - compute_ssh_keys = var.compute_ssh_keys - management_image_name = var.management_image_name - management_instances = var.management_instances - static_compute_instances = var.static_compute_instances - dynamic_compute_instances = var.dynamic_compute_instances - compute_image_name = var.compute_image_name - storage_subnets = local.storage_subnets - storage_ssh_keys = var.storage_ssh_keys - storage_instances = var.storage_instances - storage_image_name = var.storage_image_name - protocol_subnets = local.protocol_subnets - protocol_instances = var.protocol_instances - nsd_details = var.nsd_details - dns_domain_names = var.dns_domain_names - kms_encryption_enabled = local.kms_encryption_enabled - boot_volume_encryption_key = local.boot_volume_encryption_key + source = "../../modules/landing_zone_vsi" + resource_group = local.resource_groups["workload_rg"] + ibmcloud_api_key = var.ibmcloud_api_key + prefix = var.cluster_prefix + zones = var.zones + vpc_id = local.vpc_id + bastion_fip = local.bastion_fip + bastion_security_group_id = local.bastion_security_group_id + bastion_public_key_content = local.bastion_public_key_content + cluster_user = local.cluster_user + compute_private_key_content = local.compute_private_key_content + bastion_private_key_content = local.bastion_ssh_private_key != null ? local.bastion_ssh_private_key : local.bastion_private_key_content + compute_subnets = local.compute_subnets + compute_ssh_keys = var.compute_ssh_keys + management_image_name = var.management_image_name + compute_image_name = var.compute_image_name + login_image_name = var.login_image_name + dns_domain_names = var.dns_domain_name + kms_encryption_enabled = local.kms_encryption_enabled + boot_volume_encryption_key = local.boot_volume_encryption_key + share_path = local.share_path + hyperthreading_enabled = var.hyperthreading_enabled + app_center_gui_pwd = var.app_center_gui_pwd + enable_app_center = var.enable_app_center + contract_id = var.reservation_id + cluster_id = local.cluster_id + management_node_count = var.management_node_count + management_node_instance_type = var.management_node_instance_type + file_share = module.file_storage.mount_paths_excluding_first + mount_path = var.custom_file_shares + login_node_instance_type = var.login_node_instance_type + bastion_subnets = local.bastion_subnets + ssh_keys = var.bastion_ssh_keys + enable_ldap = var.enable_ldap + ldap_basedns = var.ldap_basedns + login_private_ips = join("", local.login_private_ips) + ldap_vsi_profile = var.ldap_vsi_profile + ldap_admin_password = var.ldap_admin_password + ldap_user_name = var.ldap_user_name + ldap_user_password = var.ldap_user_password + ldap_server = var.ldap_server + ldap_vsi_osimage_name = var.ldap_vsi_osimage_name + ldap_primary_ip = local.ldap_private_ips + app_center_high_availability = var.app_center_high_availability + db_instance_info = var.enable_app_center && var.app_center_high_availability ? module.db[0].db_instance_info : null + storage_security_group_id = var.storage_security_group_id + observability_monitoring_enable = var.observability_monitoring_enable + observability_monitoring_on_compute_nodes_enable = var.observability_monitoring_on_compute_nodes_enable + cloud_monitoring_access_key = var.observability_monitoring_enable ? module.cloud_monitoring_instance_creation.cloud_monitoring_access_key : "" + cloud_monitoring_ingestion_url = var.observability_monitoring_enable ? module.cloud_monitoring_instance_creation.cloud_monitoring_ingestion_url : "" + cloud_monitoring_prws_key = var.observability_monitoring_enable ? module.cloud_monitoring_instance_creation.cloud_monitoring_prws_key : "" + cloud_monitoring_prws_url = var.observability_monitoring_enable ? module.cloud_monitoring_instance_creation.cloud_monitoring_prws_url : "" + bastion_instance_name = var.bastion_instance_name + depends_on = [ + module.local_exec_script, + module.validate_ldap_server_connection + ] } module "file_storage" { - source = "../../modules/file_storage" - ibmcloud_api_key = var.ibmcloud_api_key - zone = var.zones[0] # always the first zone - file_shares = local.file_shares - encryption_key_crn = local.boot_volume_encryption_key - security_group_ids = local.compute_security_group_id - subnet_id = local.compute_subnet_id + source = "../../modules/file_storage" + zone = var.zones[0] # always the first zone + resource_group = local.resource_groups["workload_rg"] + file_shares = local.file_shares + encryption_key_crn = local.boot_volume_encryption_key + security_group_ids = local.compute_security_group_id + subnet_id = local.compute_subnet_id + prefix = var.cluster_prefix + existing_kms_instance_guid = local.existing_kms_instance_guid + skip_iam_share_authorization_policy = var.skip_iam_share_authorization_policy + kms_encryption_enabled = local.kms_encryption_enabled } module "dns" { source = "./../../modules/dns" - ibmcloud_api_key = var.ibmcloud_api_key - prefix = var.prefix - resource_group_id = local.resource_group_id + prefix = var.cluster_prefix + resource_group_id = local.resource_groups["service_rg"] vpc_crn = local.vpc_crn - subnets_crn = local.subnets_crn + subnets_crn = local.compute_subnets_crn dns_instance_id = var.dns_instance_id dns_custom_resolver_id = var.dns_custom_resolver_id - dns_domain_names = values(var.dns_domain_names) + dns_domain_names = values(var.dns_domain_name) } +module "alb" { + source = "./../../modules/alb" + bastion_subnets = local.bastion_subnets + resource_group_id = local.resource_groups["workload_rg"] + prefix = var.cluster_prefix + security_group_ids = concat(local.compute_security_group_id, [local.bastion_security_group_id]) + vsi_ids = local.vsi_management_ids + certificate_instance = var.enable_app_center && var.app_center_high_availability ? var.existing_certificate_instance : "" + create_load_balancer = !local.alb_created_by_api && var.app_center_high_availability && var.enable_app_center +} + +module "alb_api" { + source = "./../../modules/alb_api" + ibmcloud_api_key = var.ibmcloud_api_key + region = data.ibm_is_region.region.name + bastion_subnets = local.bastion_subnets + resource_group_id = local.resource_groups["workload_rg"] + prefix = var.cluster_prefix + security_group_ids = concat(local.compute_security_group_id, [local.bastion_security_group_id]) + vsi_ips = concat([local.management_private_ip], local.management_candidate_private_ips) + certificate_instance = var.enable_app_center && var.app_center_high_availability ? var.existing_certificate_instance : "" + create_load_balancer = local.alb_created_by_api && var.app_center_high_availability && var.enable_app_center +} + +################################################### +# DNS Modules to create DNS domains and records +################################################## module "compute_dns_records" { source = "./../../modules/dns_record" - ibmcloud_api_key = var.ibmcloud_api_key dns_instance_id = local.dns_instance_id dns_zone_id = local.compute_dns_zone_id dns_records = local.compute_dns_records + dns_domain_names = var.dns_domain_name } -module "storage_dns_records" { +module "compute_candidate_dns_records" { source = "./../../modules/dns_record" - ibmcloud_api_key = var.ibmcloud_api_key dns_instance_id = local.dns_instance_id - dns_zone_id = local.storage_dns_zone_id - dns_records = local.storage_dns_records + dns_zone_id = local.compute_dns_zone_id + dns_records = local.mgmt_candidate_dns_records + dns_domain_names = var.dns_domain_name } -module "protocol_dns_records" { +module "login_vsi_dns_records" { source = "./../../modules/dns_record" - ibmcloud_api_key = var.ibmcloud_api_key dns_instance_id = local.dns_instance_id - dns_zone_id = local.protocol_dns_zone_id - dns_records = local.protocol_dns_records + dns_zone_id = local.compute_dns_zone_id + dns_records = local.login_vsi_dns_records + dns_domain_names = var.dns_domain_name +} + +module "ldap_vsi_dns_records" { + source = "./../../modules/dns_record" + dns_instance_id = local.dns_instance_id + dns_zone_id = local.compute_dns_zone_id + dns_records = local.ldap_vsi_dns_records + dns_domain_names = var.dns_domain_name +} + +# DNS entry needed to ALB, can be moved in dns_record module for example +resource "ibm_dns_resource_record" "pac_cname" { + count = var.enable_app_center && var.app_center_high_availability ? 1 : 0 + instance_id = local.dns_instance_id + zone_id = local.compute_dns_zone_id + type = "CNAME" + name = "pac" + ttl = 300 + rdata = local.alb_hostname } module "compute_inventory" { source = "./../../modules/inventory" hosts = local.compute_hosts + user = local.cluster_user + server_name = "[HPCAASCluster]" inventory_path = local.compute_inventory_path } -module "storage_inventory" { +################################################### +# Creation of inventory files for the automation usage +################################################## +module "bastion_inventory" { + source = "./../../modules/inventory" + hosts = local.bastion_host + user = local.login_user + server_name = "[BastionServer]" + inventory_path = local.bastion_inventory_path +} + +module "login_inventory" { + source = "./../../modules/inventory" + hosts = local.login_host + user = local.cluster_user + server_name = "[LoginServer]" + inventory_path = local.login_inventory_path +} + +module "ldap_inventory" { source = "./../../modules/inventory" - hosts = local.storage_hosts - inventory_path = local.storage_inventory_path -} - -module "compute_playbook" { - source = "./../../modules/playbook" - bastion_fip = local.bastion_fip - private_key_path = local.compute_private_key_path - inventory_path = local.compute_inventory_path - playbook_path = local.compute_playbook_path - depends_on = [module.compute_inventory] -} - -module "storage_playbook" { - source = "./../../modules/playbook" - bastion_fip = local.bastion_fip - private_key_path = local.storage_private_key_path - inventory_path = local.storage_inventory_path - playbook_path = local.storage_playbook_path - depends_on = [module.storage_inventory] + hosts = local.ldap_host + user = local.ldap_user + server_name = "[LDAPServer]" + inventory_path = local.ldap_inventory_path +} + +################################################### +# REMOTE_EXEC : Remote exec block to perform certain checks on the cluster nodes +################################################## +module "check_cluster_status" { + source = "./../../modules/null/remote_exec" + cluster_host = [local.management_private_ip] #["10.10.10.4"] + cluster_user = local.cluster_user #"root" + cluster_private_key = local.compute_private_key_content + login_host = local.bastion_fip + login_user = "ubuntu" + login_private_key = local.bastion_ssh_private_key != null ? local.bastion_ssh_private_key : local.bastion_private_key_content + command = ["lshosts -w; lsid || (sleep 5; lsid) || (sleep 15; lsid)"] # we give it more time if not ready + depends_on = [ + module.landing_zone_vsi, # this implies vsi have been configured too + module.bootstrap + ] +} + +module "check_node_status" { + source = "./../../modules/null/remote_exec" + cluster_host = concat(local.management_candidate_private_ips, [local.management_private_ip]) + cluster_user = local.cluster_user + cluster_private_key = local.compute_private_key_content + login_host = local.bastion_fip + login_user = "ubuntu" + login_private_key = local.bastion_ssh_private_key != null ? local.bastion_ssh_private_key : local.bastion_private_key_content + command = ["systemctl --no-pager -n 5 status lsfd"] + depends_on = [ + module.landing_zone_vsi, + module.bootstrap, + module.check_cluster_status + ] +} + +module "validate_ldap_server_connection" { + source = "./../../modules/null/ldap_remote_exec" + ldap_server = var.ldap_server + enable_ldap = var.enable_ldap + login_private_key = local.bastion_ssh_private_key != null ? local.bastion_ssh_private_key : local.bastion_private_key_content + login_host = local.bastion_fip + login_user = "ubuntu" + depends_on = [module.bootstrap] +} + +# Module used to destroy the non-essential resources +module "login_fip_removal" { + source = "./../../modules/null/local_exec" + count = var.enable_fip ? 0 : 1 + region = data.ibm_is_region.region.name + ibmcloud_api_key = var.ibmcloud_api_key + trigger_resource_id = local.bastion_fip_id + command = "ibmcloud is ipd ${local.bastion_fip_id} -f" + depends_on = [module.check_cluster_status] +} + +resource "null_resource" "destroy_compute_resources" { + triggers = { + conn_user = local.cluster_user + conn_host = local.management_private_ip + conn_private_key = local.compute_private_key_content + conn_bastion_host = local.bastion_fip + conn_bastion_private_key = local.bastion_ssh_private_key != null ? local.bastion_ssh_private_key : local.bastion_private_key_content + } + + # only works if fip is enabled & vpn is disabled (conn is must) + count = false && var.enable_fip == true && var.vpn_enabled == false ? 1 : 0 + + connection { + type = "ssh" + host = self.triggers.conn_host + user = self.triggers.conn_user + private_key = self.triggers.conn_private_key + bastion_host = self.triggers.conn_bastion_host + bastion_user = "ubuntu" + bastion_private_key = self.triggers.conn_bastion_private_key + timeout = "60m" + } + + provisioner "remote-exec" { + when = destroy + on_failure = fail + inline = [file("${path.module}/scripts/destroy_script.sh")] + } +} + +######################################################################################################### +# validation_script_executor Module +# +# Purpose: This module is included for testing purposes. +# It provides a conditional mechanism for executing remote scripts on cluster hosts. +# The execution is triggered if the script filenames listed in TF_VALIDATION_SCRIPT_FILES are provided. +# +# Usage: +# - When scripts are listed in TF_VALIDATION_SCRIPT_FILES, the corresponding scripts +# will be executed on the cluster hosts using remote command execution. +# - The conditional nature ensures that scripts are executed only when necessary. +# This can be useful for various validation or maintenance tasks. +######################################################################################################### + +module "validation_script_executor" { + source = "./../../modules/null/remote_exec" + count = var.TF_VALIDATION_SCRIPT_FILES != null && length(var.TF_VALIDATION_SCRIPT_FILES) > 0 ? 1 : 0 + + cluster_host = [local.management_private_ip] + cluster_user = local.cluster_user + cluster_private_key = local.compute_private_key_content + login_host = local.bastion_fip + login_user = "ubuntu" + login_private_key = local.bastion_ssh_private_key != null ? local.bastion_ssh_private_key : local.bastion_private_key_content + + command = [ + for script_name in var.TF_VALIDATION_SCRIPT_FILES : + file("${path.module}/examples/scripts/${script_name}") + ] + depends_on = [ + module.landing_zone_vsi, + module.bootstrap, + module.check_cluster_status + ] +} + +################################################### +# Observability Modules +################################################### + +module "cloud_monitoring_instance_creation" { + source = "../../modules/observability_instance" + location = local.region + ibmcloud_api_key = var.ibmcloud_api_key + rg = local.resource_groups["service_rg"] + cloud_monitoring_provision = var.observability_monitoring_enable + observability_monitoring_plan = var.observability_monitoring_plan + cloud_monitoring_instance_name = "${var.cluster_prefix}-metrics" + tags = ["hpc", var.cluster_prefix] +} + +# Code for SCC Instance +module "scc_instance_and_profile" { + count = var.scc_enable ? 1 : 0 + source = "./../../modules/security/scc" + location = var.scc_location != "" ? var.scc_location : "us-south" + rg = local.resource_groups["service_rg"] + scc_profile = var.scc_enable ? var.scc_profile : "" + scc_profile_version = var.scc_profile != "" && var.scc_profile != null ? var.scc_profile_version : "" + event_notification_plan = var.scc_event_notification_plan + tags = ["hpc", var.cluster_prefix] + prefix = var.cluster_prefix + cos_bucket = [for name in module.landing_zone.cos_buckets_names : name if strcontains(name, "scc-bucket")][0] + cos_instance_crn = module.landing_zone.cos_instance_crns[0] +} + +module "my_ip" { + source = "../../modules/my_ip" } diff --git a/solutions/hpc/outputs.tf b/solutions/hpc/outputs.tf index 605fd4c2..2292f831 100644 --- a/solutions/hpc/outputs.tf +++ b/solutions/hpc/outputs.tf @@ -1,18 +1,89 @@ -/* -output "landing-zone" { - value = module.landing-zone +output "region_name" { + description = "The region name in which the cluster resources have been deployed" + value = data.ibm_is_region.region.name } -output "bootstrap" { - value = module.bootstrap - sensitive = true +output "image_entry_found" { + description = "Available if the image name provided is located within the image map" + value = module.landing_zone_vsi.image_map_entry_found } -output "landing-zone-vsi" { - value = module.landing-zone-vsi +output "vpc_name" { + description = "The VPC name in which the cluster resources have been deployed" + value = "${data.ibm_is_vpc.vpc.name} -- - ${data.ibm_is_vpc.vpc.id}" } -*/ -output "ssh_command" { + +output "ssh_to_management_node_1" { description = "SSH command to connect to HPC cluster" - value = "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J ubuntu@${module.bootstrap.bastion_fip} vpcuser@${local.compute_hosts[0]}" + value = local.bastion_instance_public_ip != null ? "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J ubuntu@${local.bastion_instance_public_ip} lsfadmin@${local.compute_hosts[0]}" : var.enable_fip ? "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J ubuntu@${module.bootstrap.bastion_fip[0]} lsfadmin@${local.compute_hosts[0]}" : "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J ubuntu@${module.bootstrap.bastion_primary_ip} lsfadmin@${local.compute_hosts[0]}" +} + +output "ssh_to_ldap_node" { + description = "SSH command to connect to LDAP node" + value = var.enable_ldap && var.ldap_server == "null" ? (var.enable_fip ? "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -J ubuntu@${module.bootstrap.bastion_fip[0]} ubuntu@${module.landing_zone_vsi.ldap_server}" : "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J ubuntu@${module.bootstrap.bastion_primary_ip} ubuntu@${module.landing_zone_vsi.ldap_server}") : null +} + +output "ssh_to_login_node" { + description = "SSH command to connect to Login node" + value = local.bastion_instance_public_ip != null ? "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J ubuntu@${local.bastion_instance_public_ip} lsfadmin@${join(",", local.login_private_ips)}" : var.enable_fip ? "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J ubuntu@${module.bootstrap.bastion_fip[0]} lsfadmin@${join(",", local.login_private_ips)}" : "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J ubuntu@${module.bootstrap.bastion_primary_ip} lsfadmin@${join(",", local.login_private_ips)}" +} + +output "application_center_tunnel" { + description = "Available if IBM Spectrum LSF Application Center GUI is installed" + value = var.enable_app_center ? local.ssh_cmd : null +} + +output "application_center_url" { + description = "Available if IBM Spectrum LSF Application Center GUI is installed" + value = var.enable_app_center ? var.app_center_high_availability ? "https://pac.${var.dns_domain_name.compute}:8443" : "https://localhost:8443" : null +} + +output "application_center_url_note" { + description = "Available if IBM Spectrum LSF Application Center GUI is installed in High Availability" + value = var.enable_app_center && var.app_center_high_availability ? "you may need '127.0.0.1 pac pac.${var.dns_domain_name.compute}' in your /etc/hosts, to let your browser use the ssh tunnel" : null +} + +output "remote_allowed_cidr" { + description = "The following IPs/networks are allow-listed for incoming connections" + value = local.allowed_cidr +} + +output "management_hostname" { + description = "Management node has this hostname:" + value = local.print_extra_outputs ? local.management_hostname : null +} + +output "management_ip" { + description = "Management node has this IP:" + value = local.print_extra_outputs ? local.management_private_ip : null +} + +output "management_candidate_hostnames" { + description = "Management candidate nodes have these hostnames:" + value = local.print_extra_outputs ? local.management_candidate_hostnames : null +} + +output "management_candidate_ips" { + description = "Management candidate nodes have these IPs:" + value = local.print_extra_outputs ? local.management_candidate_private_ips : null +} + +output "login_hostnames" { + description = "Login nodes have these hostnames:" + value = local.print_extra_outputs ? local.login_hostnames : null +} + +output "login_ips" { + description = "Login nodes have these IPs:" + value = local.print_extra_outputs ? local.login_private_ips : null +} + +output "ldap_hostnames" { + description = "LDAP nodes have these hostnames:" + value = local.print_extra_outputs ? local.ldap_hostnames : null +} + +output "ldap_ips" { + description = "LDAP nodes have these IPs:" + value = local.print_extra_outputs ? local.ldap_private_ips : null } diff --git a/solutions/hpc/provider.tf b/solutions/hpc/provider.tf new file mode 100644 index 00000000..73d31835 --- /dev/null +++ b/solutions/hpc/provider.tf @@ -0,0 +1,4 @@ +provider "ibm" { + ibmcloud_api_key = var.ibmcloud_api_key + region = local.region +} diff --git a/solutions/hpc/scripts/check_reservation.sh b/solutions/hpc/scripts/check_reservation.sh new file mode 100755 index 00000000..1a098f8c --- /dev/null +++ b/solutions/hpc/scripts/check_reservation.sh @@ -0,0 +1,546 @@ +#!/bin/bash +# +# Licensed Materials - Property of IBM +# 5725-S00 (C) Copyright IBM Corp. 2024. All Rights Reserved. +# US Government Users Restricted Rights - Use, duplication or +# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. +SCRIPT_FOLDER=$(realpath "$(dirname "$0")") +PROJECT_FOLDER=$(realpath "$SCRIPT_FOLDER/..") + +IAM_ENDPOINT_URL="https://iam.cloud.ibm.com/identity/token" +RESOURCE_CONTROLLER_ENDPOINT_URL="https://resource-controller.cloud.ibm.com" +CODE_ENGINE_API_ENDPOINT_URL="https://api.REGION.codeengine.cloud.ibm.com" +V2_CONTEXT_ROOT="v2" +V2BETA_CONTEXT_ROOT="v2beta" +RESOURCE_PLAN_GUID="2e390ff1-fe87-458f-9a23-dfb6719509e1" + +TMP_DIR="/tmp" +HTTP_OUTPUT_FILE="${TMP_DIR}/hpcaas_http_output.log" +CODE_ENGINE_PROJECT_GUID_FILE="${PROJECT_FOLDER}/assets/hpcaas-ce-project-guid.cfg" + +REGION="" +RESOURCE_GROUP_ID="" + +LOG_FILE="/tmp/hpcaas-check-reservation.log" + +# Script return code: +# 0 - Success, a Reservation for the input RESERVATION_ID exists and a Code Engine Project exists for it. +# 1 - IBM_CLOUD_API_KEY and/or RESERVATION_ID environment variables are not provided. +# 2 - Parsing error, the script was not invoked correctly. +# 3 - Cannot retrieve JWT token, the script cannot exchange the IBM Cloud API key with a JWT token. +# 4 - Cannot retrieve a GUID for the input Reservation ID. +# 5 - Reservation doesn't exist, a Reservation for the input RESERVATION_ID doesn't. +# 6 - Cannot create the Code Engine project. +# 7 - Code Engine project creation timeout expired. +# 8 - Cannot associate the Code Engine project with guid GUID to the Reservation with id RESERVATION_ID. + +#################################################################################################### +# init_logging +# +# Description: +# this function initialize the hpcaas-check-reservation.log file +#################################################################################################### +init_logging() { + # Calculate the folder of the log file + LOG_FILE_FOLDER=$(dirname "${LOG_FILE}") + # Verify the folder exists, if not create it + if [ ! -d "${LOG_FILE_FOLDER}" ]; then + # Se non esiste, crea la directory + mkdir -p "${LOG_FILE_FOLDER}" + fi + # Remove everything from the log file + echo "" > "${LOG_FILE}" +} + +#################################################################################################### +# log +# +# Description: +# this function print the input message on a log file. +# Input: +# message, the message to print +# Output: +# message, the message with variable and timestamp rendered. +#################################################################################################### +log() { + local message=$1 + + # Create the timestamp to add in the log message + timestamp=$(date +'%Y%m%d %H:%M:%S') + # Print the message on the output and in the log file + echo "[$timestamp] ${message}" + echo "[$timestamp] ${message}" >> "${LOG_FILE}" +} + +#################################################################################################### +# usage +# +# Description: +# this function prints the usage and exit with an error +#################################################################################################### +usage() { + log "Usage: $0 [options]" + log "Options:" + log " --region id | -e id : Specify the Region" + log " --resource-group-id id | -e id : Specify the Resource Group ID" + log " [--output ] | -o [--output ] : Specify the log file. Default is stdout." + exit 2 +} + +#################################################################################################### +# parse_args +# +# Description: +# this function parse the input parameters. The following parameters are supported: +# --region id | -e id +# --resource-group-id id | -e id : Specify the Resource Group ID +# [--output ] | -o [--output ] : Specify the log file. Default is stdout +# Input: +# input parameters, the input parameters to parse +# Output: +# usage, the usage is printed if an error occured +#################################################################################################### +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --region|-e) + shift + REGION="$1";; + --resource-group-id|-s) + shift + RESOURCE_GROUP_ID="$1";; + --output|-o) + shift + LOG_FILE="$1";; + *) + log "ERROR: parsing of the input arguments failed." + log "ERROR Details: invalid option $1." + usage;; + esac + shift + done + # Verify if the required options have been provided + if [[ -z "${REGION}" || -z "${RESOURCE_GROUP_ID}" ]]; then + log "ERROR: parsing of the input arguments failed." + log "ERROR Details: the options --region, and --resource-group-id are required." + usage + fi + + # Array contenente i valori consentiti per la regione + local allowed_regions=("us-east" "us-south" "eu-de") + + # Verifica se la regione specificata è tra i valori consentiti + # shellcheck disable=SC2199,SC2076 + if [[ ! " ${allowed_regions[@]} " =~ " ${REGION} " ]]; then + log "ERROR: parsing of the input arguments failed." + log "ERROR Details: Invalid region specified. Region must be one of: ${allowed_regions[*]}." + usage + fi +} + +#################################################################################################### +# get_token +# +# Description: +# this function validates the IBM Cloud API Key and return a JWT Baerer token to use for authentication +# when Code Engine API are invoked. This function takes in input the IBM Cloud API Key and return the +# JWT token. +# Input: +# api_key, the IBM Cloud API key that identify the IBM Cloud User account. +# Output: +# token, the JWT token if the function is successful +# http status, in case of failure the HTTP status is printed +# error message, in case of failure the error message is printed +# Return: +# 0, success +# 1, failure +#################################################################################################### +get_token() { + local api_key="$1" + local response + local http_status + local json_response + local token + local error_message + + # The IBM tool https://github.com/ibm/detect-secrets detected the secret we passed to the API. + # However, this is aa public secret so no real exposure exists. + + # This is the curl used to retrieve the JWT token given the IBM Cloud API Key in input + response=$(curl -s -w "%{http_code}" --request POST --url ${IAM_ENDPOINT_URL} --header 'Authorization: Basic Yng6Yng=' --header 'Content-Type: application/x-www-form-urlencoded' --data grant_type=urn:ibm:params:oauth:grant-type:apikey --data apikey="${api_key}") # pragma: allowlist secret + + # The curl return a reply with the following format { ... JSON ... }HTTPSTATUS. + # These two lines separate the HTTP STATUS from the JSON reply. + http_status="${response: -3}" + json_response=${response%???} + + # If HTTP Status = 200 the JWT token is printed and 0 is returned, otherwise + # 1 is printed (meaning error) and HTTP STATUS and error messages are printed. + # The reason for this is that if something goes wrong, the caller can print the HTTP STATUS + # code and the error messages so that the customer can understand the problem. + if [ "$http_status" -eq 200 ]; then + token=$(echo "$json_response" | jq -r '.access_token') + echo "$token" + return 0 + else + error_message=$(echo "$json_response" | jq -r '.errorMessage') + echo "$http_status" + echo "$error_message" + return 1 + fi +} + +#################################################################################################### +# get_guid_from_reservation_id +# +# Description: +# this function check if a Code Engine Project exists for the input reservation_id. If so, +# the function return with success, otherwise an error is returned. +# Input: +# jwt_token, the jwt token +# reservation_id, the reservation id to check +# Output: +# http_code, the HTTP code returned by Code Engine +# message, the HTTP message returned by Code Engine +# +# Return: +# 200 if everything is OK, otherwise an error code with relative message +#################################################################################################### +get_guid_from_reservation_id() { + local jwt_token="$1" + local result + local http_status + local response_message + + # This curl check if the input reservation id exists + result=$(curl -s -w "%{http_code}" -o ${HTTP_OUTPUT_FILE} \ + -H "Authorization: Bearer ${jwt_token}" \ + "${CODE_ENGINE_API_ENDPOINT_URL}/${V2BETA_CONTEXT_ROOT}/capacity_reservations") + + # The curl return a reply with the following format { ... JSON ... }HTTPSTATUS. + # These two lines separate the HTTP STATUS from the JSON reply. + http_status="${result: -3}" + response_message=$(cat "${HTTP_OUTPUT_FILE}") + + # Show both the HTTP code and the response message + echo "${http_status}" + echo "${response_message}" +} + +#################################################################################################### +# check_reservation +# +# Description: +# this function check if a Code Engine Project exists for the input reservation_id. If so, +# the function return with success, otherwise an error is returned. +# Input: +# jwt_token, the jwt token +# reservation_guid, the reservation guid to check +# Output: +# http_code, the HTTP code returned by Code Engine +# message, the HTTP message returned by Code Engine +# +# Return: +# 200 if everything is OK, otherwise an error code with relative message +#################################################################################################### +check_reservation() { + local jwt_token="$1" + local reservation_guid="$2" + local result + local http_status + local response_message + + # This curl check if the input reservation id exists + result=$(curl -s -w "%{http_code}" -o ${HTTP_OUTPUT_FILE} \ + -H "Authorization: Bearer ${jwt_token}" \ + "${CODE_ENGINE_API_ENDPOINT_URL}/${V2BETA_CONTEXT_ROOT}/capacity_reservations/${reservation_guid}") + + # The curl return a reply with the following format { ... JSON ... }HTTPSTATUS. + # These two lines separate the HTTP STATUS from the JSON reply. + http_status="${result: -3}" + response_message=$(cat "${HTTP_OUTPUT_FILE}") + + # Show both the HTTP code and the response message + echo "${http_status}" + echo "${response_message}" +} + +#################################################################################################### +# create_ce_project +# +# Description: +# this function creates a Code Engine Project. +# Input: +# jwt_token, the jwt token +# region, the region +# resource_group_id, the resource group id +# Output: +# http_code, the HTTP code returned by Code Engine +# message, the HTTP message returned by Code Engine +# Return: +# 201 or 202 if everything is OK, otherwise an error code with relative message +#################################################################################################### +create_ce_project() { + local jwt_token="$1" + local region="$2" + local resource_group_id="$3" + local timestamp + local project_name + local parameters + local allow_cleanup + local result + local http_code + local response_message + + timestamp=$(date "+%Y%m%d%H%M%S") + project_name="HPC-Default-${timestamp}" + parameters='{"name":"'"${project_name}"'","profile":"hpc"}' + allow_cleanup=false + + # This curl create an empty Code Engine project via Resource Controller + result=$(curl -s -w "%{http_code}" -o ${HTTP_OUTPUT_FILE} \ + -X POST \ + -H "Authorization: Bearer ${jwt_token}" \ + -H "Content-Type: application/json" \ + -d "{\"name\":\"${project_name}\",\"resource_plan_id\":\"${RESOURCE_PLAN_GUID}\",\"resource_group\":\"${resource_group_id}\",\"parameters\":${parameters},\"target\":\"${region}\",\"allow_cleanup\":${allow_cleanup}}" \ + "${RESOURCE_CONTROLLER_ENDPOINT_URL}/${V2_CONTEXT_ROOT}/resource_instances") + + # The curl return a reply with the following format { ... JSON ... }HTTPSTATUS. + # These two lines separate the HTTP STATUS from the JSON reply. + http_code="${result: -3}" + response_message=$(cat "${HTTP_OUTPUT_FILE}") + + # Show both the HTTP code and the response message + echo "$http_code" + echo "$response_message" +} + +#################################################################################################### +# wait_ce_project_creation +# +# Description: +# this function waits the Code Engine Project was successfully created. +# Input: +# guid, the Code Engine project guid +# Return: +# 0, successful +# 1, timeout expired +#################################################################################################### +wait_ce_project_creation() { + local jwt_token="$1" + local region="$2" + local ce_project_guid="$3" + # 3 minutes and 20s timeout + local timeout=300 + local start_time + local http_code + local response_message + local status + local current_time + local elapsed_time + local result + + start_time=$(date +%s) + # Loop until the Code Engine project is ready or the timeout expired + while true; do + # Check if the Code Engine project is ready + result=$(curl -s -w "%{http_code}" -o ${HTTP_OUTPUT_FILE} \ + -H "Authorization: Bearer ${jwt_token}" \ + "${CODE_ENGINE_API_ENDPOINT_URL}/${V2_CONTEXT_ROOT}/projects/${ce_project_guid}") + + # The curl return a reply with the following format { ... JSON ... }HTTPSTATUS. + # These two lines separate the HTTP STATUS from the JSON reply. + http_code="${result: -3}" + response_message=$(cat "${HTTP_OUTPUT_FILE}") + + # If the Code Engine project is ready, return + if [ "$http_code" -eq 200 ]; then + status=$(jq -r '.status' "${HTTP_OUTPUT_FILE}") + + # If status is not active exit from this cycle, Code Engine API returns this status when the project is ready + if [ "$status" == "active" ]; then + return 0 + fi + fi + + # Check if the timeout expired + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ "$elapsed_time" -ge "$timeout" ]; then + break + fi + + # Wait 10 seconds before retry the check. + sleep 10 + done + + # The Code Engine project wasn't successfully created, the timeout expired, so return error + return 1 +} + +#################################################################################################### +# associate_ce_project_to_reservation +# +# Description: +# this function associates the Code Engine project to the HPC Reservation +# Input: +# guid, the Code Engine project guid +# Return: +# 200 if everything is OK, otherwise an error code with relative message +#################################################################################################### +associate_ce_project_to_reservation() { + local jwt_token="$1" + local region="$2" + local ce_project_guid="$3" + local reservation_guid="$4" + local http_code + local response_message + + # This Code Engine API associate the Reservation ID to the Code Engine project previously created + result=$(curl -s -w "%{http_code}" -o "${HTTP_OUTPUT_FILE}" \ + -X PATCH \ + -H "Authorization: Bearer ${jwt_token}" \ + -H "Content-Type: application/json" \ + -d "{\"project_id\":\"${ce_project_guid}\"}" \ + "${CODE_ENGINE_API_ENDPOINT_URL}/${V2BETA_CONTEXT_ROOT}/projects/${ce_project_guid}/capacity_reservations/${reservation_guid}") + + # The curl return a reply with the following format { ... JSON ... }HTTPSTATUS. + # These two lines separate the HTTP STATUS from the JSON reply. + http_code="${result: -3}" + response_message=$(cat "${HTTP_OUTPUT_FILE}") + + # Show both the HTTP code and the response message + echo "${http_code}" + echo "${response_message}" +} + +#################################################################################################### +# Main program +#################################################################################################### +# First of all, let's parse the input parameters so that we have the input variables to work on. +# This parsing will populate the globa variables: +# - RESERVATION_ID +# - REGION +# - RESOURCE_GROUP_ID +# - LOG_FILE +parse_args "$@" +# Initialize the logging file +init_logging + +# The IBM tool https://github.com/ibm/detect-secrets detected a secret keyword. +# However, it detected only the API keyword but no real secret expure exists here. +if [ -z "$IBM_CLOUD_API_KEY" ]; then # pragma: allowlist secret + log "ERROR: environment variable IBM_CLOUD_API_KEY not provided. Run the command:" + log " export IBM_CLOUD_API_KEY=\"\"" # pragma: allowlist secret + exit 1 +fi + +if [ -z "$RESERVATION_ID" ]; then + log "ERROR: environment variable RESERVATION_ID not provided. Run the command:" + log " export RESERVATION_ID=\"\"" + exit 1 +fi + +# Since I have now the value for the REGION variable I can set correctly the: +# - Code Engine API Endpoint URL correctly +# - HPC API Endpoint URL correctly +CODE_ENGINE_API_ENDPOINT_URL=${CODE_ENGINE_API_ENDPOINT_URL//REGION/${REGION}} + +# Try to exchange the IBM Cloud API key for a JWT token. +log "INFO: Retrieving the JWT Token for the IBM_CLOUD_API_KEY." +if ! JWT_TOKEN=$(get_token "${IBM_CLOUD_API_KEY}"); then + HTTP_STATUS=$(echo "${JWT_TOKEN}" | head -n 1) + ERROR_MESSAGE=$(echo "${JWT_TOKEN}" | tail -n 1) + log "ERROR: cannot retrieve JWT token. HTTP Status ${HTTP_STATUS}. ${ERROR_MESSAGE}" + exit 3 +fi + +# HPC Tile has the parameter RESERVATION_ID that is meaningful name like Contract-IBM-WDC-OB +# As first step, we need to get the RESERVATION_GUID starting from the RESERVATION_ID. To do that, +# we retrieve a JSOn file contaaining the list of all the reservations. +log "INFO: Getting the Reservation GUID starting from the ID ." +response=$(get_guid_from_reservation_id "${JWT_TOKEN}") +http_code=$(echo "${response}" | head -n 1) +response_message=$(echo "${response}" | tail -n +2) + +# Check if the RESERVATION_GUID is available +if [ "${http_code}" != "200" ]; then + log "ERROR: Reservation GUID for the ID , wasn't found." + log "ERROR Details: ${response_message}." + exit 4 +fi + +# Now we have to check if in the JSON list exists a reservation equal to RESERVATION_ID +RESERVATION_GUID=$(echo "${response_message}" | jq -r ".capacity_reservations[] | select (.name == \"${RESERVATION_ID}\") | .id") +if [ -z "$RESERVATION_GUID" ]; then + log "ERROR: Reservation GUID for the ID , wasn't found." + exit 4 +fi + +# We found the RESERVATION_GUID associated with the RESERVATION_ID +log "INFO: Reservation (ID: ) has the GUID: ${RESERVATION_GUID}." + +# Now we have to check if the reservation RESERVATION_ID has a Code Engine Project exists for it. +log "INFO: Verifying the existence of a Reservation (GUID: ${RESERVATION_GUID})." +response=$(check_reservation "${JWT_TOKEN}" "${RESERVATION_GUID}") +http_code=$(echo "${response}" | head -n 1) +response_message=$(echo "${response}" | tail -n +2) + +# Check if the a Code Engine Project relative to the RESERVATION ID was found in Code Engine +if [ "${http_code}" != "200" ]; then + log "ERROR: Reservation with GUID ${RESERVATION_GUID}, wasn't found." + log "ERROR Details: ${response_message}." + exit 5 +fi + +# A Reservation with id RESERVATION_ID exists. We need to verify that a Code Engine project exists. +log "INFO: Verifying if the Reservation (GUID: ${RESERVATION_GUID}) is associated with a Code Engine project." + +# Check if a project_id exists in the response_message +CODE_ENGINE_PROJECT_GUID=$(echo "${response_message}" | jq -e -r '.project_id // empty') +if [ -n "${CODE_ENGINE_PROJECT_GUID}" ]; then + log "INFO: Reservation (GUID: ${RESERVATION_GUID}) exists and is associated with the Code Engine project (ID: ${CODE_ENGINE_PROJECT_GUID})." + log "INFO: Write the Code Engine project (ID: ${CODE_ENGINE_PROJECT_GUID} in the ${CODE_ENGINE_PROJECT_GUID_FILE} file." + echo -n "${CODE_ENGINE_PROJECT_GUID}" > "${CODE_ENGINE_PROJECT_GUID_FILE}" + log "INFO: ${0} successfully completed." + exit 0 +fi + +# A Reservation with id RESERVATION_ID exists but a Code Engine project doesn't, we need to create it. +log "INFO: No Code Engine project is associated with the Reservation (GUID: ${RESERVATION_GUID}). Initiating project creation." +response=$(create_ce_project "${JWT_TOKEN}" "${REGION}" "${RESOURCE_GROUP_ID}") +http_code=$(echo "${response}" | head -n 1) +response_message=$(echo "${response}" | tail -n +2) + +# Check if the a Code Engine Project has been created +if [ "${http_code}" != "201" ] && [ "${http_code}" != "202" ]; then + log "ERROR: Cannot create a Code Engine project." + log "ERROR Details: ${response_message}." + exit 6 +fi + +# If Code Engine project has been created, wait for its completion +CODE_ENGINE_PROJECT_GUID=$(echo "${response_message}" | jq -e -r '.guid') +log "INFO: Code Engine project (GUID: ${CODE_ENGINE_PROJECT_GUID}) for the Reservation (GUID: ${RESERVATION_GUID}) has been created. Waiting for its completion." +if ! wait_ce_project_creation "${JWT_TOKEN}" "${REGION}" "${CODE_ENGINE_PROJECT_GUID}"; then + log "ERROR: Code Engine project creation timeout expired." + exit 7 +fi + +# We can associate the Code Engine project id to the Reservation +log "INFO: Code Engine project (GUID: ${CODE_ENGINE_PROJECT_GUID}) is going to be associated to the Reservation with GUID ${RESERVATION_GUID}." +response=$(associate_ce_project_to_reservation "${JWT_TOKEN}" "${REGION}" "${CODE_ENGINE_PROJECT_GUID}" "${RESERVATION_GUID}") +http_code=$(echo "${response}" | head -n 1) +response_message=$(echo "${response}" | tail -n +2) + +# Check if the a Code Engine Project has been created +if [ "${http_code}" != "200" ]; then + log "ERROR: Cannot associate the Code Engine project with guid ${CODE_ENGINE_PROJECT_GUID} to the Reservation with GUID ${RESERVATION_GUID}." + log "ERROR Details: ${response_message}." + exit 8 +fi + +log "INFO: Code Engine project (GUID: ${CODE_ENGINE_PROJECT_GUID}) has been successfully associated to the Reservation with GUID ${RESERVATION_GUID}." +log "INFO: Write the Code Engine project (ID: ${CODE_ENGINE_PROJECT_GUID} in the ${CODE_ENGINE_PROJECT_GUID_FILE} file." +echo -n "${CODE_ENGINE_PROJECT_GUID}" > "${CODE_ENGINE_PROJECT_GUID_FILE}" +log "INFO: ${0} successfully completed." diff --git a/solutions/hpc/scripts/destroy_script.sh b/solutions/hpc/scripts/destroy_script.sh new file mode 100644 index 00000000..01db6abe --- /dev/null +++ b/solutions/hpc/scripts/destroy_script.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +badmin qclose all +bkill -u all +while true; do + if (bhosts -o status) | grep ok; then + sleep 1m + else + sleep 2m + exit 0 + fi +done diff --git a/solutions/hpc/variables.tf b/solutions/hpc/variables.tf index fe248e63..b1cb5115 100644 --- a/solutions/hpc/variables.tf +++ b/solutions/hpc/variables.tf @@ -1,41 +1,15 @@ ############################################################################## -# Offering Variations -############################################################################## -# Future use -/* -variable "scheduler" { - type = string - default = "LSF" - description = "Select one of the scheduler (LSF/Symphony/Slurm/None)" -} - -variable "storage_type" { - type = string - default = "scratch" - description = "Select the required storage type(scratch/persistent/eval)." -} - -variable "ibm_customer_number" { - type = string - sensitive = true - default = "" - description = "Comma-separated list of the IBM Customer Number(s) (ICN) that is used for the Bring Your Own License (BYOL) entitlement check. For more information on how to find your ICN, see [What is my IBM Customer Number (ICN)?](https://www.ibm.com/support/pages/what-my-ibm-customer-number-icn)." - validation { - # regex(...) fails if the IBM customer number has special characters. - condition = can(regex("^[0-9A-Za-z]*([0-9A-Za-z]+,[0-9A-Za-z]+)*$", var.ibm_customer_number)) - error_message = "The IBM customer number input value cannot have special characters." - } -} -*/ -############################################################################## # Account Variables ############################################################################## variable "ibmcloud_api_key" { - description = "IBM Cloud API Key that will be used for authentication in scripts run in this module. Only required if certain options are required." + description = "IBM Cloud API key for the IBM Cloud account where the IBM Cloud HPC cluster needs to be deployed. For more information on how to create an API key, see [Managing user API keys](https://cloud.ibm.com/docs/account?topic=account-userapikey)." type = string sensitive = true - default = null + validation { + condition = var.ibmcloud_api_key != "" + error_message = "The API key for IBM Cloud must be set." + } } ############################################################################## @@ -43,409 +17,571 @@ variable "ibmcloud_api_key" { ############################################################################## variable "resource_group" { - description = "String describing resource groups to create or reference" + description = "Resource group name from your IBM Cloud account where the VPC resources should be deployed. Note. If the resource group value is set as null, automation creates two different RG with the name (workload-rg and service-rg). For additional information on resource groups, see [Managing resource groups](https://cloud.ibm.com/docs/account?topic=account-rgs)." type = string - default = null + default = "Default" + validation { + condition = var.resource_group != null + error_message = "If you want to provide null for resource_group variable, it should be within double quotes." + } } ############################################################################## # Module Level Variables ############################################################################## -variable "prefix" { - description = "A unique identifier for resources. Must begin with a letter and end with a letter or number. This prefix will be prepended to any resources provisioned by this template. Prefixes must be 16 or fewer characters." +variable "cluster_prefix" { + description = "Prefix that is used to name the IBM Cloud HPC cluster and IBM Cloud resources that are provisioned to build the IBM Cloud HPC cluster instance. You cannot create more than one instance of the IBM Cloud HPC cluster with the same name. Ensure that the name is unique. Prefix must start with a lowercase letter and contain only lowercase letters, digits, and hyphens in between. Hyphens must be followed by at least one lowercase letter or digit. There are no leading, trailing, or consecutive hyphens.Character length for cluster_prefix should be less than 16." type = string + default = "hpcaas" validation { - error_message = "Prefix must begin and end with a letter and contain only letters, numbers, and - characters." - condition = can(regex("^([A-z]|[a-z][-a-z0-9]*[a-z0-9])$", var.prefix)) + error_message = "Prefix must start with a lowercase letter and contain only lowercase letters, digits, and hyphens in between. Hyphens must be followed by at least one lowercase letter or digit. There are no leading, trailing, or consecutive hyphens." + condition = can(regex("^[a-z](?:[a-z0-9]*(-[a-z0-9]+)*)?$", var.cluster_prefix)) + } + validation { + condition = length(var.cluster_prefix) <= 16 + error_message = "The cluster_prefix must be 16 characters or fewer." } } variable "zones" { - description = "Region where VPC will be created. To find your VPC region, use `ibmcloud is regions` command to find available regions." + description = "The IBM Cloud zone name within the selected region where the IBM Cloud HPC cluster should be deployed and requires a single zone input value. Supported zones are: eu-de-2 and eu-de-3 for eu-de, us-east-1 and us-east-3 for us-east, and us-south-1 for us-south. The management nodes, file storage shares, and compute nodes will be deployed in the same zone.[Learn more](https://cloud.ibm.com/docs/vpc?topic=vpc-creating-a-vpc-in-a-different-region#get-zones-using-the-cli)." type = list(string) + default = ["us-east-1"] + validation { + condition = length(var.zones) == 1 + error_message = "HPC product deployment supports only a single zone. Provide a value for a single zone from the supported regions: eu-de-2 or eu-de-3 for eu-de, us-east-1 or us-east-3 for us-east, and us-south-1 for us-south." + } +} + +variable "cluster_id" { + type = string + description = "Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservations. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment." + validation { + condition = 0 < length(var.cluster_id) && length(var.cluster_id) < 40 && can(regex("^[a-zA-Z0-9_.-]+$", var.cluster_id)) + error_message = "The ID can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. Other special characters and spaces are not allowed." + } +} + +variable "reservation_id" { + type = string + sensitive = true + description = "Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_)." + validation { + condition = can(regex("^[a-zA-Z][a-zA-Z0-9-_]*$", var.reservation_id)) + error_message = "Reservation ID must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_)." + } } ############################################################################## # VPC Variables ############################################################################## -variable "vpc" { +variable "vpc_name" { type = string description = "Name of an existing VPC in which the cluster resources will be deployed. If no value is given, then a new VPC will be provisioned for the cluster. [Learn more](https://cloud.ibm.com/docs/vpc)" default = null } -variable "network_cidr" { - description = "Network CIDR for the VPC. This is used to manage network ACL rules for cluster provisioning." - type = string - default = "10.0.0.0/8" +variable "cluster_subnet_ids" { + type = list(string) + default = [] + description = "Provide the list of existing subnet ID under the existing VPC where the cluster will be provisioned. One subnet ID is required as input value. Supported zones are: eu-de-2 and eu-de-3 for eu-de, us-east-1 and us-east-3 for us-east, and us-south-1 for us-south. The management nodes, file storage shares, and compute nodes will be deployed in the same zone." + validation { + condition = contains([0, 1], length(var.cluster_subnet_ids)) + error_message = "The subnet_id value should either be empty or contain exactly one element. Provide only a single subnet value from the supported zones." + } } -variable "placement_strategy" { +variable "login_subnet_id" { type = string default = null - description = "VPC placement groups to create (null / host_spread / power_spread)" -} -############################################################################## -# Access Variables -############################################################################## -variable "enable_bastion" { - type = bool - default = true - description = "The solution supports multiple ways to connect to your HPC cluster for example, using bastion node, via VPN or direct connection. If connecting to the HPC cluster via VPN or direct connection, set this value to false." + description = "Provide the list of existing subnet ID under the existing VPC, where the login/bastion server will be provisioned. One subnet id is required as input value for the creation of login node and bastion in the same zone as the management nodes. Note: Provide a different subnet id for login_subnet_id, do not overlap or provide the same subnet id that was already provided for cluster_subnet_ids." } -variable "enable_bootstrap" { - type = bool - default = false - description = "Bootstrap should be only used for better deployment performance" -} - -variable "bootstrap_instance_profile" { +variable "vpc_cidr" { + description = "Creates the address prefix for the new VPC, when the vpc_name variable is empty. The VPC requires an address prefix for creation of subnet in a single zone. The subnet are created with the specified CIDR blocks. For more information, see [Setting IP ranges](https://cloud.ibm.com/docs/vpc?topic=vpc-vpc-addressing-plan-design)." type = string - default = "mx2-4x32" - description = "Bootstrap should be only used for better deployment performance" + default = "10.241.0.0/18" } -variable "bastion_ssh_keys" { +variable "vpc_cluster_private_subnets_cidr_blocks" { type = list(string) - description = "The key pair to use to access the bastion host." + default = ["10.241.0.0/20"] + description = "Provide the CIDR block required for the creation of the compute cluster's private subnet. One CIDR block is required. If using a hybrid environment, modify the CIDR block to avoid conflicts with any on-premises CIDR blocks. Ensure the selected CIDR block size can accommodate the maximum number of management and dynamic compute nodes expected in your cluster. For more information on CIDR block size selection, refer to the documentation, see [Choosing IP ranges for your VPC](https://cloud.ibm.com/docs/vpc?topic=vpc-choosing-ip-ranges-for-your-vpc)." + validation { + condition = length(var.vpc_cluster_private_subnets_cidr_blocks) == 1 + error_message = "Single zone is supported to deploy resources. Provide a CIDR range of subnets creation." + } } -variable "bastion_subnets_cidr" { +variable "vpc_cluster_login_private_subnets_cidr_blocks" { type = list(string) - default = ["10.0.0.0/24"] - description = "Subnet CIDR block to launch the bastion host." + default = ["10.241.16.0/28"] + description = "Provide the CIDR block required for the creation of the login cluster's private subnet. Only one CIDR block is needed. If using a hybrid environment, modify the CIDR block to avoid conflicts with any on-premises CIDR blocks. Since the login subnet is used only for the creation of login virtual server instances, provide a CIDR range of /28." + validation { + condition = length(var.vpc_cluster_login_private_subnets_cidr_blocks) <= 1 + error_message = "Only a single zone is supported to deploy resources. Provide a CIDR range of subnet creation." + } + validation { + condition = tonumber(regex("/(\\d+)", join(",", var.vpc_cluster_login_private_subnets_cidr_blocks))[0]) <= 28 + error_message = "This subnet is used to create only a login virtual server instance. Providing a larger CIDR size will waste the usage of available IPs. A CIDR range of /28 is sufficient for the creation of the login subnet." + } } -variable "enable_vpn" { - type = bool - default = false - description = "The solution supports multiple ways to connect to your HPC cluster for example, using bastion node, via VPN or direct connection. If connecting to the HPC cluster via VPN, set this value to true." -} +############################################################################## +# Access Variables +############################################################################## -variable "vpn_peer_cidr" { +variable "remote_allowed_ips" { type = list(string) - default = null - description = "The peer CIDRs (e.g., 192.168.0.0/24) to which the VPN will be connected." + description = "Comma-separated list of IP addresses that can access the IBM Cloud HPC cluster instance through an SSH interface. For security purposes, provide the public IP addresses assigned to the devices that are authorized to establish SSH connections (for example, [\"169.45.117.34\"]). To fetch the IP address of the device, use [https://ipv4.icanhazip.com/](https://ipv4.icanhazip.com/)." + validation { + condition = alltrue([ + for o in var.remote_allowed_ips : !contains(["0.0.0.0/0", "0.0.0.0"], o) + ]) + error_message = "For security, provide the public IP addresses assigned to the devices authorized to establish SSH connections. Use https://ipv4.icanhazip.com/ to fetch the ip address of the device." + } + validation { + condition = alltrue([ + for a in var.remote_allowed_ips : can(regex("^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(/(3[0-2]|2[0-9]|1[0-9]|[0-9]))?$", a)) + ]) + error_message = "The provided IP address format is not valid. Check if the IP address contains a comma instead of a dot, and ensure there are double quotation marks between each IP address range if using multiple IP ranges. For multiple IP address, use the format [\"169.45.117.34\",\"128.122.144.145\"]." + } } -variable "vpn_peer_address" { - type = string - default = null - description = "The peer public IP address to which the VPN will be connected." -} +############################################################################## +# Compute Variables +############################################################################## -variable "vpn_preshared_key" { - type = string - default = null - description = "The pre-shared key for the VPN." +variable "bastion_ssh_keys" { + type = list(string) + description = "Provide the list of SSH key names configured in your IBM Cloud account to establish a connection to the IBM Cloud HPC bastion and login node. Ensure the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by following the provided instructions.[SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys)." } -variable "allowed_cidr" { - description = "Network CIDR to access the VPC. This is used to manage network ACL rules for accessing the cluster." +variable "compute_ssh_keys" { type = list(string) - default = ["10.0.0.0/8"] + description = "Provide the list of SSH key names configured in your IBM Cloud account to establish a connection to the IBM Cloud HPC cluster node. Ensure the SSH key is present in the same resource group and region where the cluster is being provisioned. If you do not have an SSH key in your IBM Cloud account, create one by following the provided instructions.[SSH Keys](https://cloud.ibm.com/docs/vpc?topic=vpc-ssh-keys)." } -############################################################################## -# Compute Variables -############################################################################## -# Future use -/* -variable "login_subnets_cidr" { - type = list(string) - default = ["10.10.10.0/24", "10.20.10.0/24", "10.30.10.0/24"] - description = "Subnet CIDR block to launch the login host." +variable "login_node_instance_type" { + type = string + default = "bx2-2x8" + description = "Specify the virtual server instance profile type to be used to create the login node for the IBM Cloud HPC cluster. For choices on profile types, see [Instance profiles](https://cloud.ibm.com/docs/vpc?topic=vpc-profiles)." + validation { + condition = can(regex("^[^\\s]+-[0-9]+x[0-9]+", var.login_node_instance_type)) + error_message = "The profile must be a valid profile name." + } } -*/ +variable "management_image_name" { + type = string + default = "hpcaas-lsf10-rhel88-v6" + description = "Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster management nodes. By default, the solution uses a RHEL88 base image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering." -variable "login_ssh_keys" { - type = list(string) - description = "The key pair to use to launch the login host." +} + +variable "compute_image_name" { + type = string + default = "hpcaas-lsf10-rhel88-compute-v5" + description = "Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster dynamic compute nodes. By default, the solution uses a RHEL 8-8 base OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v4). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering." } variable "login_image_name" { type = string - default = "ibm-redhat-8-6-minimal-amd64-5" - description = "Image name to use for provisioning the login instances." + default = "hpcaas-lsf10-rhel88-compute-v5" + description = "Name of the custom image that you want to use to create virtual server instances in your IBM Cloud account to deploy the IBM Cloud HPC cluster login node. By default, the solution uses a RHEL 8-8 OS image with additional software packages mentioned [here](https://cloud.ibm.com/docs/ibm-spectrum-lsf#create-custom-image). The solution also offers, Ubuntu 22-04 OS base image (hpcaas-lsf10-ubuntu2204-compute-v4). If you would like to include your application-specific binary files, follow the instructions in [ Planning for custom images ](https://cloud.ibm.com/docs/vpc?topic=vpc-planning-custom-images) to create your own custom image and use that to build the IBM Cloud HPC cluster through this offering." } -variable "login_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 1 - }] - description = "Number of instances to be launched for login." +variable "management_node_instance_type" { + type = string + default = "bx2-16x64" + description = "Specify the virtual server instance profile type to be used to create the management nodes for the IBM Cloud HPC cluster. For choices on profile types, see [Instance profiles](https://cloud.ibm.com/docs/vpc?topic=vpc-profiles)." + validation { + condition = can(regex("^[^\\s]+-[0-9]+x[0-9]+", var.management_node_instance_type)) + error_message = "The profile must be a valid profile name." + } } -variable "compute_subnets_cidr" { - type = list(string) - default = ["10.10.20.0/24", "10.20.20.0/24", "10.30.20.0/24"] - description = "Subnet CIDR block to launch the compute cluster host." +variable "management_node_count" { + type = number + default = 3 + description = "Number of management nodes. This is the total number of management nodes. Enter a value between 1 and 10." + validation { + condition = 1 <= var.management_node_count && var.management_node_count <= 10 + error_message = "Input \"management_node_count\" must be must be greater than or equal to 1 and less than or equal to 10." + } } -variable "compute_ssh_keys" { - type = list(string) - description = "The key pair to use to launch the compute host." +variable "custom_file_shares" { + type = list(object({ + mount_path = string, + size = optional(number), + iops = optional(number), + nfs_share = optional(string) + })) + default = [{ mount_path = "/mnt/vpcstorage/tools", size = 100, iops = 2000 }, { mount_path = "/mnt/vpcstorage/data", size = 100, iops = 6000 }, { mount_path = "/mnt/scale/tools", nfs_share = "" }] + description = "Mount points and sizes in GB and IOPS range of file shares that can be used to customize shared file storage layout. Provide the details for up to 5 shares. Each file share size in GB supports different range of IOPS. For more information, see [file share IOPS value](https://cloud.ibm.com/docs/vpc?topic=vpc-file-storage-profiles&interface=ui)." + validation { + condition = length([for item in var.custom_file_shares : item if item.nfs_share == null]) <= 5 + error_message = "The VPC storage custom file share count \"custom_file_shares\" must be less than or equal to 5. Unlimited NFS mounts are allowed." + } + validation { + condition = !anytrue([for mounts in var.custom_file_shares : mounts.mount_path == "/mnt/lsf"]) + error_message = "The mount path /mnt/lsf is reserved for internal usage and can't be used as file share mount_path." + } + validation { + condition = length([for mounts in var.custom_file_shares : mounts.mount_path]) == length(toset([for mounts in var.custom_file_shares : mounts.mount_path])) + error_message = "Mount path values should not be duplicated." + } + validation { + condition = alltrue([for mounts in var.custom_file_shares : can(mounts.size) && mounts.size != null ? (10 <= mounts.size && mounts.size <= 32000) : true]) + error_message = "The custom_file_share size must be greater than or equal to 10 and less than or equal to 32000." + } } -variable "management_image_name" { +variable "storage_security_group_id" { type = string - default = "ibm-redhat-8-6-minimal-amd64-5" - description = "Image name to use for provisioning the management cluster instances." -} - -variable "management_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 3 - }] - description = "Number of instances to be launched for management." -} - -variable "static_compute_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 0 - }] - description = "Min Number of instances to be launched for compute cluster." -} - -variable "dynamic_compute_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 250 - }] - description = "MaxNumber of instances to be launched for compute cluster." + default = null + description = "Provide the storage security group ID created from the Spectrum Scale storage cluster if the nfs_share value is updated to use the scale fileset mountpoints under the cluster_file_share variable." } -variable "compute_image_name" { +############################################################################## +# DNS Template Variables +############################################################################## + +variable "dns_instance_id" { type = string - default = "ibm-redhat-8-6-minimal-amd64-5" - description = "Image name to use for provisioning the compute cluster instances." + default = null + description = "Provide the id of existing IBM Cloud DNS services domain to skip creating a new DNS service instance name. Note: If dns_instance_id is not equal to null, a new dns zone will be created under the existing dns service instance." } -# Future use -/* -variable "compute_gui_username" { - type = string - default = "admin" - sensitive = true - description = "GUI user to perform system management and monitoring tasks on compute cluster." + +variable "dns_domain_name" { + type = object({ + compute = string + #storage = string + #protocol = string + }) + default = { + compute = "hpcaas.com" + } + description = "IBM Cloud DNS Services domain name to be used for the IBM Cloud HPC cluster." + validation { + condition = can(regex("^([a-zA-Z0-9][a-zA-Z0-9-]{0,61}[a-zA-Z0-9])\\.com$", var.dns_domain_name.compute)) + #condition = can(regex("^([a-zA-Z0-9][a-zA-Z0-9-]{0,61}[a-zA-Z0-9]\\.)+[a-zA-Z]{2,6}$", var.dns_domain_names.compute)) + #condition = can(regex("^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\\.[a-zA-Z]{2,6}$", var.dns_domain_names.compute)) + #condition = can(regex("^([[:alnum:]]*[A-Za-z0-9-]{1,63}\\.)+[A-Za-z]{2,6}$", var.dns_domain_names.compute)) + error_message = "The domain name provided for compute is not a fully qualified domain name (FQDN). An FQDN can contain letters (a-z, A-Z), digits (0-9), hyphens (-), dots (.), and must start and end with an alphanumeric character." + } } -variable "compute_gui_password" { +variable "dns_custom_resolver_id" { type = string - sensitive = true - description = "Password for compute cluster GUI" + default = null + description = "Provide the id of existing IBM Cloud DNS custom resolver to skip creating a new custom resolver. Note: A VPC can be associated only to a single custom resolver, please provide the id of custom resolver if it is already associated to the VPC." } -*/ + ############################################################################## -# Scale Storage Variables +# Observability Variables ############################################################################## -variable "storage_subnets_cidr" { - type = list(string) - default = ["10.10.30.0/24", "10.20.30.0/24", "10.30.30.0/24"] - description = "Subnet CIDR block to launch the storage cluster host." +variable "enable_cos_integration" { + type = bool + default = false + description = "Set to true to create an extra cos bucket to integrate with HPC cluster deployment." } -variable "storage_ssh_keys" { - type = list(string) - description = "The key pair to use to launch the storage cluster host." +variable "cos_instance_name" { + type = string + default = null + description = "Provide the name of the existing cos instance to store vpc flow logs." } -variable "storage_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "bx2-2x8" - count = 3 - }] - description = "Number of instances to be launched for storage cluster." +variable "observability_atracker_on_cos_enable" { + type = bool + default = true + description = "Enable Activity tracker service instance connected to Cloud Object Storage (COS). All the events will be stored into COS so that customers can connect to it and read those events or ingest them in their system." } -variable "storage_image_name" { - type = string - default = "ibm-redhat-8-6-minimal-amd64-5" - description = "Image name to use for provisioning the storage cluster instances." +variable "enable_vpc_flow_logs" { + type = bool + default = false + description = "Flag to enable VPC flow logs. If true, a flow log collector will be created." } -variable "protocol_subnets_cidr" { - type = list(string) - default = ["10.10.40.0/24", "10.20.40.0/24", "10.30.40.0/24"] - description = "Subnet CIDR block to launch the storage cluster host." -} - -variable "protocol_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "bx2-2x8" - count = 2 - }] - description = "Number of instances to be launched for protocol hosts." -} -# Future use -/* -variable "storage_gui_username" { - type = string - default = "admin" - sensitive = true - description = "GUI user to perform system management and monitoring tasks on storage cluster." +variable "vpn_enabled" { + type = bool + default = false + description = "Set the value as true to deploy a VPN gateway for VPC in the cluster." +} + +variable "observability_monitoring_enable" { + description = "Set false to disable IBM Cloud Monitoring integration. If enabled, infrastructure and LSF application metrics from Management Nodes will be ingested." + type = bool + default = false } -variable "storage_gui_password" { +variable "observability_monitoring_on_compute_nodes_enable" { + description = "Set false to disable IBM Cloud Monitoring integration. If enabled, infrastructure metrics from Compute Nodes will be ingested." + type = bool + default = false +} + +variable "observability_monitoring_plan" { + description = "Type of service plan for IBM Cloud Monitoring instance. You can choose one of the following: lite, graduated-tier. For all details visit [IBM Cloud Monitoring Service Plans](https://cloud.ibm.com/docs/monitoring?topic=monitoring-service_plans)." type = string - sensitive = true - description = "Password for storage cluster GUI" -} -*/ - -variable "file_shares" { - type = list( - object({ - mount_path = string, - size = number, - iops = number - }) - ) - default = [{ - mount_path = "/mnt/binaries" - size = 100 - iops = 1000 - }, { - mount_path = "/mnt/data" - size = 100 - iops = 1000 - }] - description = "Custom file shares to access shared storage" -} - -variable "nsd_details" { - type = list( - object({ - profile = string - capacity = optional(number) - iops = optional(number) - }) - ) - default = [{ - profile = "custom" - size = 100 - iops = 100 - }] - description = "Storage scale NSD details" + default = "graduated-tier" + validation { + condition = can(regex("lite|graduated-tier", var.observability_monitoring_plan)) + error_message = "Please enter a valid plan for IBM Cloud Monitoring, for all details visit https://cloud.ibm.com/docs/monitoring?topic=monitoring-service_plans." + } } ############################################################################## -# DNS Template Variables +# Encryption Variables ############################################################################## -variable "dns_instance_id" { +variable "key_management" { + type = string + default = "key_protect" + description = "Set the value as key_protect to enable customer managed encryption for boot volume and file share. If the key_management is set as null, encryption will be always provider managed." + validation { + condition = var.key_management == "null" || var.key_management == null || var.key_management == "key_protect" + error_message = "key_management must be either 'null' or 'key_protect'." + } +} + +variable "kms_instance_name" { type = string default = null - description = "IBM Cloud HPC DNS service instance id." + description = "Provide the name of the existing Key Protect instance associated with the Key Management Service. Note: To use existing kms_instance_name shall be considered only if key_management value is set as key_protect under key_management variable. The name can be found under the details of the KMS, see [View key-protect ID](https://cloud.ibm.com/docs/key-protect?topic=key-protect-retrieve-instance-ID&interface=ui)." } -variable "dns_custom_resolver_id" { +variable "kms_key_name" { type = string default = null - description = "IBM Cloud DNS custom resolver id." + description = "Provide the existing KMS encryption key name that you want to use for the IBM Cloud HPC cluster. Note: kms_key_name to be considered only if key_management value is set as key_protect under key_management variable.(for example kms_key_name: my-encryption-key)." } -variable "dns_domain_names" { - type = object({ - compute = string - storage = string - protocol = string - }) - default = { - compute = "comp.com" - storage = "strg.com" - protocol = "ces.com" +############################################################################## +# SCC Variables +############################################################################## + +variable "scc_enable" { + type = bool + default = false + description = "Flag to enable SCC instance creation. If true, an instance of SCC (Security and Compliance Center) will be created." +} + +variable "scc_profile" { + type = string + default = "CIS IBM Cloud Foundations Benchmark" + description = "Profile to be set on the SCC Instance (accepting empty, 'CIS IBM Cloud Foundations Benchmark' and 'IBM Cloud Framework for Financial Services')" + validation { + condition = can(regex("^(|CIS IBM Cloud Foundations Benchmark|IBM Cloud Framework for Financial Services)$", var.scc_profile)) + error_message = "Provide SCC Profile Name to be used (accepting empty, 'CIS IBM Cloud Foundations Benchmark' and 'IBM Cloud Framework for Financial Services')." + } +} + +variable "scc_profile_version" { + type = string + default = "1.0.0" + description = "Version of the Profile to be set on the SCC Instance (accepting empty, CIS and Financial Services profiles versions)" + validation { + condition = can(regex("^(|\\d+\\.\\d+(\\.\\d+)?)$", var.scc_profile_version)) + error_message = "Provide SCC Profile Version to be used." + } +} + +variable "scc_location" { + description = "Location where the SCC instance is provisioned (possible choices 'us-south', 'eu-de', 'ca-tor', 'eu-es')" + type = string + default = "us-south" + validation { + condition = can(regex("^(|us-south|eu-de|ca-tor|eu-es)$", var.scc_location)) + error_message = "Provide region where it's possible to deploy an SCC Instance (possible choices 'us-south', 'eu-de', 'ca-tor', 'eu-es') or leave blank and it will default to 'us-south'." + } +} + +variable "scc_event_notification_plan" { + type = string + default = "lite" + description = "Event Notifications Instance plan to be used (it's used with S.C.C. instance), possible values 'lite' and 'standard'." + validation { + condition = can(regex("^(|lite|standard)$", var.scc_event_notification_plan)) + error_message = "Provide Event Notification instance plan to be used (accepting 'lite' and 'standard', defaulting to 'lite'). This instance is used in conjuction with S.C.C. one." } - description = "IBM Cloud HPC DNS domain names." } ############################################################################## -# Observability Variables +# Hyper-Threading in Compute Nodes ############################################################################## -variable "enable_cos_integration" { +variable "hyperthreading_enabled" { type = bool default = true - description = "Integrate COS with HPC solution" + description = "Setting this to true will enable hyper-threading in the compute nodes of the cluster (default). Otherwise, hyper-threading will be disabled." } -variable "cos_instance_name" { +############################################################################## +# Encryption Variables +############################################################################## +variable "enable_app_center" { + type = bool + default = false + description = "Set to true to enable the IBM Spectrum LSF Application Center GUI (default: false). [System requirements](https://www.ibm.com/docs/en/slac/10.2.0?topic=requirements-system-102-fix-pack-14) for IBM Spectrum LSF Application Center Version 10.2 Fix Pack 14." +} + +variable "app_center_gui_pwd" { type = string - default = null - description = "Exiting COS instance name" + sensitive = true + default = "" + description = "Password for IBM Spectrum LSF Application Center GUI. Note: Password should be at least 8 characters, must have one number, one lowercase letter, one uppercase letter, and at least one special character." } -variable "enable_atracker" { +variable "app_center_high_availability" { type = bool default = true - description = "Enable Activity tracker" + description = "Set to false to disable the IBM Spectrum LSF Application Center GUI High Availability (default: true)." } -variable "enable_vpc_flow_logs" { +variable "enable_fip" { type = bool default = true - description = "Enable Activity tracker" + description = "The solution supports multiple ways to connect to your IBM Cloud HPC cluster for example, using a login node, or using VPN or direct connection. If connecting to the IBM Cloud HPC cluster using VPN or direct connection, set this value to false." } ############################################################################## -# Encryption Variables +# ldap Variables ############################################################################## +variable "enable_ldap" { + type = bool + default = false + description = "Set this option to true to enable LDAP for IBM Cloud HPC, with the default value set to false." +} -variable "key_management" { +variable "ldap_basedns" { type = string - default = "key_protect" - description = "null/key_protect/hs_crypto" + default = "hpcaas.com" + description = "The dns domain name is used for configuring the LDAP server. If an LDAP server is already in existence, ensure to provide the associated DNS domain name." } -variable "hpcs_instance_name" { +variable "ldap_server" { type = string - default = null - description = "Hyper Protect Crypto Service instance" + default = "null" + description = "Provide the IP address for the existing LDAP server. If no address is given, a new LDAP server will be created." +} + +variable "ldap_admin_password" { + type = string + sensitive = true + default = "" + description = "The LDAP administrative password should be 8 to 20 characters long, with a mix of at least three alphabetic characters, including one uppercase and one lowercase letter. It must also include two numerical digits and at least one special character from (~@_+:) are required. It is important to avoid including the username in the password for enhanced security.[This value is ignored for an existing LDAP server]." +} + +variable "ldap_user_name" { + type = string + default = "" + description = "Custom LDAP User for performing cluster operations. Note: Username should be between 4 to 32 characters, (any combination of lowercase and uppercase letters).[This value is ignored for an existing LDAP server]" +} + +variable "ldap_user_password" { + type = string + sensitive = true + default = "" + description = "The LDAP user password should be 8 to 20 characters long, with a mix of at least three alphabetic characters, including one uppercase and one lowercase letter. It must also include two numerical digits and at least one special character from (~@_+:) are required.It is important to avoid including the username in the password for enhanced security.[This value is ignored for an existing LDAP server]." +} + +variable "ldap_vsi_profile" { + type = string + default = "cx2-2x4" + description = "Specify the virtual server instance profile type to be used to create the ldap node for the IBM Cloud HPC cluster. For choices on profile types, see [Instance profiles](https://cloud.ibm.com/docs/vpc?topic=vpc-profiles)." +} + +variable "ldap_vsi_osimage_name" { + type = string + default = "ibm-ubuntu-22-04-3-minimal-amd64-1" + description = "Image name to be used for provisioning the LDAP instances. By default ldap server are created on Ubuntu based OS flavour." +} + +variable "skip_iam_authorization_policy" { + type = string + default = false + description = "Set it to false if authorization policy is required for VPC block storage volumes to access kms. This can be set to true if authorization policy already exists. For more information on how to create authorization policy manually, see [creating authorization policies for block storage volume](https://cloud.ibm.com/docs/vpc?topic=vpc-block-s2s-auth&interface=ui)." +} + +variable "skip_iam_share_authorization_policy" { + type = bool + default = false + description = "Set it to false if authorization policy is required for VPC file share to access kms. This can be set to true if authorization policy already exists. For more information on how to create authorization policy manually, see [creating authorization policies for VPC file share](https://cloud.ibm.com/docs/vpc?topic=vpc-file-s2s-auth&interface=ui)." +} + +########################################################################### +# IBM Cloud ALB Variables +########################################################################### +variable "existing_certificate_instance" { + description = "When app_center_high_availability is enable/set as true, The Application Center will be configured for high availability and requires a Application Load Balancer Front End listener to use a certificate CRN value stored in the Secret Manager. Provide the valid 'existing_certificate_instance' to configure the Application load balancer." + type = string + default = "" } ############################################################################## -# TODO: Auth Server (LDAP/AD) Variables +# Environment Variables ############################################################################## + +# tflint-ignore: all +variable "TF_VERSION" { + type = string + default = "1.5" + description = "The version of the Terraform engine that's used in the Schematics workspace." +} + +# tflint-ignore: all +variable "TF_PARALLELISM" { + type = string + default = "250" + description = "Parallelism/ concurrent operations limit. Valid values are between 1 and 256, both inclusive. [Learn more](https://www.terraform.io/docs/internals/graph.html#walking-the-graph)." + validation { + condition = 1 <= var.TF_PARALLELISM && var.TF_PARALLELISM <= 256 + error_message = "Input \"TF_PARALLELISM\" must be greater than or equal to 1 and less than or equal to 256." + } +} + +# tflint-ignore: terraform_naming_convention +variable "TF_VALIDATION_SCRIPT_FILES" { + type = list(string) + default = [] + description = "List of script file names used by validation test suites. If provided, these scripts will be executed as part of validation test suites execution." + validation { + condition = alltrue([for filename in var.TF_VALIDATION_SCRIPT_FILES : can(regex(".*\\.sh$", filename))]) + error_message = "All validation script file names must end with .sh." + } +} +########################################################################### +# Existing Bastion Support variables +########################################################################### + +variable "bastion_instance_name" { + type = string + default = null + description = "Bastion instance name. If none given then new bastion will be created." +} + +variable "bastion_instance_public_ip" { + type = string + default = null + description = "Bastion instance public ip address." +} + +variable "bastion_security_group_id" { + type = string + default = null + description = "Bastion security group id." +} + +variable "bastion_ssh_private_key" { + type = string + sensitive = true + default = null + description = "Bastion SSH private key path, which will be used to login to bastion host." +} diff --git a/solutions/hpc/version.tf b/solutions/hpc/version.tf index 0c47ca34..d53e9639 100644 --- a/solutions/hpc/version.tf +++ b/solutions/hpc/version.tf @@ -3,12 +3,15 @@ terraform { required_providers { ibm = { source = "IBM-Cloud/ibm" - version = ">= 1.56.2" + version = "1.65.1" + } + null = { + source = "hashicorp/null" + version = "3.2.2" + } + http = { + source = "hashicorp/http" + version = "3.4.2" } } } - -provider "ibm" { - ibmcloud_api_key = var.ibmcloud_api_key - region = local.region -} diff --git a/tests/README.md b/tests/README.md index e69de29b..7cded2a4 100644 --- a/tests/README.md +++ b/tests/README.md @@ -0,0 +1,115 @@ + +# IBM Cloud HPC - Running Tests with Terratest + +## Prerequisites + +Ensure the following tools and utilities are installed and configured on your system: + +- **Go Programming Language** +- **Git** +- **Terraform** +- **IBM Cloud Plugins**: + ```sh + ibmcloud plugin install cloud-object-storage + ibmcloud plugin install key-protect -r "IBM Cloud" + ``` +- **Initialize Git Submodules**: + ```sh + git submodule update --init + ``` + +## Clone the Repository + +Clone the repository containing your Go project: + +```sh +git clone https://github.ibm.com/workload-eng-services/HPCaaS.git +``` + +## Set up Your Go Project + +1. Navigate to the project directory: + ```sh + cd HPCaaS/tests + ``` + +2. Install project dependencies using Go modules: + ```sh + go mod tidy + ``` + +## Running the Tests + +### Option 1: Use Default Parameters from YAML File + +You can run the tests using the default parameter values specified in the YAML file: + +```sh +go test -v -timeout 900m -parallel 4 -run "TestRunBasic" | tee test_output.log +``` + +### Option 2: Override Parameters + +If you want to override the default values, you can pass only the parameters you need to change, or you can override all the values based on your requirements. To do this, execute the following command with your desired parameter values: + +```sh +SSH_KEY=your_ssh_key ZONE=your_zone RESOURCE_GROUP=your_resource_group RESERVATION_ID=your_reservation_id KMS_INSTANCE_ID=kms_instance_id KMS_KEY_NAME=kms_key_name IMAGE_NAME=image_name CLUSTER=your_cluster_id DEFAULT_RESOURCE_GROUP=default_resource_group NON_DEFAULT_RESOURCE_GROUP=non_default_resource_group LOGIN_NODE_INSTANCE_TYPE=login_node_instance_type MANAGEMENT_IMAGE_NAME=management_image_name COMPUTE_IMAGE_NAME=compute_image_name MANAGEMENT_NODE_INSTANCE_TYPE=management_node_instance_type MANAGEMENT_NODE_COUNT=management_node_count ENABLE_VPC_FLOW_LOGS=enable_vpc_flow_logs KEY_MANAGEMENT=key_management KMS_INSTANCE_NAME=kms_instance_name HYPERTHREADING_ENABLED=hyperthreading_enabled US_EAST_ZONE=us_east_zone US_EAST_RESERVATION_ID=us_east_reservation_id US_EAST_CLUSTER_ID=us_east_cluster_id US_SOUTH_ZONE=us_south_zone US_SOUTH_RESERVATION_ID=us_south_reservation_id US_SOUTH_CLUSTER_ID=us_south_cluster_id EU_DE_ZONE=eu_de_zone EU_DE_RESERVATION_ID=eu_de_reservation_id EU_DE_CLUSTER_ID=eu_de_cluster_id SSH_FILE_PATH=ssh_file_path go test -v -timeout 900m -parallel 4 -run "TestRunBasic" | tee test_output.log +``` + +Replace placeholders (e.g., `your_ssh_key`, `your_zone`) with actual values. + +### Running Multiple Tests Simultaneously + +To run multiple tests at the same time: + +```sh +go test -v -timeout 900m -parallel 10 -run="TestRunDefault|TestRunBasic|TestRunLDAP|TestRunAppCenter" | tee test_output.log +``` + +### Export API Key + +Before running tests, export the IBM Cloud API key: + +```sh +export TF_VAR_ibmcloud_api_key=your_api_key //pragma: allowlist secret +``` + +Replace `your_api_key` with your actual API key. //pragma: allowlist secret + +## Analyzing Test Results + +### Review Test Output + +- **Passing Test Example**: + ```sh + --- PASS: TestRunHpcBasicExample (514.35s) + PASS + ok github.com/terraform-ibm-modules/terraform-ibmcloud-hpc 514.369s + ``` + +- **Failing Test Example**: + ```sh + --- FAIL: TestRunHpcBasicExample (663.30s) + FAIL + exit status 1 + FAIL github.com/terraform-ibm-modules/terraform-ibmcloud-hpc 663.323s + ``` + +### Test Output Logs + +- **Console Output**: Check the console for detailed test results. +- **Log Files**: Review `test_output.log` and custom logs in the `/tests/test_output` folder with a timestamp for detailed analysis and troubleshooting. For example, a log file might be named `log_20XX-MM-DD_HH-MM-SS.log`. + +## Troubleshooting + +### Common Issues + +- **Missing Test Directories**: Verify the project structure and required files. +- **Invalid API Key**: Ensure `TF_VAR_ibmcloud_api_key` is correct. +- **Invalid SSH Key**: Check the `SSH_KEY` value. +- **Invalid Zone**: Ensure `ZONE` is set correctly. +- **Remote IP Configuration**: Customize `REMOTE_ALLOWED_IPS` if needed. +- **Terraform Initialization**: Ensure Terraform modules and plugins are up-to-date. +- **Test Output Logs**: Review logs for errors and failure messages. + +For additional assistance, contact the project maintainers. diff --git a/tests/common_utils/deploy_utils.go b/tests/common_utils/deploy_utils.go new file mode 100644 index 00000000..3f3c035e --- /dev/null +++ b/tests/common_utils/deploy_utils.go @@ -0,0 +1,189 @@ +package tests + +import ( + "fmt" + "os" + "strings" + + "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/common" + "gopkg.in/yaml.v3" +) + +var ( + ip string + reservationIDSouth string + reservationIDEast string +) + +// Define a struct with fields that match the structure of the YAML data +const yamlLocation = "../common-dev-assets/common-go-assets/common-permanent-resources.yaml" + +// Config represents the structure of the configuration file. +type Config struct { + DefaultResourceGroup string `yaml:"default_resource_group"` + NonDefaultResourceGroup string `yaml:"non_default_resource_group"` + Zone string `yaml:"zone"` + ClusterID string `yaml:"cluster_id"` + ReservationID string `yaml:"reservation_id"` + RemoteAllowedIPs string `yaml:"remote_allowed_ips"` + SSHKey string `yaml:"ssh_key"` + LoginNodeInstanceType string `yaml:"login_node_instance_type"` + LoginNodeImageName string `yaml:"login_image_name"` + ManagementImageName string `yaml:"management_image_name"` + ComputeImageName string `yaml:"compute_image_name"` + ManagementNodeInstanceType string `yaml:"management_node_instance_type"` + ManagementNodeCount int `yaml:"management_node_count"` + EnableVPCFlowLogs bool `yaml:"enable_vpc_flow_logs"` + KeyManagement string `yaml:"key_management"` + KMSInstanceName string `yaml:"kms_instance_name"` + KMSKeyName string `yaml:"kms_key_name"` + HyperthreadingEnabled bool `yaml:"hyperthreading_enabled"` + DnsDomainName string `yaml:"dns_domain_name"` + EnableAppCenter bool `yaml:"enable_app_center"` + AppCenterGuiPassword string `yaml:"app_center_gui_pwd"` // pragma: allowlist secret + EnableLdap bool `yaml:"enable_ldap"` + LdapBaseDns string `yaml:"ldap_basedns"` + LdapServer string `yaml:"ldap_server"` + LdapAdminPassword string `yaml:"ldap_admin_password"` // pragma: allowlist secret + LdapUserName string `yaml:"ldap_user_name"` + LdapUserPassword string `yaml:"ldap_user_password"` // pragma: allowlist secret + USEastZone string `yaml:"us_east_zone"` + USEastClusterID string `yaml:"us_east_cluster_id"` + USEastReservationID string `yaml:"us_east_reservation_id"` + EUDEZone string `yaml:"eu_de_zone"` + EUDEClusterID string `yaml:"eu_de_cluster_id"` + EUDEReservationID string `yaml:"eu_de_reservation_id"` + USSouthZone string `yaml:"us_south_zone"` + USSouthClusterID string `yaml:"us_south_cluster_id"` + USSouthReservationID string `yaml:"us_south_reservation_id"` + SSHFilePath string `yaml:"ssh_file_path"` +} + +// GetConfigFromYAML reads configuration from a YAML file and sets environment variables based on the configuration. +// It returns a Config struct populated with the configuration values. +func GetConfigFromYAML(filePath string) (*Config, error) { + var config Config + + // Open the YAML file + file, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open YAML file %s: %v", filePath, err) + } + defer file.Close() + + // Decode the YAML file into the config struct + if err := yaml.NewDecoder(file).Decode(&config); err != nil { + return nil, fmt.Errorf("failed to decode YAML: %v", err) + } + + // Get the public IP + ip, err := GetPublicIP() + if err != nil { + return nil, fmt.Errorf("failed to get public IP: %v", err) + } + config.RemoteAllowedIPs = ip + + // Load permanent resources from YAML + permanentResources, err := common.LoadMapFromYaml(yamlLocation) + if err != nil { + return nil, fmt.Errorf("failed to load permanent resources from YAML: %v", err) + } + + // Retrieve reservation ID from Secret Manager // pragma: allowlist secret + reservationIDEastPtr, err := GetSecretsManagerKey( + permanentResources["secretsManagerGuid"].(string), + permanentResources["secretsManagerRegion"].(string), + permanentResources["reservation_id_secret_id"].(string), + ) + if err != nil { + fmt.Printf("error retrieving reservation id from secrets: %v", err) // pragma: allowlist secret + } else if reservationIDEastPtr != nil { + reservationIDEast = *reservationIDEastPtr + } + + // Set environment variables from config + if err := setEnvFromConfig(&config); err != nil { + return nil, fmt.Errorf("failed to set environment variables: %v", err) + } + + return &config, nil +} + +// setEnvFromConfig sets environment variables based on the provided configuration. +func setEnvFromConfig(config *Config) error { + envVars := map[string]interface{}{ + "DEFAULT_RESOURCE_GROUP": config.DefaultResourceGroup, + "NON_DEFAULT_RESOURCE_GROUP": config.NonDefaultResourceGroup, + "ZONE": config.Zone, + "CLUSTER_ID": config.ClusterID, + "RESERVATION_ID": config.ReservationID, + "REMOTE_ALLOWED_IPS": config.RemoteAllowedIPs, + "SSH_KEY": config.SSHKey, + "LOGIN_NODE_INSTANCE_TYPE": config.LoginNodeInstanceType, + "LOGIN_NODE_IMAGE_NAME": config.LoginNodeImageName, + "MANAGEMENT_IMAGE_NAME": config.ManagementImageName, + "COMPUTE_IMAGE_NAME": config.ComputeImageName, + "MANAGEMENT_NODE_INSTANCE_TYPE": config.ManagementNodeInstanceType, + "MANAGEMENT_NODE_COUNT": config.ManagementNodeCount, + "ENABLE_VPC_FLOW_LOGS": config.EnableVPCFlowLogs, + "KEY_MANAGEMENT": config.KeyManagement, + "KMS_INSTANCE_NAME": config.KMSInstanceName, + "KMS_KEY_NAME": config.KMSKeyName, + "HYPERTHREADING_ENABLED": config.HyperthreadingEnabled, + "DNS_DOMAIN_NAME": config.DnsDomainName, + "ENABLE_APP_CENTER": config.EnableAppCenter, + "APP_CENTER_GUI_PASSWORD": config.AppCenterGuiPassword, //pragma: allowlist secret + "ENABLE_LDAP": config.EnableLdap, + "LDAP_BASEDNS": config.LdapBaseDns, + "LDAP_SERVER": config.LdapServer, + "LDAP_ADMIN_PASSWORD": config.LdapAdminPassword, //pragma: allowlist secret + "LDAP_USER_NAME": config.LdapUserName, + "LDAP_USER_PASSWORD": config.LdapUserPassword, //pragma: allowlist secret + "US_EAST_ZONE": config.USEastZone, + "US_EAST_RESERVATION_ID": config.USEastReservationID, + "US_EAST_CLUSTER_ID": config.USEastClusterID, + "EU_DE_ZONE": config.EUDEZone, + "EU_DE_RESERVATION_ID": config.EUDEReservationID, + "EU_DE_CLUSTER_ID": config.EUDEClusterID, + "US_SOUTH_ZONE": config.USSouthZone, + "US_SOUTH_RESERVATION_ID": config.USSouthReservationID, + "US_SOUTH_CLUSTER_ID": config.USSouthClusterID, + "SSH_FILE_PATH": config.SSHFilePath, + } + + for key, value := range envVars { + val, ok := os.LookupEnv(key) + switch { + case strings.Contains(key, "KEY_MANAGEMENT") && val == "null" && ok: + os.Setenv(key, "null") + case strings.Contains(key, "REMOTE_ALLOWED_IPS") && !ok && value == "": + os.Setenv(key, ip) + case value != "" && !ok: + switch v := value.(type) { + case string: + os.Setenv(key, v) + case bool: + os.Setenv(key, fmt.Sprintf("%t", v)) + case int: + os.Setenv(key, fmt.Sprintf("%d", v)) + default: + return fmt.Errorf("unsupported type for key %s", key) + } + } + } + + for key, value := range envVars { + _, ok := os.LookupEnv(key) + switch { + case key == "RESERVATION_ID" && !ok && value == "": + os.Setenv("RESERVATION_ID", GetValueForKey(map[string]string{"us-south": reservationIDSouth, "us-east": reservationIDEast}, strings.ToLower(GetRegion(os.Getenv("ZONE"))))) + case key == "US_EAST_RESERVATION_ID" && !ok && value == "": + os.Setenv("US_EAST_RESERVATION_ID", reservationIDEast) + case key == "EU_DE_RESERVATION_ID" && !ok && value == "": + os.Setenv("EU_DE_RESERVATION_ID", reservationIDEast) + case key == "US_SOUTH_RESERVATION_ID" && !ok && value == "": + os.Setenv("US_SOUTH_RESERVATION_ID", reservationIDSouth) + } + } + return nil +} diff --git a/tests/common_utils/log_utils.go b/tests/common_utils/log_utils.go new file mode 100644 index 00000000..bf1bffb8 --- /dev/null +++ b/tests/common_utils/log_utils.go @@ -0,0 +1,66 @@ +package tests + +import ( + "log" + "os" + "path/filepath" + "testing" + "time" + + "github.com/gruntwork-io/terratest/modules/logger" +) + +// AggregatedLogger represents an aggregated logger with different log levels. +type AggregatedLogger struct { + infoLogger *log.Logger + warnLogger *log.Logger + errorLogger *log.Logger +} + +// NewAggregatedLogger creates a new instance of AggregatedLogger. +func NewAggregatedLogger(logFileName string) (*AggregatedLogger, error) { + + absPath, err := filepath.Abs("test_output") + if err != nil { + return nil, err + } + + file, err := os.Create(filepath.Join(absPath, logFileName)) + if err != nil { + return nil, err + } + + return &AggregatedLogger{ + infoLogger: log.New(file, "", 0), + warnLogger: log.New(file, "", 0), + errorLogger: log.New(file, "", 0), + }, nil +} + +// getLogArgs is a helper function to generate common log arguments. +func getLogArgs(t *testing.T, message string) []interface{} { + return []interface{}{ + time.Now().Format("2006-01-02 15:04:05"), + t.Name(), + message, + } +} + +// Info logs informational messages. +func (l *AggregatedLogger) Info(t *testing.T, message string) { + format := "[%s] [INFO] [%s] : %v\n" + l.infoLogger.Printf(format, getLogArgs(t, message)...) +} + +// Warn logs warning messages. +func (l *AggregatedLogger) Warn(t *testing.T, message string) { + format := "[%s] [WARN] [%s] : %v\n" + l.warnLogger.Printf(format, getLogArgs(t, message)...) +} + +// Error logs error messages. +func (l *AggregatedLogger) Error(t *testing.T, message string) { + format := "[%s] [ERROR] [%s] : %v\n" + l.errorLogger.Printf(format, getLogArgs(t, message)...) + logger.Log(t, getLogArgs(t, message)...) +} diff --git a/tests/common_utils/ssh_utils.go b/tests/common_utils/ssh_utils.go new file mode 100644 index 00000000..b1adab23 --- /dev/null +++ b/tests/common_utils/ssh_utils.go @@ -0,0 +1,269 @@ +package tests + +import ( + "bytes" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/gruntwork-io/terratest/modules/retry" + "golang.org/x/crypto/ssh" +) + +// sshClientJumpHost establishes an SSH connection through a jump host. +// It takes two SSH client configurations (config and config1) for the jump host and the target machine, +// along with the IP and port of the jump host (jumpHostIPPort) and the target machine (machineIPPort). +// The function returns an SSH client connected to the target machine through the jump host. +// If any step in the process fails, an error is returned with a descriptive message. +func sshClientJumpHost(config, config1 *ssh.ClientConfig, publicHostIP, privateHostIP string) (*ssh.Client, error) { + client, err := ssh.Dial("tcp", publicHostIP, config) + if err != nil { + return nil, fmt.Errorf("failed to dial jump host: %w", err) + } + + conn, err := client.Dial("tcp", privateHostIP) + if err != nil { + return nil, fmt.Errorf("failed to dial target machine: %w", err) + } + + ncc, chans, reqs, err := ssh.NewClientConn(conn, privateHostIP, config1) + if err != nil { + return nil, fmt.Errorf("failed to create new client connection: %w", err) + } + + sClient := ssh.NewClient(ncc, chans, reqs) + return sClient, nil +} + +// runCommandInSSHSession executes a command in a new SSH session and returns the output. +// It takes an existing SSH client (sClient) and the command (cmd) to be executed. +// The function opens an SSH session, runs the specified command, captures the output, +// and returns the output as a string. If any step in the process fails, an error is returned with a descriptive message. +func RunCommandInSSHSession(sClient *ssh.Client, cmd string) (string, error) { + session, err := sClient.NewSession() + if err != nil { + return "", fmt.Errorf("failed to create SSH session: %w", err) + } + defer session.Close() + + var b bytes.Buffer + session.Stdout = &b + + if err := session.Run(cmd); err != nil { + return "", fmt.Errorf("failed to execute command '%s': %w", cmd, err) + } + + return b.String(), nil +} + +// getSshConfig retrieves SSH configuration variables. +// It takes an SSH private key (key) and a username (user). +// The function creates and returns an SSH client configuration (ClientConfig) +// with the specified user, ignoring host key verification, and using public key authentication. +func getSshConfig(key ssh.Signer, user string) *ssh.ClientConfig { + config := &ssh.ClientConfig{ + User: user, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(key), + }, + } + + return config +} + +// ConnectToHost establishes an SSH connection to a target machine. +// It takes the target machine's floating IP (floatingIP), IP and port (machineIPPort), +// and the login usernames for the jump host (loginUserName) and the target machine (vsiUserName). +// The function retrieves the SSH private key, creates SSH client configurations for the jump host and target machine, +// and establishes an SSH connection to the target machine through the jump host. +// If any step in the process fails, an error is returned with a descriptive message. +func ConnectToHost(publicHostName, publicHostIP, privateHostName, privateHostIP string) (*ssh.Client, error) { + // Get the SSH private key file path from the environment variable + sshFilePath := os.Getenv("SSH_FILE_PATH") + + // Check if the file exists + _, err := os.Stat(sshFilePath) + if os.IsNotExist(err) { + return nil, fmt.Errorf("SSH private key file '%s' does not exist", sshFilePath) + } else if err != nil { + return nil, fmt.Errorf("error checking SSH private key file: %v", err) + } + + key, err := getSshKeyFile(sshFilePath) + if err != nil { + return nil, fmt.Errorf("failed to get SSH key: %w", err) + } + + config := getSshConfig(key, publicHostName) + config1 := getSshConfig(key, privateHostName) + + sClient, err := sshClientJumpHost(config, config1, publicHostIP+":22", privateHostIP+":22") + if err != nil { + return nil, fmt.Errorf("unable to log in to the node: %w", err) + } + return sClient, nil +} + +// getSshKeyFile reads an SSH private key file. +// It takes the file path (filepath) of the SSH private key. +// The function reads the private key file, parses it, and returns an SSH signer. +// If any step in the process fails, an error is returned with a descriptive message. +func getSshKeyFile(filepath string) (key ssh.Signer, err error) { + privateKey, err := os.ReadFile(filepath) + if err != nil { + return nil, fmt.Errorf("failed to read private key file: %w", err) + } + + key, err = ssh.ParsePrivateKey(privateKey) + if err != nil { + return nil, fmt.Errorf("failed to parse private key: %w", err) + } + + return key, nil +} + +func ConnectionE(t *testing.T, publicHostName, publicHostIP, privateHostName, privateHostIP, command string) (string, error) { + sshFilePath := os.Getenv("SSH_FILE_PATH") + if _, err := os.Stat(sshFilePath); os.IsNotExist(err) { + return "", fmt.Errorf("SSH private key file '%s' does not exist", sshFilePath) + } else if err != nil { + return "", fmt.Errorf("error checking SSH private key file: %v", err) + } + + key, err := getSshKeyFile(sshFilePath) + if err != nil { + return "", fmt.Errorf("failed to get SSH key: %w", err) + } + + maxRetries := 1 + timeBetweenRetries := 10 * time.Second + description := "SSH into host" + var output string + var sClient *ssh.Client + + retry.DoWithRetry(t, description, maxRetries, timeBetweenRetries, func() (string, error) { + var config *ssh.ClientConfig + + if len(strings.TrimSpace(publicHostIP)) != 0 && len(strings.TrimSpace(publicHostName)) != 0 { + config = getSshConfig(key, publicHostName) + config1 := getSshConfig(key, privateHostName) + sClient, err = sshClientJumpHost(config, config1, publicHostIP+":22", privateHostIP+":22") + } else { + config = getSshConfig(key, privateHostName) + sClient, err = ssh.Dial("tcp", privateHostIP+":22", config) + } + + if err != nil { + return "", fmt.Errorf("unable to log in to the node: %w", err) + } + + output, err = RunCommandInSSHSession(sClient, command) + return output, err + }) + + return output, err +} + +// connectToHostsWithMultipleUsers establishes SSH connections to a host using multiple user credentials. +// It takes the public and private IP addresses and host names for two different users. +// Returns two SSH clients for the respective users, along with any errors encountered during the process. +func ConnectToHostsWithMultipleUsers(publicHostName, publicHostIP, privateHostName, privateHostIP string) (*ssh.Client, *ssh.Client, error, error) { + // Get the SSH private key file path for the first user from the environment variable + sshKeyFilePathUserOne := os.Getenv("SSHFILEPATH") + // Check if the file exists + if _, err := os.Stat(sshKeyFilePathUserOne); os.IsNotExist(err) { + return nil, nil, fmt.Errorf("SSH private key file '%s' does not exist", sshKeyFilePathUserOne), nil + } else if err != nil { + return nil, nil, fmt.Errorf("error checking SSH private key file: %v", err), nil + } + sshKeyUserOne, errUserOne := getSshKeyFile(sshKeyFilePathUserOne) + if errUserOne != nil { + return nil, nil, fmt.Errorf("failed to get SSH key for user one: %w", errUserOne), nil + } + + // Get the SSH private key file path for the second user from the environment variable + sshKeyFilePathUserTwo := os.Getenv("SSHFILEPATHTWO") + // Check if the file exists + if _, err := os.Stat(sshKeyFilePathUserTwo); os.IsNotExist(err) { + return nil, nil, nil, fmt.Errorf("SSH private key file '%s' does not exist", sshKeyFilePathUserTwo) + } else if err != nil { + return nil, nil, nil, fmt.Errorf("error checking SSH private key file: %v", err) + } + sshKeyUserTwo, errUserTwo := getSshKeyFile(sshKeyFilePathUserTwo) + if errUserTwo != nil { + return nil, nil, nil, fmt.Errorf("failed to get SSH key for user two: %w", errUserTwo) + } + + // Combine errors for better readability + var combinedErrUserOne error + if errUserOne != nil { + combinedErrUserOne = fmt.Errorf("user one SSH key error: %v", errUserOne) + } + var combinedErrUserTwo error + if errUserTwo != nil { + combinedErrUserTwo = fmt.Errorf("user two SSH key error: %v", errUserTwo) + } + + if combinedErrUserOne != nil && combinedErrUserTwo != nil { + return nil, nil, combinedErrUserOne, combinedErrUserTwo + } + + // Create SSH configurations for each user and host combination + sshConfigUserOnePrivate := getSshConfig(sshKeyUserOne, privateHostName) + sshConfigUserOnePublic := getSshConfig(sshKeyUserOne, publicHostName) + sshConfigUserTwoPrivate := getSshConfig(sshKeyUserTwo, privateHostName) + sshConfigUserTwoPublic := getSshConfig(sshKeyUserTwo, publicHostName) + + // Establish SSH connections for each user to the host + clientUserOne, errUserOne := sshClientJumpHost(sshConfigUserOnePrivate, sshConfigUserOnePublic, publicHostIP+":22", privateHostIP+":22") + clientUserTwo, errUserTwo := sshClientJumpHost(sshConfigUserTwoPrivate, sshConfigUserTwoPublic, publicHostIP+":22", privateHostIP+":22") + + // Combine errors for better readability + var combinedErrClientUserOne error + if errUserOne != nil { + combinedErrClientUserOne = fmt.Errorf("user one unable to log in to the node: %v", errUserOne) + } + var combinedErrClientUserTwo error + if errUserTwo != nil { + combinedErrClientUserTwo = fmt.Errorf("user two unable to log in to the node: %v", errUserTwo) + } + + return clientUserOne, clientUserTwo, combinedErrClientUserOne, combinedErrClientUserTwo +} + +func ConnectToHostAsLDAPUser(publicHostName, publicHostIP, privateHostIP, ldapUser, ldapPassword string) (*ssh.Client, error) { + + sshFilePath := os.Getenv("SSH_FILE_PATH") + + // Check if the file exists + _, err := os.Stat(sshFilePath) + if os.IsNotExist(err) { + return nil, fmt.Errorf("SSH private key file '%s' does not exist", sshFilePath) + } else if err != nil { + return nil, fmt.Errorf("error checking SSH private key file: %v", err) + } + + key, err := getSshKeyFile(sshFilePath) + if err != nil { + return nil, fmt.Errorf("failed to get SSH key: %w", err) + } + + config := getSshConfig(key, publicHostName) + + config1 := &ssh.ClientConfig{ + User: ldapUser, + Auth: []ssh.AuthMethod{ + ssh.Password(ldapPassword), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + + sClient, err := sshClientJumpHost(config, config1, publicHostIP+":22", privateHostIP+":22") + if err != nil { + return nil, fmt.Errorf("unable to log in to the node: %w", err) + } + return sClient, nil +} diff --git a/tests/common_utils/utils.go b/tests/common_utils/utils.go new file mode 100644 index 00000000..896124be --- /dev/null +++ b/tests/common_utils/utils.go @@ -0,0 +1,725 @@ +package tests + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "math/rand" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/IBM/go-sdk-core/core" + "github.com/IBM/secrets-manager-go-sdk/secretsmanagerv2" + "github.com/stretchr/testify/assert" + "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testhelper" + "golang.org/x/crypto/ssh" +) + +const ( + // TimeLayout is the layout string for date and time format (DDMonHHMMSS). + TimeLayout = "Jan02" +) + +// GetValueFromIniFile retrieves a value from an INI file based on the provided section and key. +// It reads the specified INI file, extracts the specified section, and returns the value associated with the key. +func GetValueFromIniFile(filePath, sectionName string) ([]string, error) { + // Read the content of the file + absolutePath, err := filepath.Abs(filePath) + if err != nil { + return nil, err + } + data, err := os.ReadFile(absolutePath) + if err != nil { + return nil, err + } + + // Convert the byte slice to a string + content := string(data) + + // Split the input into sections based on empty lines + sections := strings.Split(content, "\n\n") + + // Loop through sections and find the one with the specified sectionName + for _, section := range sections { + + if strings.Contains(section, "["+sectionName+"]") { + // Split the section into lines + lines := strings.Split(section, "\n") + + // Extract values + var sectionValues []string + for i := 1; i < len(lines); i++ { + // Skip the first line, as it contains the section name + sectionValues = append(sectionValues, strings.TrimSpace(lines[i])) + } + + return sectionValues, nil + } + } + + return nil, fmt.Errorf("section [%s] not found in file %s", sectionName, filePath) +} + +// ######### file operations functions ############## + +// ToCreateFile creates a file on the remote server using SSH. +// It takes an SSH client, the path where the file should be created, and the file name. +// Returns a boolean indicating success or failure and an error if any. +func ToCreateFile(t *testing.T, sClient *ssh.Client, filePath, fileName string, logger *AggregatedLogger) (bool, error) { + // Check if the specified path exists on the remote server + isPathExist, err := IsPathExist(t, sClient, filePath, logger) + if err != nil { + // Error occurred while checking path existence + return false, err + } + + if isPathExist { + // Path exists, create the file using SSH command + command := "cd " + filePath + " && touch " + fileName + _, createFileErr := RunCommandInSSHSession(sClient, command) + + if createFileErr == nil { + logger.Info(t, "File created successfully: "+fileName) + // File created successfully + return true, nil + } else { + // Error occurred while creating the file + return false, fmt.Errorf(" %s file not created", fileName) + } + } + + // Path does not exist + return false, fmt.Errorf("directory not exist : %s", filePath) +} + +// IsFileExist checks if a file exists on the remote server using SSH. +// It takes an SSH client, the path where the file should exist, and the file name. +// Returns a boolean indicating whether the file exists and an error if any. +func IsFileExist(t *testing.T, sClient *ssh.Client, filePath, fileName string, logger *AggregatedLogger) (bool, error) { + // Check if the specified path exists on the remote server + isPathExist, err := IsPathExist(t, sClient, filePath, logger) + if err != nil { + // Error occurred while checking path existence + return false, err + } + + if isPathExist { + // Path exists, check if the file exists using an SSH command + command := "[ -f " + filePath + fileName + " ] && echo 'File exist' || echo 'File not exist'" + isFileExist, err := RunCommandInSSHSession(sClient, command) + + if err != nil { + // Error occurred while running the command + return false, err + } + + if strings.Contains(isFileExist, "File exist") { + logger.Info(t, fmt.Sprintf("File exist : %s", fileName)) + // File exists + return true, nil + } else { + logger.Info(t, fmt.Sprintf("File not exist : %s", fileName)) + // File does not exist + return false, nil + } + } + + // Path does not exist + return false, fmt.Errorf("directory not exist : %s", filePath) +} + +// IsPathExist checks if a directory exists on the remote server using SSH. +// It takes an SSH client and the path to check for existence. +// Returns a boolean indicating whether the directory exists and an error if any. +func IsPathExist(t *testing.T, sClient *ssh.Client, filePath string, logger *AggregatedLogger) (bool, error) { + // Run an SSH command to test if the directory exists + command := "test -d " + filePath + " && echo 'Directory Exist' || echo 'Directory Not Exist'" + result, err := RunCommandInSSHSession(sClient, command) + + if err != nil { + // Error occurred while running the command + return false, err + } + + if strings.Contains(result, "Directory Exist") { + logger.Info(t, fmt.Sprintf("Directory exist : %s", filePath)) + // Directory exists + return true, nil + } else { + logger.Info(t, fmt.Sprintf("Directory not exist : %s", filePath)) + // Directory does not exist + return false, nil + } +} + +// GetDirList retrieves the list of files in a directory on a remote server via SSH. +// It takes an SSH client and the path of the directory. +// Returns a string containing the list of files and an error if any. +func GetDirList(t *testing.T, sClient *ssh.Client, filePath string, logger *AggregatedLogger) ([]string, error) { + command := "cd " + filePath + " && ls" + output, err := RunCommandInSSHSession(sClient, command) + if err == nil { + listDir := strings.Split(strings.TrimSpace(output), "\n") + logger.Info(t, fmt.Sprintf("Directory list : %q ", listDir)) + return listDir, nil + } + return nil, err +} + +// GetDirectoryFileList retrieves the list of files in a directory on a remote server via SSH. +// It takes an SSH client and the path of the directory. +// Returns a slice of strings representing the file names and an error if any. +func GetDirectoryFileList(t *testing.T, sClient *ssh.Client, directoryPath string, logger *AggregatedLogger) ([]string, error) { + command := "cd " + directoryPath + " && ls" + output, err := RunCommandInSSHSession(sClient, command) + if err == nil { + fileList := strings.Split(strings.TrimSpace(output), "\n") + logger.Info(t, fmt.Sprintf("Directory file list : %q", fileList)) + return fileList, nil + } + + return nil, err +} + +// ToDeleteFile deletes a file on the remote server using SSH. +// It takes an SSH client, the path of the file's directory, and the file name. +// Returns a boolean indicating whether the file was deleted successfully and an error if any. +func ToDeleteFile(t *testing.T, sClient *ssh.Client, filePath, fileName string, logger *AggregatedLogger) (bool, error) { + isPathExist, err := IsPathExist(t, sClient, filePath, logger) + if isPathExist { + command := "cd " + filePath + " && rm -rf " + fileName + _, deleFileErr := RunCommandInSSHSession(sClient, command) + if deleFileErr == nil { + logger.Info(t, fmt.Sprintf("File deleted successfully: %s", fileName)) + return true, nil + } + return false, fmt.Errorf("files not deleted: %s", fileName) + } + return isPathExist, err +} + +// ToCreateFileWithContent creates a file on the remote server using SSH. +// It takes an SSH client, the path where the file should be created, the file name, content to write to the file, +// a log file for logging, and returns a boolean indicating success or failure and an error if any. +func ToCreateFileWithContent(t *testing.T, sClient *ssh.Client, filePath, fileName, content string, logger *AggregatedLogger) (bool, error) { + // Check if the specified path exists on the remote server + isPathExist, err := IsPathExist(t, sClient, filePath, logger) + if err != nil { + // Error occurred while checking path existence + return false, fmt.Errorf("error checking path existence: %w", err) + } + + if isPathExist { + // Path exists, create the file using SSH command + command := "cd " + filePath + " && echo '" + content + "' > " + fileName + _, createFileErr := RunCommandInSSHSession(sClient, command) + + if createFileErr == nil { + // File created successfully + logger.Info(t, "File created successfully: "+fileName) + return true, nil + } + + // Error occurred while creating the file + return false, fmt.Errorf("error creating file %s: %w", fileName, createFileErr) + } + + // Path does not exist + return false, fmt.Errorf("directory not exist : %s", filePath) +} + +// ReadRemoteFileContents reads the content of a file on the remote server via SSH. +// It checks if the specified file path exists, creates an SSH command to concatenate the file contents, +// and returns the content as a string upon success. In case of errors, an empty string and an error are returned. +func ReadRemoteFileContents(t *testing.T, sClient *ssh.Client, filePath, fileName string, logger *AggregatedLogger) (string, error) { + isPathExist, err := IsPathExist(t, sClient, filePath, logger) + if err != nil { + return "", fmt.Errorf("error checking path existence: %w", err) + } + + if isPathExist { + command := "cd " + filePath + " && cat " + fileName + actualText, outErr := RunCommandInSSHSession(sClient, command) + + if outErr == nil { + logger.Info(t, "content: "+actualText) + return actualText, nil + } + + return "", fmt.Errorf("error reading file %s: %w", fileName, outErr) + } + + return "", fmt.Errorf("directory does not exist: %s", filePath) +} + +// VerifyDataContains is a generic function that checks if a value is present in data (string or string array) +// VerifyDataContains performs a verification operation on the provided data +// to determine if it contains the specified value. It supports string and +// string array types, logging results with the provided AggregatedLogger. +// Returns true if the value is found, false otherwise. +func VerifyDataContains(t *testing.T, data interface{}, val interface{}, logger *AggregatedLogger) bool { + //The data.(type) syntax is used to check the actual type of the data variable. + switch d := data.(type) { + case string: + //check if the val variable is of type string. + substr, ok := val.(string) + if !ok { + logger.Info(t, "Invalid type for val parameter") + return false + } + if substr != "" && strings.Contains(d, substr) { + logger.Info(t, fmt.Sprintf("The string '%s' contains the substring '%s'\n", d, substr)) + return true + } + logger.Info(t, fmt.Sprintf("The string '%s' does not contain the substring '%s'\n", d, substr)) + return false + + case []string: + switch v := val.(type) { + case string: + for _, arrVal := range d { + if arrVal == v { + logger.Info(t, fmt.Sprintf("The array '%q' contains the value: %s\n", d, v)) + return true + } + } + logger.Info(t, fmt.Sprintf("The array '%q' does not contain the value: %s\n", d, v)) + return false + + case []string: + if reflect.DeepEqual(d, v) { + logger.Info(t, fmt.Sprintf("The array '%q' contains the subarray '%q'\n", d, v)) + return true + } + logger.Info(t, fmt.Sprintf("The array '%q' does not contain the subarray '%q'\n", d, v)) + return false + + default: + logger.Info(t, "Invalid type for val parameter") + return false + } + + default: + logger.Info(t, "Unsupported type for data parameter") + return false + } +} + +func CountStringOccurrences(str string, substr string) int { + return strings.Count(str, substr) +} + +func SplitString(strValue string, splitCharacter string, indexValue int) string { + split := strings.Split(strValue, splitCharacter) + return split[indexValue] +} + +// StringToInt converts a string to an integer. +// Returns the converted integer and an error if the conversion fails. +func StringToInt(str string) (int, error) { + num, err := strconv.Atoi(str) + if err != nil { + return 0, err + } + return num, nil +} + +// RemoveNilValues removes nil value keys from the given map. +func RemoveNilValues(data map[string]interface{}) map[string]interface{} { + for key, value := range data { + if value == nil { + delete(data, key) + } + } + return data +} + +// LogVerificationResult logs the result of a verification check. +func LogVerificationResult(t *testing.T, err error, checkName string, logger *AggregatedLogger) { + if err == nil { + logger.Info(t, fmt.Sprintf("%s verification successful", checkName)) + } else { + assert.Nil(t, err, fmt.Sprintf("%s verification failed", checkName)) + logger.Error(t, fmt.Sprintf("%s verification failed: %s", checkName, err.Error())) + } +} + +// ParsePropertyValue parses the content of a string, searching for a property with the specified key. +// It returns the value of the property if found, or an empty string and an error if the property is not found. +func ParsePropertyValue(content, propertyKey string) (string, error) { + lines := strings.Split(content, "\n") + for _, line := range lines { + if strings.HasPrefix(line, propertyKey+"=") { + // Extract the value, removing quotes if present + value := strings.Trim(line[len(propertyKey)+1:], `"`) + return value, nil + } + } + + // Property not found + return "", fmt.Errorf("property '%s' not found in content:\n%s", propertyKey, content) +} + +// ParsePropertyValue parses the content of a string, searching for a property with the specified key. +// It returns the value of the property if found, or an empty string and an error if the property is not found. +func FindImageNamesByCriteria(name string) (string, error) { + // Get the absolute path of the image map file + absPath, err := filepath.Abs("modules/landing_zone_vsi/image_map.tf") + if err != nil { + return "", err + } + + absPath = strings.ReplaceAll(absPath, "tests", "") + readFile, err := os.Open(absPath) + if err != nil { + return "", err + } + defer readFile.Close() + + // Create a scanner to read lines from the file + fileScanner := bufio.NewScanner(readFile) + + var imageName string + + for fileScanner.Scan() { + line := fileScanner.Text() + if strings.Contains(strings.ToLower(line), strings.ToLower(name)) && strings.Contains(line, "compute") { + pattern := "[^a-zA-Z0-9\\s-]" + + // Compile the regular expression. + regex, err := regexp.Compile(pattern) + if err != nil { + return "", errors.New("error on image compiling regex") + } + // Use the regex to replace all matches with an empty string. + imageName = strings.TrimSpace(regex.ReplaceAllString(line, "")) + } + } + + // Check if any image names were found + if len(imageName) == 0 { + return imageName, errors.New("no image found with the specified criteria") + } + + return imageName, nil +} + +//Create IBMCloud Resources + +// LoginIntoIBMCloudUsingCLI logs into IBM Cloud using CLI. +// LoginIntoIBMCloudUsingCLI logs into IBM Cloud using CLI. +func LoginIntoIBMCloudUsingCLI(t *testing.T, apiKey, region, resourceGroup string) error { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + // Configure IBM Cloud CLI + configCmd := exec.CommandContext(ctx, "ibmcloud", "config", "--check-version=false") + configOutput, err := configCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to configure IBM Cloud CLI: %w. Output: %s", err, string(configOutput)) + } + + // Login to IBM Cloud and set the target resource group + loginCmd := exec.CommandContext(ctx, "ibmcloud", "login", "--apikey", apiKey, "-r", region, "-g", resourceGroup) + loginOutput, err := loginCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to login to IBM Cloud: %w. Output: %s", err, string(loginOutput)) + } + + return nil +} + +// Prerequisite: Ensure that you have logged into IBM Cloud using the LoginIntoIBMCloudUsingCLI function before calling this function +// CreateVPC creates a new Virtual Private Cloud (VPC) in IBM Cloud with the specified VPC name. +func CreateVPC(vpcName string) error { + existingVPC, err := IsVPCExist(vpcName) + if err != nil { + return err + } + if !existingVPC { + cmd := exec.Command("ibmcloud", "is", "vpc-create", vpcName) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("VPC creation failed: %v\nCommand output: %s", err, string(output)) + } + fmt.Printf("VPC %s created successfully\n", vpcName) + } else { + fmt.Println("An existing VPC is available") + } + return nil +} + +// Prerequisite: Ensure that you have logged into IBM Cloud using the LoginIntoIBMCloudUsingCLI function before calling this function +// IsVPCExist checks if a VPC with the given name exists. +func IsVPCExist(vpcName string) (bool, error) { + cmd := exec.Command("ibmcloud", "is", "vpcs", "--output", "json") + output, err := cmd.Output() + if err != nil { + return false, fmt.Errorf("failed to list VPCs: %w", err) + } + + return bytes.Contains(output, []byte(vpcName)), nil +} + +// GetRegion returns the region from a given zone. +func GetRegion(zone string) string { + // Extract region from zone + region := zone[:len(zone)-2] + return region +} + +// SplitAndTrim splits the input string into a list of trimmed strings based on the comma separator. +func SplitAndTrim(str, separator string) []string { + words := strings.Split(str, separator) + var trimmedWords []string + + for _, word := range words { + trimmedWord := strings.TrimSpace(word) + if trimmedWord != "" { + trimmedWords = append(trimmedWords, trimmedWord) + } + } + return trimmedWords +} + +// RemoveKeys removes the key-value pairs with the specified keys from the given map. +func RemoveKeys(m map[string]interface{}, keysToRemove []string) { + for _, key := range keysToRemove { + delete(m, key) + } +} + +// GetBastionServerIP retrieves the IP address from the BastionServer section in the specified INI file. +func GetBastionServerIP(t *testing.T, filePath string, logger *AggregatedLogger) (string, error) { + value, err := GetValueFromIniFile(filePath+"/bastion.ini", "BastionServer") + if err != nil { + return "", fmt.Errorf("failed to get value from bastion.ini: %w", err) + } + logger.Info(t, fmt.Sprintf("Bastion Server IP: %s", value[1])) + return value[1], nil +} + +// GetManagementNodeIPs retrieves the IP addresses from the HPCAASCluster section in the specified INI file. +func GetManagementNodeIPs(t *testing.T, filePath string, logger *AggregatedLogger) ([]string, error) { + value, err := GetValueFromIniFile(filePath+"/compute.ini", "HPCAASCluster") + if err != nil { + return nil, fmt.Errorf("failed to get value from compute.ini: %w", err) + } + logger.Info(t, fmt.Sprintf("Management Node IPs List: %q", value[1:])) + return value[1:], nil +} + +// GetLoginNodeIP retrieves the IP address from the LoginServer section in the specified login INI file. +func GetLoginNodeIP(t *testing.T, filePath string, logger *AggregatedLogger) (string, error) { + value, err := GetValueFromIniFile(filePath+"/login.ini", "LoginServer") + if err != nil { + return "", fmt.Errorf("failed to get value from login.ini: %w", err) + } + logger.Info(t, fmt.Sprintf("Login Server IP: %s", value[1])) + return value[1], nil +} + +// GetLdapServerIP retrieves the IP address from the LdapServer section in the specified login INI file. +func GetLdapServerIP(t *testing.T, filePath string, logger *AggregatedLogger) (string, error) { + value, err := GetValueFromIniFile(filePath+"/ldap.ini", "LDAPServer") + if err != nil { + return "", fmt.Errorf("failed to get value from ldap.ini: %w", err) + } + logger.Info(t, fmt.Sprintf("Ldap Server IP: %s", value[1])) + return value[1], nil +} + +// GetServerIPs retrieves the IP addresses of the bastion server, management nodes, and login node +// from the specified file path in the provided test options, using the provided logger for logging. +// It returns the bastion server IP, a list of management node IPs, the login node IP, and any error encountered. +func GetServerIPs(t *testing.T, options *testhelper.TestOptions, logger *AggregatedLogger) (bastionIP string, managementNodeIPList []string, loginNodeIP string, err error) { + + filePath := options.TerraformOptions.TerraformDir + + // Get bastion server IP and handle errors + bastionIP, err = GetBastionServerIP(t, filePath, logger) + if err != nil { + return "", nil, "", fmt.Errorf("error getting bastion server IP: %v", err) + } + + // Get management node IPs and handle errors + managementNodeIPList, err = GetManagementNodeIPs(t, filePath, logger) + if err != nil { + return "", nil, "", fmt.Errorf("error getting management node IPs: %v", err) + } + + // Get login node IP and handle errors + loginNodeIP, err = GetLoginNodeIP(t, filePath, logger) + if err != nil { + return "", nil, "", fmt.Errorf("error getting login node IP: %v", err) + } + + return bastionIP, managementNodeIPList, loginNodeIP, nil +} + +// GetServerIPsWithLDAP retrieves the IP addresses of various servers, including the LDAP server. +// from the specified file path in the provided test options, using the provided logger for logging. +// It returns the bastion server IP, a list of management node IPs, the login node IP, ldap server IP and any error encountered. +func GetServerIPsWithLDAP(t *testing.T, options *testhelper.TestOptions, logger *AggregatedLogger) (bastionIP string, managementNodeIPList []string, loginNodeIP, ldapIP string, err error) { + // Retrieve the Terraform directory from the options. + filePath := options.TerraformOptions.TerraformDir + + // Get the bastion server IP and handle errors. + bastionIP, err = GetBastionServerIP(t, filePath, logger) + if err != nil { + return "", nil, "", "", fmt.Errorf("error getting bastion server IP: %v", err) + } + + // Get the management node IPs and handle errors. + managementNodeIPList, err = GetManagementNodeIPs(t, filePath, logger) + if err != nil { + return "", nil, "", "", fmt.Errorf("error getting management node IPs: %v", err) + } + + // Get the login node IP and handle errors. + loginNodeIP, err = GetLoginNodeIP(t, filePath, logger) + if err != nil { + return "", nil, "", "", fmt.Errorf("error getting login node IP: %v", err) + } + + // Get the LDAP server IP and handle errors. + ldapIP, err = GetLdapServerIP(t, filePath, logger) + if err != nil { + return "", nil, "", "", fmt.Errorf("error getting LDAP server IP: %v", err) + } + + // Return the retrieved IP addresses and any error. + return bastionIP, managementNodeIPList, loginNodeIP, ldapIP, nil +} + +// GenerateTimestampedClusterPrefix generates a cluster prefix by appending a timestamp to the given prefix. +func GenerateTimestampedClusterPrefix(prefix string) string { + //Place current time in the string. + t := time.Now() + return strings.ToLower("cicd" + "-" + t.Format(TimeLayout) + "-" + prefix) + +} + +// GetPublicIP returns the public IP address using ifconfig.io API +func GetPublicIP() (string, error) { + cmd := exec.Command("bash", "-c", "(curl -s ifconfig.io)") + output, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + return strings.TrimSpace(string(output)), nil +} + +// GetOrDefault returns the environment variable value if it's not empty, otherwise returns the default value. +func GetOrDefault(envVar, defaultValue string) string { + if envVar != "" { + return envVar + } + return defaultValue +} + +// GenerateRandomString generates a random string of length 4 using lowercase characters +func GenerateRandomString() string { + // Define the character set containing lowercase letters + charset := "abcdefghijklmnopqrstuvwxyz" + + b := make([]byte, 4) + + // Loop through each index of the byte slice + for i := range b { + // Generate a random index within the length of the character set + randomIndex := rand.Intn(len(charset)) + + b[i] = charset[randomIndex] + } + + // Convert the byte slice to a string and return it + return string(b) +} + +// GetSecretsManagerKey retrieves a secret from IBM Secrets Manager. +func GetSecretsManagerKey(smID, smRegion, smKeyID string) (*string, error) { + secretsManagerService, err := secretsmanagerv2.NewSecretsManagerV2(&secretsmanagerv2.SecretsManagerV2Options{ + URL: fmt.Sprintf("https://%s.%s.secrets-manager.appdomain.cloud", smID, smRegion), + Authenticator: &core.IamAuthenticator{ + ApiKey: os.Getenv("TF_VAR_ibmcloud_api_key"), + }, + }) + if err != nil { + return nil, err + } + + getSecretOptions := secretsManagerService.NewGetSecretOptions(smKeyID) + + secret, _, err := secretsManagerService.GetSecret(getSecretOptions) + if err != nil { + return nil, err + } + + secretPayload, ok := secret.(*secretsmanagerv2.ArbitrarySecret) + if !ok { + return nil, fmt.Errorf("unexpected secret type: %T", secret) + } + + return secretPayload.Payload, nil +} + +// GetValueForKey retrieves the value associated with the specified key from the given map. +func GetValueForKey(inputMap map[string]string, key string) string { + return inputMap[key] +} + +// Getting BastionID and ComputeID from separately created brand new VPC +func GetSubnetIds(outputs map[string]interface{}) (string, string) { + subnetDetailsList := outputs["subnet_detail_list"] + var bastion []string + var compute []string + for _, val := range subnetDetailsList.(map[string]interface{}) { + for key1, val1 := range val.(map[string]interface{}) { + isContains := strings.Contains(key1, "bastion") + if isContains { + for key2, val2 := range val1.(map[string]interface{}) { + if key2 == "id" { + bastion = append(bastion, val2.(string)) + } + } + } else { + for key2, val2 := range val1.(map[string]interface{}) { + if key2 == "id" { + compute = append(compute, val2.(string)) + } + } + } + + } + } + bastionsubnetId := strings.Join(bastion, ",") + computesubnetIds := strings.Join(compute, ",") + return bastionsubnetId, computesubnetIds +} + +// Getting DNSInstanceID and CustomResolverID from separately created brand new VPC +func GetDnsCustomResolverIds(outputs map[string]interface{}) (string, string) { + customResolverHub := outputs["custom_resolver_hub"] + var instanceId string + var customResolverId string + for _, details := range customResolverHub.([]interface{}) { + for key, val := range details.(map[string]interface{}) { + if key == "instance_id" { + instanceId = val.(string) + } + if key == "custom_resolver_id" { + customResolverId = val.(string) + } + } + } + return instanceId, customResolverId +} diff --git a/tests/constants.go b/tests/constants.go new file mode 100644 index 00000000..c731c773 --- /dev/null +++ b/tests/constants.go @@ -0,0 +1,13 @@ +package tests + +const ( + IMAGE_NAME_PATH = "modules/landing_zone_vsi/image_map.tf" + HYPERTHREADTING_TRUE = "true" + HYPERTHREADTING_FALSE = "false" + LSF_DEFAULT_RESOURCE_GROUP = "Default" + LSF_CUSTOM_RESOURCE_GROUP_VALUE_AS_NULL = "null" + LSF_CUSTOM_RESOURCE_GROUP_OTHER_THAN_DEFAULT = "WES_TEST" + PRIVATE_KEY_SM_ID = "103a2267-c682-42e5-9393-4ed8f8e738c2" + PRIVATE_KEY_SM_REGION = "us-south" + PRIVATE_KEY_SECRET_ID_SOUTH = "48d56d5d-78fc-dc41-3089-5be6041bc00f" +) diff --git a/tests/go.mod b/tests/go.mod index 5306e419..5cf1f77e 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -1,82 +1,87 @@ module github.com/terraform-ibm-modules/terraform-ibm-hpc -go 1.21 - -toolchain go1.21.3 +go 1.21.3 require ( + github.com/IBM/go-sdk-core v1.1.0 + github.com/IBM/secrets-manager-go-sdk v1.2.0 + github.com/gruntwork-io/terratest v0.46.14 github.com/stretchr/testify v1.9.0 - github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper v1.30.7 + github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper v1.34.1 + golang.org/x/crypto v0.23.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( - cloud.google.com/go v0.110.4 // indirect - cloud.google.com/go/compute v1.21.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.1 // indirect - cloud.google.com/go/storage v1.30.1 // indirect + cloud.google.com/go v0.112.2 // indirect + cloud.google.com/go/auth v0.4.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/iam v1.1.8 // indirect + cloud.google.com/go/storage v1.40.0 // indirect dario.cat/mergo v1.0.0 // indirect - github.com/IBM-Cloud/bluemix-go v0.0.0-20240212062122-3386b538a495 // indirect + github.com/IBM-Cloud/bluemix-go v0.0.0-20240423071914-9e96525baef4 // indirect github.com/IBM-Cloud/power-go-client v1.6.0 // indirect - github.com/IBM/cloud-databases-go-sdk v0.6.0 // indirect - github.com/IBM/go-sdk-core/v5 v5.16.5 // indirect - github.com/IBM/platform-services-go-sdk v0.62.5 // indirect - github.com/IBM/vpc-go-sdk v1.0.2 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/IBM/cloud-databases-go-sdk v0.7.0 // indirect + github.com/IBM/go-sdk-core/v5 v5.17.2 // indirect + github.com/IBM/platform-services-go-sdk v0.62.11 // indirect + github.com/IBM/project-go-sdk v0.3.0 // indirect + github.com/IBM/vpc-go-sdk v0.50.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/agext/levenshtein v1.2.3 // indirect - github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/aws/aws-sdk-go v1.44.281 // indirect + github.com/aws/aws-sdk-go v1.52.4 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect - github.com/cloudflare/circl v1.3.7 // indirect - github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/cloudflare/circl v1.3.8 // indirect + github.com/cyphar/filepath-securejoin v0.2.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/emirpasic/gods v1.18.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-git/go-git/v5 v5.12.0 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/analysis v0.21.5 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect github.com/go-openapi/errors v0.22.0 // indirect - github.com/go-openapi/jsonpointer v0.20.1 // indirect - github.com/go-openapi/jsonreference v0.20.3 // indirect - github.com/go-openapi/loads v0.21.3 // indirect - github.com/go-openapi/runtime v0.26.0 // indirect - github.com/go-openapi/spec v0.20.12 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/loads v0.22.0 // indirect + github.com/go-openapi/runtime v0.28.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect github.com/go-openapi/strfmt v0.23.0 // indirect - github.com/go-openapi/swag v0.22.5 // indirect - github.com/go-openapi/validate v0.22.4 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/validate v0.24.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.19.0 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/s2a-go v0.1.4 // indirect + github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect - github.com/googleapis/gax-go/v2 v2.11.0 // indirect - github.com/gruntwork-io/terratest v0.46.13 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.4 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-getter v1.7.1 // indirect + github.com/hashicorp/go-getter v1.7.4 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/hashicorp/hcl/v2 v2.17.0 // indirect + github.com/hashicorp/hcl/v2 v2.20.1 // indirect github.com/hashicorp/terraform-json v0.21.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jinzhu/copier v0.4.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-zglob v0.0.4 // indirect @@ -90,32 +95,33 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect - github.com/tmccombs/hcl2json v0.5.0 // indirect - github.com/ulikunitz/xz v0.5.11 // indirect + github.com/tmccombs/hcl2json v0.6.3 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect - github.com/zclconf/go-cty v1.14.1 // indirect - go.mongodb.org/mongo-driver v1.14.0 // indirect + github.com/zclconf/go-cty v1.14.4 // indirect + go.mongodb.org/mongo-driver v1.15.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel v1.16.0 // indirect - go.opentelemetry.io/otel/metric v1.16.0 // indirect - go.opentelemetry.io/otel/trace v1.16.0 // indirect - golang.org/x/crypto v0.22.0 // indirect - golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/oauth2 v0.10.0 // indirect - golang.org/x/sync v0.5.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.16.1 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/api v0.127.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect - google.golang.org/grpc v1.58.3 // indirect - google.golang.org/protobuf v1.33.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.21.0 // indirect + google.golang.org/api v0.178.0 // indirect + google.golang.org/genproto v0.0.0-20240506185236-b8a5c65736ae // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240506185236-b8a5c65736ae // indirect + google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/go-playground/validator.v9 v9.31.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + ) diff --git a/tests/go.sum b/tests/go.sum index 86916404..b2530429 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -30,8 +30,8 @@ cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w9 cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= -cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= -cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= +cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbDms= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= @@ -46,6 +46,10 @@ cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjby cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/auth v0.4.0 h1:vcJWEguhY8KuiHoSs/udg1JtIRYm3YAWPBE1moF1m3U= +cloud.google.com/go/auth v0.4.0/go.mod h1:tO/chJN3obc5AbRYFQDsuFbL4wW5y8LfbPtDCfgwOVE= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -68,10 +72,8 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= -cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= @@ -109,8 +111,8 @@ cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y97 cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= -cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= @@ -171,8 +173,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= -cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= -cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= +cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= +cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= @@ -189,22 +191,27 @@ dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/IBM-Cloud/bluemix-go v0.0.0-20240212062122-3386b538a495 h1:lGKFLp41foq73oCSyr9MRELhwjNjF+XxNw7w7GdO5uk= -github.com/IBM-Cloud/bluemix-go v0.0.0-20240212062122-3386b538a495/go.mod h1:/7hMjdZA6fEpd/dQAOEABxKEwN0t72P3PlpEDu0Y7bE= +github.com/IBM-Cloud/bluemix-go v0.0.0-20240423071914-9e96525baef4 h1:43l8CU5cW4pOea10+jWtqRJj/4F4Ghfn6Oc82jB9RhM= +github.com/IBM-Cloud/bluemix-go v0.0.0-20240423071914-9e96525baef4/go.mod h1:/7hMjdZA6fEpd/dQAOEABxKEwN0t72P3PlpEDu0Y7bE= github.com/IBM-Cloud/power-go-client v1.6.0 h1:X+QX+WSF66+aouyaf4r+IeBLXUurAJj9+Bd+vH7G5I0= github.com/IBM-Cloud/power-go-client v1.6.0/go.mod h1:0ad5Lcq1utoYVJx0uqooMjCpUaYaK0ItP9QJYtY6k0Y= -github.com/IBM/cloud-databases-go-sdk v0.6.0 h1:QK3eif7+kusgeuMB54Zw5nco/kDwsDg2sD/84/foDxo= -github.com/IBM/cloud-databases-go-sdk v0.6.0/go.mod h1:nCIVfeZnhBYIiwByT959dFP4VWUeNLxomDYy63tTC6M= -github.com/IBM/go-sdk-core/v5 v5.9.2/go.mod h1:YlOwV9LeuclmT/qi/LAK2AsobbAP42veV0j68/rlZsE= -github.com/IBM/go-sdk-core/v5 v5.16.5 h1:5ZltNcryRI8kVcuvNJGR2EXKqb7HtM4atA0Nm5QwAFE= -github.com/IBM/go-sdk-core/v5 v5.16.5/go.mod h1:GatGZpxlo1KaxiRN6E10/rNgWtUtx1hN/GoHSCaSPKA= -github.com/IBM/platform-services-go-sdk v0.62.5 h1:Yi+eUny+IbNmzJNF1DxwC32sPuUphfenCzOh3y37eJ4= -github.com/IBM/platform-services-go-sdk v0.62.5/go.mod h1:M26dloj9C48k9AjfMcKGsgH/acEjaUvxjVS8z41Q8dg= -github.com/IBM/vpc-go-sdk v1.0.2 h1:WhI1Cb8atA8glUdFg0SEUh9u8afjnKHxZAj9onQBi04= -github.com/IBM/vpc-go-sdk v1.0.2/go.mod h1:42NO/XCXsyrYqpvtxoX5xwSEv/jBU1MKEoyaYkIUico= +github.com/IBM/cloud-databases-go-sdk v0.7.0 h1:prvLebKD1kcIk81D6yRhOr/TWp1VQJGLhGAasQr7RtA= +github.com/IBM/cloud-databases-go-sdk v0.7.0/go.mod h1:JYucI1PdwqbAd8XGdDAchxzxRP7bxOh1zUnseovHKsc= +github.com/IBM/go-sdk-core v1.1.0 h1:pV73lZqr9r1xKb3h08c1uNG3AphwoV5KzUzhS+pfEqY= +github.com/IBM/go-sdk-core v1.1.0/go.mod h1:2pcx9YWsIsZ3I7kH+1amiAkXvLTZtAq9kbxsfXilSoY= +github.com/IBM/go-sdk-core/v5 v5.17.2 h1:MyFCUPYqcNUQIx9d9srq9znMEZcvu6X3DOGIPjegP8o= +github.com/IBM/go-sdk-core/v5 v5.17.2/go.mod h1:GatGZpxlo1KaxiRN6E10/rNgWtUtx1hN/GoHSCaSPKA= +github.com/IBM/platform-services-go-sdk v0.62.11 h1:EGsiY90bM9M9sSdBVgpsX4QK1z99JZzedVDlrY2gzmc= +github.com/IBM/platform-services-go-sdk v0.62.11/go.mod h1:M26dloj9C48k9AjfMcKGsgH/acEjaUvxjVS8z41Q8dg= +github.com/IBM/project-go-sdk v0.3.0 h1:lZR4wT6UCsOZ8QkEBITrfM6OZkLlL70/HXiPxF/Olt4= +github.com/IBM/project-go-sdk v0.3.0/go.mod h1:FOJM9ihQV3EEAY6YigcWiTNfVCThtdY8bLC/nhQHFvo= +github.com/IBM/secrets-manager-go-sdk v1.2.0 h1:bgFfBF+LjHLtUfV3hTLkfgE8EjFsJaeU2icA2Hg+M50= +github.com/IBM/secrets-manager-go-sdk v1.2.0/go.mod h1:qv+tQg8Z3Vb11DQYxDjEGeROHDtTLQxUWuOIrIdWg6E= +github.com/IBM/vpc-go-sdk v0.50.0 h1:+vnXYK0FXFXYqaS/5/X1XEqH0bbRotkzkerRk21ZEjE= +github.com/IBM/vpc-go-sdk v0.50.0/go.mod h1:iBg9UJY1y/XpkweyP6YH7G6guzKPV8BYDoBMTdPupH4= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= @@ -215,18 +222,15 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuW github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= -github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= -github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.44.281 h1:z/ptheJvINaIAsKXthxONM+toTKw2pxyk700Hfm6yUw= -github.com/aws/aws-sdk-go v1.44.281/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.52.4 h1:9VsBVJ2TKf8xPP3+yIPGSYcEBIEymXsJzQoFgQuyvA0= +github.com/aws/aws-sdk-go v1.52.4/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= @@ -239,8 +243,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= -github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= -github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI= +github.com/cloudflare/circl v1.3.8/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -251,11 +255,13 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= -github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo= +github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -271,6 +277,8 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -295,45 +303,43 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/analysis v0.21.5 h1:3tHfEBh6Ia8eKc4M7khOGjPOAlWKJ10d877Cr9teujI= -github.com/go-openapi/analysis v0.21.5/go.mod h1:25YcZosX9Lwz2wBsrFrrsL8bmjjXdlyP6zsr2AMy29M= -github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= -github.com/go-openapi/jsonpointer v0.20.1 h1:MkK4VEIEZMj4wT9PmjaUmGflVBr9nvud4Q4UVFbDoBE= -github.com/go-openapi/jsonpointer v0.20.1/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= -github.com/go-openapi/jsonreference v0.20.3 h1:EjGcjTW8pD1mRis6+w/gmoBdqv5+RbE9B85D1NgDOVQ= -github.com/go-openapi/jsonreference v0.20.3/go.mod h1:FviDZ46i9ivh810gqzFLl5NttD5q3tSlMLqLr6okedM= -github.com/go-openapi/loads v0.21.3 h1:8sSH2FIm/SnbDUGv572md4YqVMFne/a9Eubvcd3anew= -github.com/go-openapi/loads v0.21.3/go.mod h1:Y3aMR24iHbKHppOj91nQ/SHc0cuPbAr4ndY4a02xydc= -github.com/go-openapi/runtime v0.26.0 h1:HYOFtG00FM1UvqrcxbEJg/SwvDRvYLQKGhw2zaQjTcc= -github.com/go-openapi/runtime v0.26.0/go.mod h1:QgRGeZwrUcSHdeh4Ka9Glvo0ug1LC5WyE+EV88plZrQ= -github.com/go-openapi/spec v0.20.12 h1:cgSLbrsmziAP2iais+Vz7kSazwZ8rsUZd6TUzdDgkVI= -github.com/go-openapi/spec v0.20.12/go.mod h1:iSCgnBcwbMW9SfzJb8iYynXvcY6C/QFrI7otzF7xGM4= -github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/runtime v0.28.0 h1:gpPPmWSNGo214l6n8hzdXYhPuJcGtziTOgUpvsFWGIQ= +github.com/go-openapi/runtime v0.28.0/go.mod h1:QN7OzcS+XuYmkQLw05akXk0jRH/eZ3kb18+1KwW9gyc= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= github.com/go-openapi/strfmt v0.21.7/go.mod h1:adeGTkxE44sPyLk0JV235VQAO/ZXUr8KAzYjclFs3ew= github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= -github.com/go-openapi/swag v0.22.5 h1:fVS63IE3M0lsuWRzuom3RLwUMVI2peDH01s6M70ugys= -github.com/go-openapi/swag v0.22.5/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0= -github.com/go-openapi/validate v0.22.4 h1:5v3jmMyIPKTR8Lv9syBAIRxG6lY0RqeBPB1LKEijzk8= -github.com/go-openapi/validate v0.22.4/go.mod h1:qm6O8ZIcPVdSY5219468Jv7kBdGvkiZLPOmqnqTUZ2A= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= -github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= @@ -370,8 +376,9 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -416,9 +423,8 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -426,8 +432,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= -github.com/googleapis/enterprise-certificate-proxy v0.2.4 h1:uGy6JWR/uMIILU8wbf+OkstIrNiMjGpEIyhx8f6W7s4= -github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -437,25 +443,23 @@ github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99 github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= -github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= -github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= +github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/gruntwork-io/terratest v0.46.13 h1:FDaEoZ7DtkomV8pcwLdBV/VsytdjnPRqJkIriYEYwjs= -github.com/gruntwork-io/terratest v0.46.13/go.mod h1:8sxu3Qup8TxtbzOHzq0MUrQffJj/G61/OwlsReaCwpo= +github.com/gruntwork-io/terratest v0.46.14 h1:nVT2JpOPLr7KbwOSNDP0GJffljH+Yu5833cwLorxRjs= +github.com/gruntwork-io/terratest v0.46.14/go.mod h1:L/IHbj195wnjfIFpZYWUhjwA3jm4O6ehO//xz7NxN8o= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.7.1 h1:SWiSWN/42qdpR0MdhaOc/bLR48PLuP1ZQtYLRlM69uY= -github.com/hashicorp/go-getter v1.7.1/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/hashicorp/go-getter v1.7.4 h1:3yQjWuxICvSpYwqSayAdKRFcvBl1y/vogCxczWSmix0= +github.com/hashicorp/go-getter v1.7.4/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= @@ -464,8 +468,8 @@ github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mO github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl/v2 v2.17.0 h1:z1XvSUyXd1HP10U4lrLg5e0JMVz6CPaJvAgxM0KNZVY= -github.com/hashicorp/hcl/v2 v2.17.0/go.mod h1:gJyW2PTShkJqQBKpAmPO3yxMxIuoXkOF2TpqXzrQyx4= +github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= +github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= github.com/hashicorp/terraform-json v0.21.0 h1:9NQxbLNqPbEMze+S6+YluEdXgJmhQykRyRNd+zTI05U= github.com/hashicorp/terraform-json v0.21.0/go.mod h1:qdeBs11ovMzo5puhrRibdD6d2Dq6TyE/28JiU4tIQxk= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -488,8 +492,8 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -497,9 +501,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -515,7 +517,6 @@ github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJ github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= @@ -527,7 +528,6 @@ github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= @@ -549,7 +549,6 @@ github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xl github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= @@ -604,20 +603,18 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper v1.30.7 h1:XCJe1LXR3a8CiL4R4F+kk/yKZ6nB+cSHuNPQkgrcjXU= -github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper v1.30.7/go.mod h1:amS0rmYNR4VsnhLxvBVqPJ8ZWIp6GaGy3UoaaHrf9vk= +github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper v1.34.1 h1:hw5Pivm1fbjpbloEjRcM68178DdAGguwJCF6ZD8UhWs= +github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper v1.34.1/go.mod h1:2fA+HDax1ZxE6o9qR1331MT67nRC/WVYgxpVLyB/ovw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tmccombs/hcl2json v0.5.0 h1:cT2sXStOzKL06c8ZTf9vh+0N8GKGzV7+9RUaY5/iUP8= -github.com/tmccombs/hcl2json v0.5.0/go.mod h1:B0ZpBthAKbQur6yZRKrtaqDmYLCvgnwHOBApE0faCpU= +github.com/tmccombs/hcl2json v0.6.3 h1:yfZO7FYuWxSBAkxN1Dw+O9bjnK12vdwCDtSJDzw7haw= +github.com/tmccombs/hcl2json v0.6.3/go.mod h1:VaIUbPyWiGThEKOsVZis0QHfMCnHLqD3IEbggSvQ8eY= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= -github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -627,12 +624,13 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zclconf/go-cty v1.14.1 h1:t9fyA35fwjjUMcmL5hLER+e/rEPqrbCK1/OSE4SI9KA= -github.com/zclconf/go-cty v1.14.1/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= -go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= +github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8= +github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI= +github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= -go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= +go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -642,23 +640,25 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= -go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= -go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= -go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= -go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= -go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= -go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= -go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 h1:A3SayB3rNyt+1S6qpI9mHPkeHTZbD7XILEqWnYZb2l0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0/go.mod h1:27iA5uvhuRNmalO+iEUdVn5ZMj2qy10Mm+XRIpRmyuU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= @@ -668,8 +668,8 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -713,8 +713,9 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -746,7 +747,6 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -780,8 +780,8 @@ golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -807,8 +807,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -826,8 +826,9 @@ golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -856,7 +857,6 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -915,8 +915,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -931,8 +931,8 @@ golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -942,7 +942,6 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -952,11 +951,14 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -966,7 +968,6 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -1020,8 +1021,9 @@ golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1029,8 +1031,9 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1079,15 +1082,14 @@ google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.127.0 h1:v7rj0vA0imM3Ou81k1eyFxQNScLzn71EyGnJDr+V/XI= -google.golang.org/api v0.127.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= +google.golang.org/api v0.178.0 h1:yoW/QMI4bRVCHF+NWOTa4cL8MoWL3Jnuc7FlcFF91Ok= +google.golang.org/api v0.178.0/go.mod h1:84/k2v8DFpDRebpGcooklv/lais3MEfqpaBLA12gl2U= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1190,12 +1192,12 @@ google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqw google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= -google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= -google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= -google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/genproto v0.0.0-20240506185236-b8a5c65736ae h1:HjgkYCl6cWQEKSHkpUp4Q8VB74swzyBwTz1wtTzahm0= +google.golang.org/genproto v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:i4np6Wrjp8EujFAUn0CM0SH+iZhY1EbrfzEIJbFkHFM= +google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae h1:AH34z6WAGVNkllnKs5raNq3yRq93VnjBG6rpfub/jYk= +google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:FfiGhwUm6CJviekPrc0oJ+7h29e+DmWU6UtjX0ZvI7Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240506185236-b8a5c65736ae h1:c55+MER4zkBS14uJhSZMGGmya0yJx5iHV4x/fpOSNRk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1231,8 +1233,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1249,8 +1251,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1260,7 +1262,10 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v9 v9.30.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -1274,7 +1279,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/tests/lsf/lsf_cluster_test_utils.go b/tests/lsf/lsf_cluster_test_utils.go new file mode 100644 index 00000000..c766ff31 --- /dev/null +++ b/tests/lsf/lsf_cluster_test_utils.go @@ -0,0 +1,499 @@ +package tests + +import ( + "errors" + "fmt" + "strings" + "testing" + + utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/common_utils" + "golang.org/x/crypto/ssh" +) + +// VerifyManagementNodeConfig verifies the configuration of a management node by performing various checks. +// It checks the cluster ID, master name, Reservation ID, MTU, IP route, hyperthreading, LSF version, Run tasks and file mount. +// The results of the checks are logged using the provided logger. +func VerifyManagementNodeConfig( + t *testing.T, + sshMgmtClient *ssh.Client, + expectedClusterID, expectedMasterName, expectedReservationID string, + expectedHyperthreadingStatus bool, + managementNodeIPList []string, + jobCommand string, + lsfVersion string, + logger *utils.AggregatedLogger, +) { + + // Verify cluster ID + checkClusterIDErr := LSFCheckClusterID(t, sshMgmtClient, expectedClusterID, logger) + utils.LogVerificationResult(t, checkClusterIDErr, "check Cluster ID on management node", logger) + + // Verify master name + checkMasterNameErr := LSFCheckMasterName(t, sshMgmtClient, expectedMasterName, logger) + utils.LogVerificationResult(t, checkMasterNameErr, "check Master name on management node", logger) + + // Verify Reservation ID + ReservationIDErr := HPCCheckReservationID(t, sshMgmtClient, expectedReservationID, logger) + utils.LogVerificationResult(t, ReservationIDErr, "check Reservation ID on management node", logger) + + // MTU check for management nodes + mtuCheckErr := LSFMTUCheck(t, sshMgmtClient, managementNodeIPList, logger) + utils.LogVerificationResult(t, mtuCheckErr, "MTU check on management node", logger) + + // IP route check for management nodes + ipRouteCheckErr := LSFIPRouteCheck(t, sshMgmtClient, managementNodeIPList, logger) + utils.LogVerificationResult(t, ipRouteCheckErr, "IP route check on management node", logger) + + // Hyperthreading check + hyperthreadErr := LSFCheckHyperthreading(t, sshMgmtClient, expectedHyperthreadingStatus, logger) + utils.LogVerificationResult(t, hyperthreadErr, "Hyperthreading check on management node", logger) + + // LSF version check + versionErr := CheckLSFVersion(t, sshMgmtClient, lsfVersion, logger) + utils.LogVerificationResult(t, versionErr, "check LSF version on management node", logger) + + //File Mount + fileMountErr := HPCCheckFileMount(t, sshMgmtClient, managementNodeIPList, "management", logger) + utils.LogVerificationResult(t, fileMountErr, "File mount check on management node", logger) + + //Run job + jobErr := LSFRunJobs(t, sshMgmtClient, jobCommand, logger) + utils.LogVerificationResult(t, jobErr, "check Run job on management node", logger) +} + +// VerifySSHKey verifies SSH keys for both management and compute nodes. +// It checks the SSH keys for a specified node type (management or compute) on a list of nodes. +// The function fails if the node list is empty, or if an invalid value (other than 'management' or 'compute') is provided for the node type. +// The verification results are logged using the provided logger. +func VerifySSHKey(t *testing.T, sshMgmtClient *ssh.Client, publicHostIP, publicHostName, privateHostName string, nodeType string, nodeList []string, logger *utils.AggregatedLogger) { + + // Check if the node list is empty + if len(nodeList) == 0 { + errorMsg := fmt.Sprintf("%s node IPs cannot be empty", nodeType) + utils.LogVerificationResult(t, fmt.Errorf(errorMsg), fmt.Sprintf("%s node SSH check", nodeType), logger) + return + } + + // Convert nodeType to lowercase for consistency + nodeType = strings.ToLower(nodeType) + + var sshKeyCheckErr error + switch nodeType { + case "management": + sshKeyCheckErr = LSFCheckSSHKeyForManagementNodes(t, publicHostName, publicHostIP, privateHostName, nodeList, logger) + case "compute": + sshKeyCheckErr = LSFCheckSSHKeyForComputeNodes(t, sshMgmtClient, nodeList, logger) + default: + // Log an error if the node type is unknown + errorMsg := fmt.Sprintf("unknown node type for SSH key verification: %s", nodeType) + utils.LogVerificationResult(t, fmt.Errorf(errorMsg), fmt.Sprintf("%s node SSH check", nodeType), logger) + return + } + // Log the result of the SSH key check + utils.LogVerificationResult(t, sshKeyCheckErr, fmt.Sprintf("%s node SSH check", nodeType), logger) +} + +// FailoverAndFailback performs a failover and failback procedure for a cluster using LSF (Load Sharing Facility). +// It stops the bctrl daemon, runs jobs, and starts the bctrl daemon. +// It logs verification results using the provided logger. +func FailoverAndFailback(t *testing.T, sshMgmtClient *ssh.Client, jobCommand string, logger *utils.AggregatedLogger) { + + //Stop sbatchd + stopDaemonsErr := LSFControlBctrld(t, sshMgmtClient, "stop", logger) + utils.LogVerificationResult(t, stopDaemonsErr, "check bctrl stop on management node", logger) + + //Run job + jobErr := LSFRunJobs(t, sshMgmtClient, jobCommand, logger) + utils.LogVerificationResult(t, jobErr, "check Run job on management node", logger) + + //Start sbatchd + startDaemonsErr := LSFControlBctrld(t, sshMgmtClient, "start", logger) + utils.LogVerificationResult(t, startDaemonsErr, "check bctrl start on management node", logger) +} + +// RestartLsfDaemon restarts the LSF (Load Sharing Facility) daemons. +// It logs verification results using the provided logger. +func RestartLsfDaemon(t *testing.T, sshMgmtClient *ssh.Client, jobCommand string, logger *utils.AggregatedLogger) { + + //Restart Daemons + restartDaemonErr := LSFRestartDaemons(t, sshMgmtClient, logger) + utils.LogVerificationResult(t, restartDaemonErr, "check lsf_daemons restart", logger) + + //Run job + jobErr := LSFRunJobs(t, sshMgmtClient, jobCommand, logger) + utils.LogVerificationResult(t, jobErr, "check Run job", logger) +} + +// RebootInstance reboots an instance in a cluster using LSF (Load Sharing Facility). +// It performs instance reboot, establishes a new SSH connection to the master node, +// checks the bhosts response, and logs verification results using the provided logger. +func RebootInstance(t *testing.T, sshMgmtClient *ssh.Client, publicHostIP, publicHostName, privateHostName, managementNodeIP string, jobCommand string, logger *utils.AggregatedLogger) { + + //Reboot the management node one + rebootErr := LSFRebootInstance(t, sshMgmtClient, logger) + utils.LogVerificationResult(t, rebootErr, "instance reboot", logger) + + // Connect to the master node via SSH and handle connection errors + sshClient, connectionErr := utils.ConnectToHost(publicHostName, publicHostIP, privateHostName, managementNodeIP) + utils.LogVerificationResult(t, connectionErr, "SSH connection to the master", logger) + + //Run bhost command + bhostRespErr := LSFCheckBhostsResponse(t, sshClient, logger) + utils.LogVerificationResult(t, bhostRespErr, "bhosts response non-empty", logger) + + //Run job + jobErr := LSFRunJobs(t, sshClient, jobCommand, logger) + utils.LogVerificationResult(t, jobErr, "check Run job", logger) + + defer sshClient.Close() + +} + +// VerifyComputetNodeConfig verifies the configuration of compute nodes by performing various checks +// It checks the cluster ID,such as MTU, IP route, hyperthreading, file mount, and Intel One MPI. +// The results of the checks are logged using the provided logger. +// NOTE : Compute Node nothing but worker node +func VerifyComputetNodeConfig( + t *testing.T, + sshMgmtClient *ssh.Client, + expectedHyperthreadingStatus bool, + computeNodeIPList []string, + logger *utils.AggregatedLogger, +) { + + // MTU check for management nodes + mtuCheckErr := LSFMTUCheck(t, sshMgmtClient, computeNodeIPList, logger) + utils.LogVerificationResult(t, mtuCheckErr, "MTU check on compute node", logger) + + // IP route check for management nodes + ipRouteCheckErr := LSFIPRouteCheck(t, sshMgmtClient, computeNodeIPList, logger) + utils.LogVerificationResult(t, ipRouteCheckErr, "IP route check on compute node", logger) + + // Hyperthreading check + hyperthreadErr := LSFCheckHyperthreading(t, sshMgmtClient, expectedHyperthreadingStatus, logger) + utils.LogVerificationResult(t, hyperthreadErr, "Hyperthreading check on compute node", logger) + + // File mount + fileMountErr := HPCCheckFileMount(t, sshMgmtClient, computeNodeIPList, "compute", logger) + utils.LogVerificationResult(t, fileMountErr, "File mount check on compute node", logger) + + // Intel One mpi + intelOneMpiErr := LSFCheckIntelOneMpiOnComputeNodes(t, sshMgmtClient, computeNodeIPList, logger) + utils.LogVerificationResult(t, intelOneMpiErr, "Intel One Mpi check on compute node", logger) + +} + +// VerifyAPPCenterConfig verifies the configuration of the application center by performing various checks. +func VerifyAPPCenterConfig( + t *testing.T, + sshMgmtClient *ssh.Client, + logger *utils.AggregatedLogger, +) { + + // Verify application center + appCenterErr := LSFAPPCenterConfiguration(t, sshMgmtClient, logger) + utils.LogVerificationResult(t, appCenterErr, "check Application center", logger) + +} + +// VerifyLoginNodeConfig verifies the configuration of a login node by performing various checks. +// It checks the cluster ID, master name, Reservation ID, MTU, IP route, hyperthreading, LSF version, Run tasks and file mount. +// The results of the checks are logged using the provided logger. +func VerifyLoginNodeConfig( + t *testing.T, + sshLoginClient *ssh.Client, + expectedClusterID, expectedMasterName, expectedReservationID string, + expectedHyperthreadingStatus bool, + loginNodeIP string, + jobCommand string, + lsfVersion string, + logger *utils.AggregatedLogger, +) { + + // Verify cluster ID + checkClusterIDErr := LSFCheckClusterID(t, sshLoginClient, expectedClusterID, logger) + utils.LogVerificationResult(t, checkClusterIDErr, "check Cluster ID on login node", logger) + + // Verify master name + checkMasterNameErr := LSFCheckMasterName(t, sshLoginClient, expectedMasterName, logger) + utils.LogVerificationResult(t, checkMasterNameErr, "check Master name on login node", logger) + + // Verify Reservation ID + ReservationIDErr := HPCCheckReservationID(t, sshLoginClient, expectedReservationID, logger) + utils.LogVerificationResult(t, ReservationIDErr, "check Reservation ID on login node", logger) + + // MTU check for login nodes + mtuCheckErr := LSFMTUCheck(t, sshLoginClient, []string{loginNodeIP}, logger) + utils.LogVerificationResult(t, mtuCheckErr, "MTU check on login node", logger) + + // IP route check for login nodes + ipRouteCheckErr := LSFIPRouteCheck(t, sshLoginClient, []string{loginNodeIP}, logger) + utils.LogVerificationResult(t, ipRouteCheckErr, "IP route check on login node", logger) + + // Hyperthreading check + hyperthreadErr := LSFCheckHyperthreading(t, sshLoginClient, expectedHyperthreadingStatus, logger) + utils.LogVerificationResult(t, hyperthreadErr, "Hyperthreading check on login node", logger) + + // LSF version check + versionErr := CheckLSFVersion(t, sshLoginClient, lsfVersion, logger) + utils.LogVerificationResult(t, versionErr, "check LSF version", logger) + + //File Mount + fileMountErr := HPCCheckFileMount(t, sshLoginClient, []string{loginNodeIP}, "login", logger) + utils.LogVerificationResult(t, fileMountErr, "File mount check on login node", logger) + + //Run job + jobErr := LSFRunJobs(t, sshLoginClient, jobCommand, logger) + utils.LogVerificationResult(t, jobErr, "check Run job on login node", logger) +} + +// VerifyTestTerraformOutputs is a function that verifies the Terraform outputs for a test scenario. +func VerifyTestTerraformOutputs( + t *testing.T, + LastTestTerraformOutputs map[string]interface{}, + isAPPCenterEnabled bool, + ldapServerEnabled bool, + logger *utils.AggregatedLogger, +) { + + // Check the Terraform logger outputs + outputErr := VerifyTerraformOutputs(t, LastTestTerraformOutputs, isAPPCenterEnabled, ldapServerEnabled, logger) + utils.LogVerificationResult(t, outputErr, "check terraform outputs", logger) + +} + +// VerifySSHConnectivityToNodesFromLogin is a function that verifies SSH connectivity from a login node to other nodes. +func VerifySSHConnectivityToNodesFromLogin( + t *testing.T, + sshLoginClient *ssh.Client, + managementNodeIPList []string, + computeNodeIPList []string, + logger *utils.AggregatedLogger, +) { + + // ssh into management node and compute node from a login node + sshConnectivityErr := LSFCheckSSHConnectivityToNodesFromLogin(t, sshLoginClient, managementNodeIPList, computeNodeIPList, logger) + utils.LogVerificationResult(t, sshConnectivityErr, "check SSH connectivity from the login node to other nodes", logger) +} + +// VerifyNoVNCConfig verifies the noVNC configuration by performing various checks. +func VerifyNoVNCConfig( + t *testing.T, + sshMgmtClient *ssh.Client, + logger *utils.AggregatedLogger, +) { + + // Verify noVNC center + appCenterErr := HPCCheckNoVNC(t, sshMgmtClient, logger) + utils.LogVerificationResult(t, appCenterErr, "check noVnc", logger) + +} + +// VerifyJobs verifies LSF job execution, logging any errors. +func VerifyJobs(t *testing.T, sshClient *ssh.Client, jobCommand string, logger *utils.AggregatedLogger) { + jobErr := LSFRunJobs(t, sshClient, jobCommand, logger) + utils.LogVerificationResult(t, jobErr, "check Run job", logger) + +} + +// VerifyFileShareEncryption verifies encryption settings for file shares. +func VerifyFileShareEncryption(t *testing.T, apiKey, region, resourceGroup, clusterPrefix, keyManagement string, logger *utils.AggregatedLogger) { + // Validate encryption + encryptErr := VerifyEncryption(t, apiKey, region, resourceGroup, clusterPrefix, keyManagement, logger) + utils.LogVerificationResult(t, encryptErr, "encryption", logger) +} + +// VerifyManagementNodeLDAPConfig verifies the configuration of a management node by performing various checks. +// It checks LDAP configuration, LSF commands, Run tasks, file mount, and SSH into all management nodes as an LDAP user. +// The results of the checks are logged using the provided logger. +func VerifyManagementNodeLDAPConfig( + t *testing.T, + sshMgmtClient *ssh.Client, + bastionIP string, + ldapServerIP string, + managementNodeIPList []string, + jobCommand string, + ldapDomainName string, + ldapUserName string, + ldapPassword string, + logger *utils.AggregatedLogger, +) { + // Verify LDAP configuration + ldapErr := VerifyLDAPConfig(t, sshMgmtClient, "management", ldapServerIP, ldapDomainName, ldapUserName, logger) + if ldapErr != nil { + utils.LogVerificationResult(t, ldapErr, "ldap configuration verification failed", logger) + return + } + + // Connect to the master node via SSH and handle connection errors + sshLdapClient, connectionErr := utils.ConnectToHostAsLDAPUser(LSF_PUBLIC_HOST_NAME, bastionIP, managementNodeIPList[0], ldapUserName, ldapPassword) + if connectionErr != nil { + utils.LogVerificationResult(t, connectionErr, "connect to the management node via SSH as LDAP User failed", logger) + return + } + defer sshLdapClient.Close() + + // Check file mount + fileMountErr := HPCCheckFileMountAsLDAPUser(t, sshLdapClient, "management", logger) + utils.LogVerificationResult(t, fileMountErr, "check file mount as an LDAP user on the management node", logger) + + // Run job + jobErr := LSFRunJobsAsLDAPUser(t, sshLdapClient, jobCommand, ldapUserName, logger) + utils.LogVerificationResult(t, jobErr, "check Run job as an LDAP user on the management node", logger) + + // Verify LSF commands on management node as LDAP user + lsfCmdErr := VerifyLSFCommands(t, sshLdapClient, ldapUserName, logger) + utils.LogVerificationResult(t, lsfCmdErr, "Check the 'lsf' command as an LDAP user on the management node", logger) + + // Loop through management node IPs and perform checks + for i := 0; i < len(managementNodeIPList); i++ { + sshLdapClientUser, connectionErr := utils.ConnectToHostAsLDAPUser(LSF_PUBLIC_HOST_NAME, bastionIP, managementNodeIPList[i], ldapUserName, ldapPassword) + if connectionErr == nil { + logger.Info(t, fmt.Sprintf("connect to the management node %s via SSH as LDAP User", managementNodeIPList[i])) + } + utils.LogVerificationResult(t, connectionErr, "connect to the management node via SSH as LDAP User", logger) + defer sshLdapClientUser.Close() + } +} + +// VerifyLoginNodeLDAPConfig verifies the configuration of a login node by performing various checks. +// It checks LDAP configuration, LSF commands, Run tasks, and file mount. +// The results of the checks are logged using the provided logger. +func VerifyLoginNodeLDAPConfig( + t *testing.T, + sshLoginClient *ssh.Client, + bastionIP string, + loginNodeIP string, + ldapServerIP string, + jobCommand string, + ldapDomainName string, + ldapUserName string, + ldapPassword string, + logger *utils.AggregatedLogger, +) { + // Verify LDAP configuration + ldapErr := VerifyLDAPConfig(t, sshLoginClient, "login", ldapServerIP, ldapDomainName, ldapUserName, logger) + if ldapErr != nil { + utils.LogVerificationResult(t, ldapErr, "ldap configuration verification failed", logger) + return + } + + // Connect to the login node via SSH and handle connection errors + sshLdapClient, connectionErr := utils.ConnectToHostAsLDAPUser(LSF_PUBLIC_HOST_NAME, bastionIP, loginNodeIP, ldapUserName, ldapPassword) + if connectionErr != nil { + utils.LogVerificationResult(t, connectionErr, "connect to the login node via SSH as LDAP User failed", logger) + return + } + defer sshLdapClient.Close() + + // Check file mount + fileMountErr := HPCCheckFileMountAsLDAPUser(t, sshLdapClient, "login", logger) + utils.LogVerificationResult(t, fileMountErr, "check file mount as an LDAP user on the login node", logger) + + // Run job + jobErr := LSFRunJobsAsLDAPUser(t, sshLdapClient, jobCommand, ldapUserName, logger) + utils.LogVerificationResult(t, jobErr, "check Run job as an LDAP user on the login node", logger) + + // Verify LSF commands on login node as LDAP user + lsfCmdErr := VerifyLSFCommands(t, sshLdapClient, ldapUserName, logger) + utils.LogVerificationResult(t, lsfCmdErr, "Check the 'lsf' command as an LDAP user on the login node", logger) +} + +// VerifyComputeNodeLDAPConfig verifies the LDAP configuration, file mount, LSF commands, +// and SSH connection to all compute nodes as an LDAP user. +func VerifyComputeNodeLDAPConfig( + t *testing.T, + bastionIP string, + ldapServerIP string, + computeNodeIPList []string, + ldapDomainName string, + ldapUserName string, + ldapPassword string, + logger *utils.AggregatedLogger, +) { + if len(computeNodeIPList) == 0 { + utils.LogVerificationResult(t, errors.New("compute node IPs cannot be empty"), "compute ldap configuration check", logger) + return + } + // Connect to the first compute node via SSH as an LDAP user + sshLdapClient, connectionErr := utils.ConnectToHostAsLDAPUser(LSF_PUBLIC_HOST_NAME, bastionIP, computeNodeIPList[0], ldapUserName, ldapPassword) + if connectionErr != nil { + utils.LogVerificationResult(t, connectionErr, "connect to the compute node via SSH as LDAP User failed", logger) + return + } + defer sshLdapClient.Close() + + // Verify LDAP configuration + ldapErr := VerifyLDAPConfig(t, sshLdapClient, "compute", ldapServerIP, ldapDomainName, ldapUserName, logger) + utils.LogVerificationResult(t, ldapErr, "ldap configuration check on the compute node", logger) + + // Check file mount + fileMountErr := HPCCheckFileMountAsLDAPUser(t, sshLdapClient, "compute", logger) + utils.LogVerificationResult(t, fileMountErr, "check file mount as an LDAP user on the compute node", logger) + + // Verify LSF commands + lsfCmdErr := VerifyLSFCommands(t, sshLdapClient, ldapUserName, logger) + utils.LogVerificationResult(t, lsfCmdErr, "Check the 'lsf' command as an LDAP user on the compute node", logger) + + // SSH connection to other compute nodes + for i := 0; i < len(computeNodeIPList); i++ { + sshLdapClientUser, connectionErr := utils.ConnectToHostAsLDAPUser(LSF_PUBLIC_HOST_NAME, bastionIP, computeNodeIPList[i], ldapUserName, ldapPassword) + if connectionErr == nil { + logger.Info(t, fmt.Sprintf("connect to the compute node %s via SSH as LDAP User", computeNodeIPList[i])) + } + utils.LogVerificationResult(t, connectionErr, "connect to the compute node via SSH as LDAP User", logger) + defer sshLdapClientUser.Close() + } +} + +// CheckLDAPServerStatus performs verification of LDAP server configuration on a remote machine +// using SSH and logs the status. +func CheckLDAPServerStatus(t *testing.T, sClient *ssh.Client, ldapAdminpassword, ldapDomain, ldapUser string, logger *utils.AggregatedLogger) { + // Validate LDAP server configuration + ldapErr := VerifyLDAPServerConfig(t, sClient, ldapAdminpassword, ldapDomain, ldapUser, logger) + utils.LogVerificationResult(t, ldapErr, "ldap Server Status", logger) +} + +// VerifyPTRRecordsForManagementAndLoginNodes verifies PTR records for 'mgmt' or 'login' nodes and ensures their resolution via SSH. +// It retrieves hostnames, performs nslookup to verify PTR records, and returns an error if any step fails. +func VerifyPTRRecordsForManagementAndLoginNodes(t *testing.T, sClient *ssh.Client, publicHostName, publicHostIP, privateHostName string, managementNodeIPList []string, loginNodeIP string, domainName string, logger *utils.AggregatedLogger) { + // Call sub-function to verify PTR records + err := verifyPTRRecords(t, sClient, publicHostName, publicHostIP, privateHostName, managementNodeIPList, loginNodeIP, domainName, logger) + // Log the verification result + utils.LogVerificationResult(t, err, "PTR Records For Management And Login Nodes", logger) + +} + +// CreateServiceInstanceandKmsKey creates a service instance on IBM Cloud and a KMS key within that instance. +// It logs into IBM Cloud using the provided API key, region, and resource group, then creates the service instance +// and the KMS key with the specified names. It logs the results of each operation. +// Returns:error - An error if any operation fails, otherwise nil. +func CreateServiceInstanceandKmsKey(t *testing.T, apiKey, expectedZone, expectedResourceGroup, kmsInstanceName, kmsKeyName string, logger *utils.AggregatedLogger) error { + // Create the service instance and return its GUID + _, createInstanceErr := CreateServiceInstanceAndReturnGUID(t, apiKey, expectedZone, expectedResourceGroup, kmsInstanceName, logger) + // Log the verification result for creating the service instance + utils.LogVerificationResult(t, createInstanceErr, "Create Service Instance", logger) + if createInstanceErr != nil { + return createInstanceErr + } + + // If the service instance creation was successful, create the KMS key + createKmsKeyErr := CreateKey(t, apiKey, expectedZone, expectedResourceGroup, kmsInstanceName, kmsKeyName, logger) + // Log the verification result for creating the KMS key + utils.LogVerificationResult(t, createKmsKeyErr, "Create KMS Key", logger) + if createKmsKeyErr != nil { + return createKmsKeyErr + } + + return nil +} + +// DeleteServiceInstanceAndAssociatedKeys deletes a service instance and its associated keys, then logs the result. +// This function deletes the specified service instance and its associated keys, then logs the verification result. +func DeleteServiceInstanceAndAssociatedKeys(t *testing.T, apiKey, expectedZone, expectedResourceGroup, kmsInstanceName string, logger *utils.AggregatedLogger) { + + deleteInstanceAndKey := DeleteServiceInstance(t, apiKey, expectedZone, expectedResourceGroup, kmsInstanceName, logger) + + // Log the verification result for deleting the service instance and associated KMS key + utils.LogVerificationResult(t, deleteInstanceAndKey, "Delete Service Instance and associated KMS Key", logger) +} diff --git a/tests/lsf/lsf_cluster_test_validation.go b/tests/lsf/lsf_cluster_test_validation.go new file mode 100644 index 00000000..964a5fd3 --- /dev/null +++ b/tests/lsf/lsf_cluster_test_validation.go @@ -0,0 +1,546 @@ +package tests + +import ( + "os" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testhelper" + + utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/common_utils" +) + +// ValidateClusterConfiguration performs comprehensive validation on the cluster setup. +// It connects to various cluster components via SSH and verifies their configurations and functionality. +// This includes the following validations: +// - Management Node: Verifies the configuration of the management node, including failover and failback procedures. +// - Compute Nodes: Ensures proper configuration and SSH connectivity to compute nodes. +// - Login Node: Validates the configuration and SSH connectivity to the login node. +// - Dynamic Compute Nodes: Verifies the proper setup and functionality of dynamic compute nodes. +// Additionally, this function logs detailed information throughout the validation process. +// This function doesn't return any value but logs errors and validation steps during the process. +func ValidateClusterConfiguration(t *testing.T, options *testhelper.TestOptions, testLogger *utils.AggregatedLogger) { + // Retrieve cluster information + expectedClusterID := options.TerraformVars["cluster_id"].(string) + expectedReservationID := options.TerraformVars["reservation_id"].(string) + expectedMasterName := options.TerraformVars["cluster_prefix"].(string) + expectedResourceGroup := options.TerraformVars["resource_group"].(string) + expectedKeyManagement := options.TerraformVars["key_management"].(string) + expectedZone := options.TerraformVars["zones"].([]string)[0] + expectedDnsDomainName, ok := options.TerraformVars["dns_domain_name"].(map[string]string)["compute"] + assert.False(t, !ok, "Key 'compute' does not exist in dns_domain_name map or dns_domain_name is not of type map[string]string") + + expectedHyperthreadingEnabled, _ := strconv.ParseBool(options.TerraformVars["hyperthreading_enabled"].(string)) + JOB_COMMAND_LOW := GetJobCommand(expectedZone, "low") + JOB_COMMAND_MED := GetJobCommand(expectedZone, "med") + + // Run the test and handle errors + output, err := options.RunTestConsistency() + require.NoError(t, err, "Error running consistency test: %v", err) + require.NotNil(t, output, "Expected non-nil output, but got nil") + + // Proceed with SSH connection and verification if there are no errors + testLogger.Info(t, t.Name()+" Cluster created successfully") + + bastionIP, managementNodeIPList, loginNodeIP, ipRetrievalError := utils.GetServerIPs(t, options, testLogger) + require.NoError(t, ipRetrievalError, "Error occurred while getting server IPs: %v", ipRetrievalError) + + testLogger.Info(t, t.Name()+" Validation started ......") + + // Connect to the master node via SSH and handle connection errors + sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.Nil(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + + testLogger.Info(t, "SSH connection to the master successful") + t.Log("Validation in progress. Please wait...") + + // Verify management node configuration + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + + // Verify SSH key + VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "management", managementNodeIPList, testLogger) + + // Perform failover and failback + FailoverAndFailback(t, sshClient, JOB_COMMAND_MED, testLogger) + + // Restart LSF daemon + RestartLsfDaemon(t, sshClient, JOB_COMMAND_LOW, testLogger) + + // Reboot instance + RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], JOB_COMMAND_MED, testLogger) + + // Connect to the master node via SSH and handle connection errors + sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.Nil(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + defer sshClient.Close() + defer func() { + if err := LSFWaitForDynamicNodeDisappearance(t, sshClient, testLogger); err != nil { + t.Errorf("Error in LSFWaitForDynamicNodeDisappearance: %v", err) + } + }() + + // Get dynamic compute node IPs and handle errors + computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) + require.Nil(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + + // Verify compute node configuration + VerifyComputetNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) + + // Verify SSH key + VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "compute", computeNodeIPList, testLogger) + + // Connect to the login node via SSH and handle connection errors + sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) + defer sshLoginNodeClient.Close() + + // Verify login node configuration + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + + // Get dynamic compute node IPs and handle errors + computeNodeIPList, computeIPErr = LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) + require.Nil(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + + // Verify SSH connectivity to nodes from login + VerifySSHConnectivityToNodesFromLogin(t, sshLoginNodeClient, managementNodeIPList, computeNodeIPList, testLogger) + + // Verify PTR records + VerifyPTRRecordsForManagementAndLoginNodes(t, sshClient, LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList, loginNodeIP, expectedDnsDomainName, testLogger) + + // Verify file share encryption + VerifyFileShareEncryption(t, os.Getenv("TF_VAR_ibmcloud_api_key"), utils.GetRegion(expectedZone), expectedResourceGroup, expectedMasterName, expectedKeyManagement, testLogger) + testLogger.Info(t, t.Name()+" Validation ended") +} + +// ValidateClusterConfigurationWithAPPCenter performs validation tasks on the cluster configuration +// with additional verification for an application center. +// It extends the validation performed by ValidateClusterConfiguration to include checks for the application center configuration. +// It connects to various cluster components via SSH and verifies their configurations and functionality. +// This includes the following validations: +// - Management Node: Verifies the configuration of the management node, including failover and failback procedures. +// - Compute Nodes: Ensures proper configuration and SSH connectivity to compute nodes. +// - Login Node: Validates the configuration and SSH connectivity to the login node. +// - Dynamic Compute Nodes: Verifies the proper setup and functionality of dynamic compute nodes. +// Additionally, this function logs detailed information throughout the validation process. +// This function doesn't return any value but logs errors and validation steps during the process. +func ValidateClusterConfigurationWithAPPCenter(t *testing.T, options *testhelper.TestOptions, testLogger *utils.AggregatedLogger) { + // Retrieve cluster information + expectedClusterID := options.TerraformVars["cluster_id"].(string) + expectedReservationID := options.TerraformVars["reservation_id"].(string) + expectedMasterName := options.TerraformVars["cluster_prefix"].(string) + expectedResourceGroup := options.TerraformVars["resource_group"].(string) + expectedKeyManagement := options.TerraformVars["key_management"].(string) + expectedZone := options.TerraformVars["zones"].([]string)[0] + expectedDnsDomainName, ok := options.TerraformVars["dns_domain_name"].(map[string]string)["compute"] + assert.False(t, !ok, "Key 'compute' does not exist in dns_domain_name map or dns_domain_name is not of type map[string]string") + expectedHyperthreadingEnabled, _ := strconv.ParseBool(options.TerraformVars["hyperthreading_enabled"].(string)) + JOB_COMMAND_LOW := GetJobCommand(expectedZone, "low") + JOB_COMMAND_MED := GetJobCommand(expectedZone, "med") + + // Run the test and handle errors + output, err := options.RunTestConsistency() + require.NoError(t, err, "Error running consistency test: %v", err) + require.NotNil(t, output, "Expected non-nil output, but got nil") + + // Proceed with SSH connection and verification if there are no errors + testLogger.Info(t, t.Name()+" Cluster created successfully") + + bastionIP, managementNodeIPList, loginNodeIP, ipRetrievalError := utils.GetServerIPs(t, options, testLogger) + require.NoError(t, ipRetrievalError, "Error occurred while getting server IPs: %v", ipRetrievalError) + + testLogger.Info(t, t.Name()+" Validation started ......") + + // Connect to the master node via SSH and handle connection errors + sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.Nil(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + + testLogger.Info(t, "SSH connection to the master successful") + t.Log("Validation in progress. Please wait...") + + // Verify management node configuration + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + + // Verify SSH key + VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "management", managementNodeIPList, testLogger) + + // Perform failover and failback + FailoverAndFailback(t, sshClient, JOB_COMMAND_MED, testLogger) + + // Restart LSF daemon + RestartLsfDaemon(t, sshClient, JOB_COMMAND_LOW, testLogger) + + // Reboot instance + RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], JOB_COMMAND_MED, testLogger) + + // Connect to the master node via SSH and handle connection errors + sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.Nil(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + defer sshClient.Close() + defer func() { + if err := LSFWaitForDynamicNodeDisappearance(t, sshClient, testLogger); err != nil { + t.Errorf("Error in LSFWaitForDynamicNodeDisappearance: %v", err) + } + }() + + // Get dynamic compute node IPs and handle errors + computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) + require.Nil(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + + // Verify compute node configuration + VerifyComputetNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) + + // Verify SSH key + VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "compute", computeNodeIPList, testLogger) + + // Connect to the login node via SSH and handle connection errors + sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) + defer sshLoginNodeClient.Close() + + // Verify login node configuration + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + + // Get dynamic compute node IPs and handle errors + computeNodeIPList, computeIPErr = LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) + require.Nil(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + + // Verify SSH connectivity to nodes from login + VerifySSHConnectivityToNodesFromLogin(t, sshLoginNodeClient, managementNodeIPList, computeNodeIPList, testLogger) + + // Verify application center configuration + VerifyAPPCenterConfig(t, sshClient, testLogger) + + // Verify noVNC configuration + VerifyNoVNCConfig(t, sshClient, testLogger) + + // Verify PTR records + VerifyPTRRecordsForManagementAndLoginNodes(t, sshClient, LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList, loginNodeIP, expectedDnsDomainName, testLogger) + + // Verify file share encryption + VerifyFileShareEncryption(t, os.Getenv("TF_VAR_ibmcloud_api_key"), utils.GetRegion(expectedZone), expectedResourceGroup, expectedMasterName, expectedKeyManagement, testLogger) + + testLogger.Info(t, t.Name()+" Validation ended") +} + +// ValidateBasicClusterConfiguration validates basic cluster configuration. +// It performs validation tasks on essential aspects of the cluster setup, +// including the management node, compute nodes, and login node configurations. +// Additionally, it ensures proper connectivity and functionality. +// This function doesn't return any value but logs errors and validation steps during the process. +func ValidateBasicClusterConfiguration(t *testing.T, options *testhelper.TestOptions, testLogger *utils.AggregatedLogger) { + // Retrieve cluster information + expectedClusterID := options.TerraformVars["cluster_id"].(string) + expectedReservationID := options.TerraformVars["reservation_id"].(string) + expectedMasterName := options.TerraformVars["cluster_prefix"].(string) + expectedResourceGroup := options.TerraformVars["resource_group"].(string) + expectedKeyManagement := options.TerraformVars["key_management"].(string) + expectedZone := options.TerraformVars["zones"].([]string)[0] + + expectedHyperthreadingEnabled, _ := strconv.ParseBool(options.TerraformVars["hyperthreading_enabled"].(string)) + + JOB_COMMAND_LOW := GetJobCommand(expectedZone, "low") + + // Run the test and handle errors + output, err := options.RunTest() + require.NoError(t, err, "Error running consistency test: %v", err) + require.NotNil(t, output, "Expected non-nil output, but got nil") + + // Proceed with SSH connection and verification if there are no errors + testLogger.Info(t, t.Name()+" Cluster created successfully") + + bastionIP, managementNodeIPList, loginNodeIP, ipRetrievalError := utils.GetServerIPs(t, options, testLogger) + require.NoError(t, ipRetrievalError, "Error occurred while getting server IPs: %v", ipRetrievalError) + + testLogger.Info(t, t.Name()+" Validation started ......") + + // Connect to the master node via SSH and handle connection errors + sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.Nil(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + + testLogger.Info(t, "SSH connection to the master successful") + t.Log("Validation in progress. Please wait...") + + // Verify management node configuration + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + + // Connect to the master node via SSH and handle connection errors + sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.Nil(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + defer sshClient.Close() + defer func() { + if err := LSFWaitForDynamicNodeDisappearance(t, sshClient, testLogger); err != nil { + t.Errorf("Error in LSFWaitForDynamicNodeDisappearance: %v", err) + } + }() + + // Get dynamic compute node IPs and handle errors + computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) + require.Nil(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + + // Verify compute node configuration + VerifyComputetNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) + + // Connect to the login node via SSH and handle connection errors + sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) + defer sshLoginNodeClient.Close() + + // Verify login node configuration + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + + // Verify file share encryption + VerifyFileShareEncryption(t, os.Getenv("TF_VAR_ibmcloud_api_key"), utils.GetRegion(expectedZone), expectedResourceGroup, expectedMasterName, expectedKeyManagement, testLogger) + + testLogger.Info(t, t.Name()+" Validation ended") +} + +// ValidateLDAPClusterConfiguration performs comprehensive validation on the cluster setup. +// It connects to various cluster components via SSH and verifies their configurations and functionality. +// This includes the following validations: +// - Management Node: Verifies the configuration of the management node, including failover and failback procedures. +// - Compute Nodes: Ensures proper configuration and SSH connectivity to compute nodes. +// - Login Node: Validates the configuration and SSH connectivity to the login node. +// - Dynamic Compute Nodes: Verifies the proper setup and functionality of dynamic compute nodes. +// Additionally, this function logs detailed information throughout the validation process. +// This function doesn't return any value but logs errors and validation steps during the process. +func ValidateLDAPClusterConfiguration(t *testing.T, options *testhelper.TestOptions, testLogger *utils.AggregatedLogger) { + // Retrieve cluster information + expectedClusterID := options.TerraformVars["cluster_id"].(string) + expectedReservationID := options.TerraformVars["reservation_id"].(string) + expectedMasterName := options.TerraformVars["cluster_prefix"].(string) + expectedResourceGroup := options.TerraformVars["resource_group"].(string) + expectedKeyManagement := options.TerraformVars["key_management"].(string) + expectedLdapDomain := options.TerraformVars["ldap_basedns"].(string) + expectedLdapAdminPassword := options.TerraformVars["ldap_admin_password"].(string) + expectedLdapUserName := options.TerraformVars["ldap_user_name"].(string) + expectedLdapUserPassword := options.TerraformVars["ldap_user_password"].(string) + expectedDnsDomainName, ok := options.TerraformVars["dns_domain_name"].(map[string]string)["compute"] + assert.False(t, !ok, "Key 'compute' does not exist in dns_domain_name map or dns_domain_name is not of type map[string]string") + + expectedZone := options.TerraformVars["zones"].([]string)[0] + + expectedHyperthreadingEnabled, _ := strconv.ParseBool(options.TerraformVars["hyperthreading_enabled"].(string)) + JOB_COMMAND_LOW := GetJobCommand(expectedZone, "low") + JOB_COMMAND_MED := GetJobCommand(expectedZone, "med") + + // Run the test and handle errors + output, err := options.RunTestConsistency() + require.NoError(t, err, "Error running consistency test: %v", err) + require.NotNil(t, output, "Expected non-nil output, but got nil") + + // Proceed with SSH connection and verification if there are no errors + testLogger.Info(t, t.Name()+" Cluster created successfully") + + // Get server IPs and handle errors + bastionIP, managementNodeIPList, loginNodeIP, LdapServerIP, ipRetrievalError := utils.GetServerIPsWithLDAP(t, options, testLogger) + require.NoError(t, ipRetrievalError, "Error occurred while getting server IPs: %v", ipRetrievalError) + + testLogger.Info(t, t.Name()+" Validation started") + + // Connect to the master node via SSH and handle connection errors + sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.Nil(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + + testLogger.Info(t, "SSH connection to the master successful") + t.Log("Validation in progress. Please wait...") + + // Verify management node configuration + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + + // Verify SSH key + VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "management", managementNodeIPList, testLogger) + + // Perform failover and failback + FailoverAndFailback(t, sshClient, JOB_COMMAND_MED, testLogger) + + // Restart LSF daemon + RestartLsfDaemon(t, sshClient, JOB_COMMAND_LOW, testLogger) + + // Reboot instance + RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], JOB_COMMAND_MED, testLogger) + + // Connect to the master node via SSH and handle connection errors + sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.Nil(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + defer sshClient.Close() + defer func() { + if err := LSFWaitForDynamicNodeDisappearance(t, sshClient, testLogger); err != nil { + t.Errorf("Error in LSFWaitForDynamicNodeDisappearance: %v", err) + } + }() + + // Get dynamic compute node IPs and handle errors + computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) + require.Nil(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + + // Verify compute node configuration + VerifyComputetNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) + + // Verify SSH key + VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "compute", computeNodeIPList, testLogger) + + // Connect to the login node via SSH and handle connection errors + sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) + + // Verify login node configuration + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + + // Verify SSH connectivity to nodes from login + VerifySSHConnectivityToNodesFromLogin(t, sshLoginNodeClient, managementNodeIPList, computeNodeIPList, testLogger) + + // Verify file share encryption + VerifyFileShareEncryption(t, os.Getenv("TF_VAR_ibmcloud_api_key"), utils.GetRegion(expectedZone), expectedResourceGroup, expectedMasterName, expectedKeyManagement, testLogger) + + // Connect to the ldap server via SSH and handle connection errors + sshLdapClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_LDAP_HOST_NAME, LdapServerIP) + require.NoError(t, connectionErr, "Failed to connect to the ldap server via SSH: %v", connectionErr) + + // Check ldap server status + CheckLDAPServerStatus(t, sshLdapClient, expectedLdapAdminPassword, expectedLdapDomain, expectedLdapUserName, testLogger) + + // Verify management node ldap config + VerifyManagementNodeLDAPConfig(t, sshClient, bastionIP, LdapServerIP, managementNodeIPList, JOB_COMMAND_LOW, expectedLdapDomain, expectedLdapUserName, expectedLdapUserPassword, testLogger) + + // Verify login node ldap config + VerifyLoginNodeLDAPConfig(t, sshClient, bastionIP, loginNodeIP, LdapServerIP, JOB_COMMAND_LOW, expectedLdapDomain, expectedLdapUserName, expectedLdapUserPassword, testLogger) + + // Verify PTR records + VerifyPTRRecordsForManagementAndLoginNodes(t, sshClient, LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList, loginNodeIP, expectedDnsDomainName, testLogger) + + // Verify compute node ldap config + VerifyComputeNodeLDAPConfig(t, bastionIP, LdapServerIP, computeNodeIPList, expectedLdapDomain, expectedLdapUserName, expectedLdapUserPassword, testLogger) + + testLogger.Info(t, t.Name()+" Validation ended") +} + +// ValidatePACANDLDAPClusterConfiguration performs comprehensive validation on the PAC and LDAP cluster setup. +// It connects to various cluster components via SSH and verifies their configurations and functionality. +// This includes the following validations: +// - Management Node: Verifies the configuration of the management node, including failover and failback procedures. +// - Compute Nodes: Ensures proper configuration and SSH connectivity to compute nodes. +// - Login Node: Validates the configuration and SSH connectivity to the login node. +// - Dynamic Compute Nodes: Verifies the proper setup and functionality of dynamic compute nodes. +// - LDAP Server: Checks the LDAP server status and verifies LDAP configurations across nodes. +// - Application Center: Verifies the application center configuration. +// - noVNC: Verifies the noVNC configuration. +// Additionally, this function logs detailed information throughout the validation process. +// This function doesn't return any value but logs errors and validation steps during the process. +func ValidatePACANDLDAPClusterConfiguration(t *testing.T, options *testhelper.TestOptions, testLogger *utils.AggregatedLogger) { + // Retrieve cluster information + expectedClusterID := options.TerraformVars["cluster_id"].(string) + expectedReservationID := options.TerraformVars["reservation_id"].(string) + expectedMasterName := options.TerraformVars["cluster_prefix"].(string) + expectedResourceGroup := options.TerraformVars["resource_group"].(string) + expectedKeyManagement := options.TerraformVars["key_management"].(string) + expectedLdapDomain := options.TerraformVars["ldap_basedns"].(string) + expectedLdapAdminPassword := options.TerraformVars["ldap_admin_password"].(string) + expectedLdapUserName := options.TerraformVars["ldap_user_name"].(string) + expectedLdapUserPassword := options.TerraformVars["ldap_user_password"].(string) + expectedDnsDomainName, ok := options.TerraformVars["dns_domain_name"].(map[string]string)["compute"] + assert.False(t, !ok, "Key 'compute' does not exist in dns_domain_name map or dns_domain_name is not of type map[string]string") + + expectedZone := options.TerraformVars["zones"].([]string)[0] + + expectedHyperthreadingEnabled, _ := strconv.ParseBool(options.TerraformVars["hyperthreading_enabled"].(string)) + JOB_COMMAND_LOW := GetJobCommand(expectedZone, "low") + JOB_COMMAND_MED := GetJobCommand(expectedZone, "med") + + // Run the test and handle errors + output, err := options.RunTestConsistency() + require.NoError(t, err, "Error running consistency test: %v", err) + require.NotNil(t, output, "Expected non-nil output, but got nil") + + // Proceed with SSH connection and verification if there are no errors + testLogger.Info(t, t.Name()+" Cluster created successfully") + + // Get server IPs and handle errors + bastionIP, managementNodeIPList, loginNodeIP, LdapServerIP, ipRetrievalError := utils.GetServerIPsWithLDAP(t, options, testLogger) + require.NoError(t, ipRetrievalError, "Error occurred while getting server IPs: %v", ipRetrievalError) + + testLogger.Info(t, t.Name()+" Validation started") + + // Connect to the master node via SSH and handle connection errors + sshClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.Nil(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + + testLogger.Info(t, "SSH connection to the master successful") + t.Log("Validation in progress. Please wait...") + + // Verify management node configuration + VerifyManagementNodeConfig(t, sshClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, managementNodeIPList, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + + // Verify SSH key + VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "management", managementNodeIPList, testLogger) + + // Perform failover and failback + FailoverAndFailback(t, sshClient, JOB_COMMAND_MED, testLogger) + + // Restart LSF daemon + RestartLsfDaemon(t, sshClient, JOB_COMMAND_LOW, testLogger) + + // Reboot instance + RebootInstance(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0], JOB_COMMAND_MED, testLogger) + + // Connect to the master node via SSH and handle connection errors + sshClient, connectionErr = utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList[0]) + require.Nil(t, connectionErr, "Failed to connect to the master via SSH: %v", connectionErr) + defer sshClient.Close() + defer func() { + if err := LSFWaitForDynamicNodeDisappearance(t, sshClient, testLogger); err != nil { + t.Errorf("Error in LSFWaitForDynamicNodeDisappearance: %v", err) + } + }() + + // Get dynamic compute node IPs and handle errors + computeNodeIPList, computeIPErr := LSFGETDynamicComputeNodeIPs(t, sshClient, testLogger) + require.Nil(t, computeIPErr, "Error getting dynamic compute node IPs: %v", computeIPErr) + + // Verify compute node configuration + VerifyComputetNodeConfig(t, sshClient, expectedHyperthreadingEnabled, computeNodeIPList, testLogger) + + // Verify SSH key + VerifySSHKey(t, sshClient, bastionIP, LSF_PUBLIC_HOST_NAME, LSF_PRIVATE_HOST_NAME, "compute", computeNodeIPList, testLogger) + + // Connect to the login node via SSH and handle connection errors + sshLoginNodeClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, loginNodeIP) + require.NoError(t, connectionErr, "Failed to connect to the login node via SSH: %v", connectionErr) + + // Verify login node configuration + VerifyLoginNodeConfig(t, sshLoginNodeClient, expectedClusterID, expectedMasterName, expectedReservationID, expectedHyperthreadingEnabled, loginNodeIP, JOB_COMMAND_LOW, EXPECTED_LSF_VERSION, testLogger) + + // Verify SSH connectivity to nodes from login + VerifySSHConnectivityToNodesFromLogin(t, sshLoginNodeClient, managementNodeIPList, computeNodeIPList, testLogger) + + // Verify file share encryption + VerifyFileShareEncryption(t, os.Getenv("TF_VAR_ibmcloud_api_key"), utils.GetRegion(expectedZone), expectedResourceGroup, expectedMasterName, expectedKeyManagement, testLogger) + + // Verify application center configuration + VerifyAPPCenterConfig(t, sshClient, testLogger) + + // Verify noVNC configuration + VerifyNoVNCConfig(t, sshClient, testLogger) + + // Connect to the ldap server via SSH and handle connection errors + sshLdapClient, connectionErr := utils.ConnectToHost(LSF_PUBLIC_HOST_NAME, bastionIP, LSF_LDAP_HOST_NAME, LdapServerIP) + require.NoError(t, connectionErr, "Failed to connect to the ldap server via SSH: %v", connectionErr) + + // Check ldap server status + CheckLDAPServerStatus(t, sshLdapClient, expectedLdapAdminPassword, expectedLdapDomain, expectedLdapUserName, testLogger) + + // Verify management node ldap config + VerifyManagementNodeLDAPConfig(t, sshClient, bastionIP, LdapServerIP, managementNodeIPList, JOB_COMMAND_LOW, expectedLdapDomain, expectedLdapUserName, expectedLdapUserPassword, testLogger) + + // Verify login node ldap config + VerifyLoginNodeLDAPConfig(t, sshClient, bastionIP, loginNodeIP, LdapServerIP, JOB_COMMAND_LOW, expectedLdapDomain, expectedLdapUserName, expectedLdapUserPassword, testLogger) + + // Verify PTR records + VerifyPTRRecordsForManagementAndLoginNodes(t, sshClient, LSF_PUBLIC_HOST_NAME, bastionIP, LSF_PRIVATE_HOST_NAME, managementNodeIPList, loginNodeIP, expectedDnsDomainName, testLogger) + + // Verify compute node ldap config + VerifyComputeNodeLDAPConfig(t, bastionIP, LdapServerIP, computeNodeIPList, expectedLdapDomain, expectedLdapUserName, expectedLdapUserPassword, testLogger) + + testLogger.Info(t, t.Name()+" Validation ended") +} diff --git a/tests/lsf/lsf_cluster_utils.go b/tests/lsf/lsf_cluster_utils.go new file mode 100644 index 00000000..8364005e --- /dev/null +++ b/tests/lsf/lsf_cluster_utils.go @@ -0,0 +1,1739 @@ +package tests + +import ( + "bufio" + "errors" + "fmt" + "maps" + "os" + "os/exec" + "regexp" + "sort" + "strings" + "testing" + "time" + + utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/common_utils" + "golang.org/x/crypto/ssh" +) + +const ( + defaultSleepDuration = 30 * time.Second + timeOutForDynamicNodeDisappear = 12 * time.Minute + jobCompletionWaitTime = 50 * time.Second + dynamicNodeWaitTime = 3 * time.Minute + appCenterPort = 8443 +) + +// LSFMTUCheck checks the MTU setting for multiple nodes of a specified type. +// It returns an error if any node's MTU is not set to 9000. +func LSFMTUCheck(t *testing.T, sClient *ssh.Client, ipsList []string, logger *utils.AggregatedLogger) error { + // commands to check MTU on different OS types + ubuntuMTUCheckCmd := "ip addr show" + rhelMTUCheckCmd := "ifconfig" + + // Check if the node list is empty + if len(ipsList) == 0 { + return fmt.Errorf("ERROR: ips cannot be empty") + } + + // Loop through each IP in the list + for _, ip := range ipsList { + var mtuCmd string + + // Get the OS name of the compute node. + osName, osNameErr := GetOSNameOfNode(t, sClient, ip, logger) + if osNameErr != nil { + // Determine the expected command to check MTU based on the OS. + switch osName { + case "Ubuntu": + mtuCmd = ubuntuMTUCheckCmd + default: + mtuCmd = rhelMTUCheckCmd + } + } else { + // Return error if OS name retrieval fails. + return osNameErr + } + + // Build the SSH command to check MTU on the node + command := fmt.Sprintf("ssh %s %s", ip, mtuCmd) + + // Execute the command and get the output + output, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + return fmt.Errorf("failed to execute '%s' command on (%s) node", mtuCmd, ip) + } + + // Check if the output contains "mtu 9000" + if !utils.VerifyDataContains(t, output, "mtu 9000", logger) { + return fmt.Errorf("MTU is not set to 9000 for (%s) node and found:\n%s", ip, output) + } + + // Log a success message if MTU is set to 9000 + logger.Info(t, fmt.Sprintf("MTU is set to 9000 for (%s) node", ip)) + } + + return nil +} + +// LSFIPRouteCheck verifies that the IP routes on the specified nodes have the MTU set to 9000. +// It returns an error if any node's IP route does not have the expected MTU value. +func LSFIPRouteCheck(t *testing.T, sClient *ssh.Client, ipsList []string, logger *utils.AggregatedLogger) error { + + // Check if the node list is empty + if len(ipsList) == 0 { + return fmt.Errorf("IPs list cannot be empty") + } + + // Loop through each IP one by one + for _, ip := range ipsList { + command := fmt.Sprintf("ssh %s ip route", ip) + output, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + return fmt.Errorf("failed to execute 'ip route' command on (%s) node: %v", ip, err) + } + + if !utils.VerifyDataContains(t, output, "mtu 9000", logger) { + return fmt.Errorf("IP route MTU is not set to 9000 for (%s) node. Found: \n%s", ip, output) + } + logger.Info(t, fmt.Sprintf("IP route MTU is set to 9000 for (%s) node", ip)) + } + + return nil +} + +// LSFCheckClusterID checks if the provided cluster ID matches the expected value. +// It uses the provided SSH client to execute the 'lsid' command and verifies +// if the expected cluster ID is present in the command output. +// Returns an error if the checks fail. +func LSFCheckClusterID(t *testing.T, sClient *ssh.Client, expectedClusterID string, logger *utils.AggregatedLogger) error { + + // Execute the 'lsid' command to get the cluster ID + command := "lsid" + output, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + return fmt.Errorf("failed to execute 'lsid' command: %w", err) + } + + // Verify if the expected cluster ID is present in the output + if !utils.VerifyDataContains(t, output, "My cluster name is "+expectedClusterID, logger) { + // Extract actual cluster version from the output for better error reporting + actualValue := strings.TrimSpace(strings.Split(strings.Split(output, "My cluster name is")[1], "My master name is")[0]) + return fmt.Errorf("expected cluster ID %s , but found %s", expectedClusterID, actualValue) + } + // Log success if no errors occurred + logger.Info(t, fmt.Sprintf("Cluster ID is set as expected : %s", expectedClusterID)) + return nil +} + +// LSFCheckMasterName checks if the provided master name matches the expected value. +// It uses the provided SSH client to execute the 'lsid' command and verifies +// if the expected master name is present in the command output. +// Returns an error if the checks fail. +func LSFCheckMasterName(t *testing.T, sClient *ssh.Client, expectedMasterName string, logger *utils.AggregatedLogger) error { + // Execute the 'lsid' command to get the cluster ID + command := "lsid" + output, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + return fmt.Errorf("failed to execute 'lsid' command: %w", err) + } + + // Verify if the expected master name is present in the output + if !utils.VerifyDataContains(t, output, "My master name is "+expectedMasterName+"-mgmt-1", logger) { + // Extract actual cluster version from the output for better error reporting + actualValue := strings.TrimSpace(strings.Split(output, "My master name is")[1]) + return fmt.Errorf("expected master name %s , but found %s", expectedMasterName, actualValue) + } + // Log success if no errors occurred + logger.Info(t, fmt.Sprintf("Master name is set as expected : %s", expectedMasterName)) + return nil +} + +// HPCCheckReservationID verifies if the provided SSH client's 'lsid' command output +// contains the expected Reservation ID. Logs and returns an error if verification fails. +func HPCCheckReservationID(t *testing.T, sClient *ssh.Client, expectedReservationID string, logger *utils.AggregatedLogger) error { + + ibmCloudHPCConfigPath := "/opt/ibm/lsf/conf/resource_connector/ibmcloudhpc/conf/ibmcloudhpc_config.json" + + command := fmt.Sprintf("cat %s", ibmCloudHPCConfigPath) + output, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil || !utils.VerifyDataContains(t, output, expectedReservationID, logger) { + return fmt.Errorf("failed Reservation ID verification: %w", err) + } + // Log success if no errors occurred + logger.Info(t, fmt.Sprintf("Reservation ID verified: %s", expectedReservationID)) + return nil +} + +// LSFCheckManagementNodeCount checks if the actual count of management nodes matches the expected count. +// It uses the provided SSH client to execute the 'bhosts' command with filters to +// count the number of nodes containing 'mgmt' in their names. The function then verifies +// if the actual count matches the expected count. +// Returns an error if the checks fail. +func LSFCheckManagementNodeCount(t *testing.T, sClient *ssh.Client, expectedManagementCount string, logger *utils.AggregatedLogger) error { + // Execute the 'bhosts' command to get the management node count + command := "bhosts -w | grep 'mgmt' | wc -l" + output, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + return fmt.Errorf("failed to execute 'bhosts' command: %w", err) + } + + // Verify if the expected management node count is present in the output + if !utils.VerifyDataContains(t, output, expectedManagementCount, logger) { + return fmt.Errorf("expected %s management nodes, but found %s", expectedManagementCount, strings.TrimSpace(output)) + } + + // Log success if no errors occurred + logger.Info(t, fmt.Sprintf("Management node count is as expected: %s", expectedManagementCount)) + return nil +} + +// LSFRestartDaemons restarts the LSF daemons on the provided SSH client. +// It executes the 'lsf_daemons restart' command as root, checks for a successful +// restart, and waits for LSF to start up. +func LSFRestartDaemons(t *testing.T, sClient *ssh.Client, logger *utils.AggregatedLogger) error { + + // Restart LSF daemons + restartCmd := "sudo su -l root -c 'lsf_daemons restart'" + out, err := utils.RunCommandInSSHSession(sClient, restartCmd) + if err != nil { + return fmt.Errorf("failed to run 'lsf_daemons restart' command: %w", err) + } + + time.Sleep(defaultSleepDuration) + + // Check if the restart was successful + if !(utils.VerifyDataContains(t, string(out), "Stopping", logger) && utils.VerifyDataContains(t, string(out), "Starting", logger)) { + return fmt.Errorf("lsf_daemons restart failed") + } + + // Wait for LSF to start up + for { + // Run 'bhosts -w' command on the remote SSH server + command := "bhosts -w" + startOut, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + return fmt.Errorf("failed to run 'bhosts' command: %w", err) + } + if !utils.VerifyDataContains(t, string(startOut), "LSF is down", logger) { + break + } + time.Sleep(5 * time.Second) + } + // Log success if no errors occurred + logger.Info(t, "lsf_daemons restart successfully") + return nil +} + +// LSFControlBctrld performs start or stop operation on the bctrld daemon on the specified machine. +// The function returns an error if any step fails or if an invalid value (other than 'start' or 'stop') is provided. +// It executes the 'bctrld' command with the specified operation and waits for the daemon to start or stop. +func LSFControlBctrld(t *testing.T, sClient *ssh.Client, startOrStop string, logger *utils.AggregatedLogger) error { + + // Make startOrStop case-insensitive + startOrStop = strings.ToLower(startOrStop) + + // Validate the operation type + if startOrStop != "start" && startOrStop != "stop" { + return fmt.Errorf("invalid operation type. Please specify 'start' or 'stop'") + } + + // Execute the 'bctrld' command to start or stop the sbd daemon + command := fmt.Sprintf("bctrld %s sbd", startOrStop) + _, bctrldErr := utils.RunCommandInSSHSession(sClient, command) + if bctrldErr != nil { + return fmt.Errorf("failed to run '%s' command: %w", command, bctrldErr) + } + + // Sleep for a specified duration to allow time for the daemon to start or stop + time.Sleep(20 * time.Second) + + // Check the status of the daemon using the 'bhosts -w' command on the remote SSH server + cmd := "bhosts -w" + out, err := utils.RunCommandInSSHSession(sClient, cmd) + if err != nil { + return fmt.Errorf("failed to run 'bhosts' command on machine IP: %w", err) + } + + // Count the number of unreachable nodes + unreachCount := strings.Count(string(out), "unreach") + + // Check the output based on the startOrStop parameter + if (startOrStop == "start" && unreachCount != 0) || (startOrStop == "stop" && unreachCount != 1) { + // If the unreachable node count does not match the expected count, return an error + return fmt.Errorf("failed to %s the sbd daemon on the management node", startOrStop) + } + // Log success if no errors occurred + logger.Info(t, fmt.Sprintf("Daemon %s successfully", startOrStop)) + return nil +} + +// LSFCheckIntelOneMpiOnComputeNodes checks the Intel OneAPI MPI on compute nodes. +// It verifies the existence of setvars.sh and mpi in the OneAPI folder, and initializes the OneAPI environment. +// and returns an error if any check fails. +func LSFCheckIntelOneMpiOnComputeNodes(t *testing.T, sClient *ssh.Client, ipsList []string, logger *utils.AggregatedLogger) error { + + // Check if the node list is empty + if len(ipsList) == 0 { + return fmt.Errorf("ERROR: ips cannot be empty") + } + + // Check Intel OneAPI MPI on each compute node + for _, ip := range ipsList { + // Check if OneAPI folder exists and contains setvars.sh and mpi + checkCmd := fmt.Sprintf("ssh %s 'ls /opt/intel/oneapi'", ip) + + checkOutput, checkErr := utils.RunCommandInSSHSession(sClient, checkCmd) + if checkErr != nil { + return fmt.Errorf("failed to run '%s' command: %w", checkCmd, checkErr) + } + + if !utils.VerifyDataContains(t, checkOutput, "setvars.sh", logger) && !utils.VerifyDataContains(t, checkOutput, "mpi", logger) { + return fmt.Errorf("setvars.sh or mpi not found in OneAPI folder: %s", checkOutput) + } + + // Initialize OneAPI environment + initCmd := fmt.Sprintf("ssh -t %s 'sudo su -l root -c \". /opt/intel/oneapi/setvars.sh\"'", ip) + initOutput, initErr := utils.RunCommandInSSHSession(sClient, initCmd) + if initErr != nil { + return fmt.Errorf("failed to run '%s' command: %w", initCmd, initErr) + } + + if !utils.VerifyDataContains(t, initOutput, ":: oneAPI environment initialized ::", logger) { + return fmt.Errorf("OneAPI environment not initialized on machine: %s and but found : \n%s", ip, initOutput) + } + logger.Info(t, fmt.Sprintf("Intel OneAPI MPI successfully checked and initialized on compute machine: %s", ip)) + + } + + return nil +} + +// LSFRebootInstance reboots an LSF instance using SSH. +// It executes the 'sudo su -l root -c 'reboot” command on the provided SSH client. +func LSFRebootInstance(t *testing.T, sClient *ssh.Client, logger *utils.AggregatedLogger) error { + + // Execute the "sudo su -l root -c 'reboot'" command to reboot the cluster + rebootCmd := "sudo su -l root -c 'reboot'" + + _, checkErr := utils.RunCommandInSSHSession(sClient, rebootCmd) + if !strings.Contains(checkErr.Error(), "remote command exited without exit status or exit signal") { + return fmt.Errorf("instance reboot failed") + } + + // Sleep for a short duration to allow the instance to restart + time.Sleep(2 * time.Minute) + + // Log success if no errors occurred + logger.Info(t, "LSF instance successfully rebooted") + return nil +} + +// LSFCheckBhostsResponse checks if the output of the 'bhosts' command is empty. +// It executes the 'bhosts' command on the provided SSH client. +// Returns an error if the 'bhosts' command output is empty +func LSFCheckBhostsResponse(t *testing.T, sClient *ssh.Client, logger *utils.AggregatedLogger) error { + + // Run 'bhosts -w' command on the remote SSH server + command := "bhosts -w" + + output, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + return fmt.Errorf("failed to run '%s' command: %w", command, err) + } + + // Check if the 'bhosts' command output is empty + if len(strings.TrimSpace(output)) == 0 || !strings.Contains(strings.TrimSpace(output), "HOST_NAME") { + return fmt.Errorf("non-empty response from 'bhosts' command: %s", output) + } + + logger.Info(t, fmt.Sprintf("bhosts value: %s", output)) + return nil +} + +// LSFRunJobs executes an LSF job on a remote server via SSH, monitors its status, +// and ensures its completion or terminates it if it exceeds a specified timeout. +// It returns an error if any step of the process fails. +func LSFRunJobs(t *testing.T, sClient *ssh.Client, jobCmd string, logger *utils.AggregatedLogger) error { + // Set the maximum timeout for the job execution + var jobMaxTimeout time.Duration + + // Record the start time of the job execution + startTime := time.Now() + + // Run the LSF job command on the remote server + jobOutput, err := utils.RunCommandInSSHSession(sClient, jobCmd) + if err != nil { + return fmt.Errorf("failed to run '%s' command: %w", jobCmd, err) + } + + // Extract the job ID from the job output + jobTime := utils.SplitAndTrim(jobCmd, "sleep")[1] + min, err := utils.StringToInt(jobTime) + if err != nil { + return err + } + min = 300 + min + jobMaxTimeout = time.Duration(min) * time.Second + + // Log the job output for debugging purposes + logger.Info(t, strings.TrimSpace(string(jobOutput))) + + // Extract the job ID from the job output + jobID, err := LSFExtractJobID(jobOutput) + if err != nil { + return err + } + + // Monitor the job's status until it completes or exceeds the timeout + for time.Since(startTime) < jobMaxTimeout { + + // Run 'bjobs -a' command on the remote SSH server + command := "bjobs -a" + + // Run the 'bjobs' command to get information about all jobs + jobStatus, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + return fmt.Errorf("failed to run 'bjobs' command: %w", err) + } + + // Create a regular expression pattern to match the job ID and status + pattern := regexp.MustCompile(fmt.Sprintf(`\b%s\s+lsfadmi DONE`, jobID)) + + // Check if the job ID appears in the 'bjobs' response with a status of 'DONE' + if pattern.MatchString(jobStatus) { + logger.Info(t, fmt.Sprintf("Job results : \n%s", jobStatus)) + logger.Info(t, fmt.Sprintf("Job %s has executed successfully", jobID)) + return nil + } + + // Sleep for a minute before checking again + logger.Info(t, fmt.Sprintf("Waiting for dynamic node creation and job completion. Elapsed time: %s", time.Since(startTime))) + time.Sleep(jobCompletionWaitTime) + } + + // If the job exceeds the specified timeout, attempt to terminate it + _, err = utils.RunCommandInSSHSession(sClient, fmt.Sprintf("bkill %s", jobID)) + if err != nil { + return fmt.Errorf("failed to run 'bkill' command: %w", err) + } + + // Return an error indicating that the job execution exceeded the specified time + return fmt.Errorf("job execution for ID %s exceeded the specified time", jobID) +} + +// LSFExtractJobID extracts the first sequence of one or more digits from the input response string. +// It uses a regular expression to find all non-overlapping matches in the response string, +// and returns the first match as the job ID. +// If no matches are found, it returns an error indicating that no job ID was found in the response. +func LSFExtractJobID(response string) (string, error) { + + // Define a regular expression pattern to extract one or more consecutive digits + re := regexp.MustCompile("[0-9]+") + + // Find all matches in the response + matches := re.FindAllString(response, -1) + + // Check if any matches are found + if len(matches) > 0 { + // Return the first match as the job ID + jobID := matches[0] + return jobID, nil + } + + // Return an error if no job ID is found + return "", fmt.Errorf("no job ID found with given response: %s", response) +} + +// LSFWaitForDynamicNodeDisappearance monitors the 'bhosts -w' command output over SSH, waiting for a dynamic node to disappear. +// It sets a timeout and checks for disappearance until completion. Returns an error if the timeout is exceeded or if +// there is an issue running the SSH command. + +func LSFWaitForDynamicNodeDisappearance(t *testing.T, sClient *ssh.Client, logger *utils.AggregatedLogger) error { + + // Record the start time of the job execution + startTime := time.Now() + + // Monitor the dynamic node; until it disappears or exceeds the timeout. + for time.Since(startTime) < timeOutForDynamicNodeDisappear { + + // Run 'bhosts -w' command on the remote SSH server + command := "bhosts -w" + output, err := utils.RunCommandInSSHSession(sClient, command) + + if err != nil { + return fmt.Errorf("failed to run SSH command '%s': %w", command, err) + } + + if utils.VerifyDataContains(t, output, "ok", logger) { + logger.Info(t, fmt.Sprintf("Waiting dynamic node to disappeard : \n%v", output)) + time.Sleep(90 * time.Second) + } else { + logger.Info(t, "Dynamic node has disappeared!") + return nil + } + } + + return fmt.Errorf("timeout of %s occurred while waiting for the dynamic node to disappear", timeOutForDynamicNodeDisappear.String()) +} + +// LSFAPPCenterConfiguration performs configuration validation for the APP Center by checking the status of essential services +// (WEBGUI and PNC) and ensuring that the APP center port (8081) is actively listening. +// Returns an error if the validation encounters issues, otherwise, nil is returned. +// LSFAPPCenterConfiguration checks and validates the configuration of the LSF App Center. +// It verifies whether the APP Center GUI or PNC is configured correctly, +// if the APP center port is listening as expected, if the APP center binary is installed, +// and if MariaDB packages are installed as expected. +// Returns an error if any validation check fails, otherwise nil. + +func LSFAPPCenterConfiguration(t *testing.T, sClient *ssh.Client, logger *utils.AggregatedLogger) error { + + lsfAppCenterPkg := "lsf-appcenter-10." + webguiStarted := "WEBGUI STARTED" + pncStarted := "PNC STARTED" + + // Command to check if APP Center GUI or PNC is configured + appConfigCommand := "sudo su -l root -c 'pmcadmin list'" + + appConfigOutput, err := utils.RunCommandInSSHSession(sClient, appConfigCommand) + if err != nil { + return fmt.Errorf("failed to execute command '%s': %w", appConfigCommand, err) + } + + if !(utils.VerifyDataContains(t, appConfigOutput, webguiStarted, logger) && utils.VerifyDataContains(t, appConfigOutput, pncStarted, logger)) { + return fmt.Errorf("APP Center GUI or PNC not configured as expected: %s", appConfigOutput) + } + + // Command to check if APP center port is listening as expected + portStatusCommand := fmt.Sprintf("netstat -tuln | grep %d", appCenterPort) + portStatusOutput, err := utils.RunCommandInSSHSession(sClient, portStatusCommand) + if err != nil { + return fmt.Errorf("failed to execute command '%s': %w", portStatusCommand, err) + } + + if !utils.VerifyDataContains(t, portStatusOutput, "LISTEN", logger) { + return fmt.Errorf("APP center port not listening as expected: %s", portStatusOutput) + } + + // Command to check if APP center binary is installed as expected + appBinaryCommand := "rpm -qa | grep lsf-appcenter" + appBinaryOutput, err := utils.RunCommandInSSHSession(sClient, appBinaryCommand) + if err != nil { + return fmt.Errorf("failed to execute command '%s': %w", appBinaryCommand, err) + } + + if !utils.VerifyDataContains(t, appBinaryOutput, lsfAppCenterPkg, logger) { + return fmt.Errorf("APP center binary not installed as expected: %s", appBinaryOutput) + } + + // Command to check if MariaDB packages are installed as expected + mariaDBCommand := "rpm -qa | grep MariaDB" + mariaDBOutput, err := utils.RunCommandInSSHSession(sClient, mariaDBCommand) + if err != nil { + return fmt.Errorf("failed to execute command '%s': %w", mariaDBCommand, err) + } + + mariaDBPackages := [4]string{"MariaDB-client", "MariaDB-common", "MariaDB-shared", "MariaDB-server"} + for _, out := range mariaDBPackages { + if !utils.VerifyDataContains(t, mariaDBOutput, out, logger) { + return fmt.Errorf("MariaDB was not installed as expected binary: %s", mariaDBOutput) + } + } + // Log success if no errors occurred + logger.Info(t, "Appcenter configuration validated successfully") + return nil +} + +// LSFGETDynamicComputeNodeIPs retrieves the IP addresses of dynamic worker nodes with a status of "ok". +// It returns a slice of IP addresses and an error if there was a problem executing the command or parsing the output. +func LSFGETDynamicComputeNodeIPs(t *testing.T, sClient *ssh.Client, logger *utils.AggregatedLogger) ([]string, error) { + + workerIPs := []string{} + + // Run the "bhosts -w" command to get the node status + command := "bhosts -w" + nodeStatus, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + return nil, fmt.Errorf("failed to execute 'bhosts' command: %w", err) + } + + scanner := bufio.NewScanner(strings.NewReader(nodeStatus)) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + + if len(fields) > 1 && fields[1] == "ok" { + // Split the input string by hyphen + parts := strings.Split(fields[0], "-") + + // Extract the IP address part + ip := strings.Join(parts[len(parts)-4:], ".") + workerIPs = append(workerIPs, ip) + } + } + sort.StringsAreSorted(workerIPs) + logger.Info(t, fmt.Sprintf("Worker IPs:%v", workerIPs)) + + return workerIPs, nil +} + +// LSFDaemonsStatus checks the status of LSF daemons (lim, res, sbatchd) on a remote server using SSH. +// It executes the 'lsf_daemons status' command and verifies if the expected status is 'running' for each daemon. +// It returns an error on command execution failure or if any daemon is not in the 'running' state. +func LSFDaemonsStatus(t *testing.T, sClient *ssh.Client, logger *utils.AggregatedLogger) error { + + // expectedStatus is the status that each daemon should have (in this case, 'running'). + expectedStatus := "running" + + // i is used as an index to track the current daemon being checked. + i := 0 + + // Execute the 'lsf_daemons status' command to get the daemons status + output, err := utils.RunCommandInSSHSession(sClient, "lsf_daemons status") + if err != nil { + return fmt.Errorf("failed to execute 'lsf_daemons status' command: %w", err) + } + + // Check if lim, res, and sbatchd are running + processes := []string{"lim", "res", "sbatchd"} + scanner := bufio.NewScanner(strings.NewReader(output)) + for scanner.Scan() { + line := scanner.Text() + if utils.VerifyDataContains(t, line, "pid", logger) { + if !(utils.VerifyDataContains(t, line, processes[i], logger) && utils.VerifyDataContains(t, line, expectedStatus, logger)) { + return fmt.Errorf("%s is not running", processes[i]) + } + i++ + } + } + // Log success if no errors occurred + logger.Info(t, "All LSF daemons are running") + return nil +} + +// LSFCheckHyperthreading checks if hyperthreading is enabled on the system by +// inspecting the output of the 'lscpu' command via an SSH session. +// It returns true if hyperthreading is enabled, false if it's disabled, and an error if +// there's an issue running the command or parsing the output. +func LSFCheckHyperthreading(t *testing.T, sClient *ssh.Client, expectedHyperthreadingStatus bool, logger *utils.AggregatedLogger) error { + + // Run the 'lscpu' command to retrieve CPU information + command := "lscpu" + + cpuInfo, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + return fmt.Errorf("failed to run '%s' command: %w", command, err) + } + + // Convert the command output to a string + content := string(cpuInfo) + + // Check if there is an "Off-line CPU(s) list:" indicating hyperthreading is disabled + hasOfflineCPU := utils.VerifyDataContains(t, content, "Off-line CPU(s) list:", logger) + + // Determine the actual hyperthreading status + var actualHyperthreadingStatus bool + if !hasOfflineCPU { + logger.Info(t, "Hyperthreading is enabled") + actualHyperthreadingStatus = true + } else { + logger.Info(t, "Hyperthreading is disabled") + actualHyperthreadingStatus = false + } + + // Compare actual and expected hyperthreading status + if actualHyperthreadingStatus != expectedHyperthreadingStatus { + return fmt.Errorf("hyperthreading status mismatch: expected %t, got %t", expectedHyperthreadingStatus, actualHyperthreadingStatus) + } + return nil +} + +// runSSHCommandAndGetPaths executes an SSH command to discover authorized_keys files +// across the filesystem and returns a list of their absolute paths. +func runSSHCommandAndGetPaths(sClient *ssh.Client) ([]string, error) { + sshKeyCheckCmd := "sudo su -l root -c 'cd / && find / -name authorized_keys'" + var err error + // Retry logic + for attempt := 0; attempt < 3; attempt++ { + output, err := utils.RunCommandInSSHSession(sClient, sshKeyCheckCmd) + if err == nil { + return strings.Split(strings.TrimSpace(output), "\n"), nil + } + + time.Sleep(30 * time.Second) + } + + return nil, fmt.Errorf("failed to run '%s' command: %w", sshKeyCheckCmd, err) +} + +// LSFCheckSSHKeyForManagementNode checks the SSH key configurations on a management server. +// It retrieves a list of authorized_keys paths, compares them with expected paths, +// and validates the occurrences of SSH keys in each path. +func LSFCheckSSHKeyForManagementNode(t *testing.T, sClient *ssh.Client, logger *utils.AggregatedLogger) error { + // Run a command to get a list of authorized_keys paths + pathList, err := runSSHCommandAndGetPaths(sClient) + if err != nil { + return fmt.Errorf("failed to retrieve authorized_keys paths: %w", err) + } + + // Log the list of authorized_keys paths + logger.Info(t, fmt.Sprintf("List of authorized_keys paths: %q", pathList)) + + // Create a map with paths as keys and expected values + filePathMap := map[string]int{ + "/home/vpcuser/.ssh/authorized_keys": 1, + "/home/lsfadmin/.ssh/authorized_keys": 2, + "/root/.ssh/authorized_keys": 2, + } + + // Iterate through the list of paths and check SSH key occurrences + for _, path := range pathList { + cmd := fmt.Sprintf("sudo su -l root -c 'cat %s'", path) + out, err := utils.RunCommandInSSHSession(sClient, cmd) + if err != nil { + return fmt.Errorf("failed to run command on %s: %w", path, err) + } + + // Log information about SSH key occurrences + value := filePathMap[path] + occur := utils.CountStringOccurrences(out, "ssh-rsa ") + logger.Info(t, fmt.Sprintf("Value: %d, Occurrences: %d, Path: %s", value, occur, path)) + + // Check for mismatch in occurrences + if value != occur { + return fmt.Errorf("mismatch in occurrences for path %s: expected %d, got %d", path, value, occur) + } + } + + // Log success for SSH key check + logger.Info(t, "SSH key check successful") + return nil +} + +// LSFCheckSSHKeyForManagementNodes checks SSH key configurations for each management node in the provided list. +// It ensures the expected paths and occurrences of SSH keys are consistent. +func LSFCheckSSHKeyForManagementNodes(t *testing.T, publicHostName, publicHostIP, privateHostName string, managementNodeIPList []string, logger *utils.AggregatedLogger) error { + // Check if the node list is empty + if len(managementNodeIPList) == 0 { + return fmt.Errorf("management node IPs cannot be empty") + } + + for _, mgmtIP := range managementNodeIPList { + // Connect to the management node via SSH + mgmtSshClient, connectionErr := utils.ConnectToHost(publicHostName, publicHostIP, privateHostName, mgmtIP) + if connectionErr != nil { + return fmt.Errorf("failed to connect to the management node %s via SSH: %v", mgmtIP, connectionErr) + } + defer mgmtSshClient.Close() + + logger.Info(t, fmt.Sprintf("SSH connection to the management node %s successful", mgmtIP)) + + // SSH key check for management node + sshKeyErr := LSFCheckSSHKeyForManagementNode(t, mgmtSshClient, logger) + if sshKeyErr != nil { + return fmt.Errorf("management node %s SSH key check failed: %v", mgmtIP, sshKeyErr) + } + } + + return nil +} + +// LSFCheckSSHKeyForComputeNode checks the SSH key configurations on a compute server. +// It considers OS variations, retrieves a list of authorized_keys paths, and validates SSH key occurrences. +func LSFCheckSSHKeyForComputeNode(t *testing.T, sClient *ssh.Client, computeIP string, logger *utils.AggregatedLogger) error { + + // authorizedKeysCmd is the command to find authorized_keys files. + authorizedKeysCmd := "sudo su -l root -c 'cd / && find / -name authorized_keys'" + // catAuthorizedKeysCmd is the command to concatenate authorized_keys files. + catAuthorizedKeysCmd := "sudo su -l root -c 'cat %s'" + expectedUbuntuPaths := 4 + expectedNonUbuntuPaths := 3 + + var expectedAuthorizedKeysPaths int + + // Get the OS name of the compute node. + osName, osNameErr := GetOSNameOfNode(t, sClient, computeIP, logger) + if osNameErr != nil { + // Determine the expected number of authorized_keys paths based on the OS. + switch osName { + case "Ubuntu": + expectedAuthorizedKeysPaths = expectedUbuntuPaths + default: + expectedAuthorizedKeysPaths = expectedNonUbuntuPaths + } + } else { + // Return error if OS name retrieval fails. + return osNameErr + } + + // Construct the SSH command to check authorized_keys paths. + sshKeyCheckCmd := fmt.Sprintf("ssh %s \"%s\"", computeIP, authorizedKeysCmd) + + // Run the SSH command to get the list of authorized_keys paths. + output, err := utils.RunCommandInSSHSession(sClient, sshKeyCheckCmd) + if err != nil { + return fmt.Errorf("failed to run '%s' command: %w", sshKeyCheckCmd, err) + } + + // Split the output into a list of authorized_keys paths. + pathList := strings.Split(strings.TrimSpace(output), "\n") + + logger.Info(t, fmt.Sprintf("List of authorized_keys paths: %q", pathList)) + + // Check if the number of authorized_keys paths matches the expected value. + if len(pathList) != expectedAuthorizedKeysPaths { + return fmt.Errorf("mismatch in the number of authorized_keys paths: expected %d, got %d", expectedAuthorizedKeysPaths, len(pathList)) + } + + // Create a map with paths as keys and set specific values for certain paths. + filePathMap := map[string]int{ + "/home/lsfadmin/.ssh/authorized_keys": 1, + } + + // Iterate through the list of paths and check SSH key occurrences. + for filePath := range filePathMap { + cmd := fmt.Sprintf("ssh %s \"%s\"", computeIP, fmt.Sprintf(catAuthorizedKeysCmd, filePath)) + out, err := utils.RunCommandInSSHSession(sClient, cmd) + if err != nil { + return fmt.Errorf("failed to run '%s' command: %w", cmd, err) + } + + // Log information about SSH key occurrences. + expectedOccurrences := filePathMap[filePath] + actualOccurrences := utils.CountStringOccurrences(out, "ssh-rsa ") + + // Check for mismatch in occurrences. + if expectedOccurrences != actualOccurrences { + return fmt.Errorf("mismatch in occurrences for path %s: expected %d, got %d", filePath, expectedOccurrences, actualOccurrences) + } + } + + // Log success for SSH key check. + logger.Info(t, "SSH key check success") + return nil +} + +// LSFCheckSSHKeyForComputeNodes checks SSH key configurations for each compute node in the provided list. +// It validates the expected paths and occurrences of SSH keys. +func LSFCheckSSHKeyForComputeNodes(t *testing.T, sClient *ssh.Client, computeNodeIPList []string, logger *utils.AggregatedLogger) error { + // Check if the node list is empty + if len(computeNodeIPList) == 0 { + return fmt.Errorf("ERROR: compute node IPs cannot be empty") + } + + for _, compIP := range computeNodeIPList { + // SSH key check for compute node + sshKeyErr := LSFCheckSSHKeyForComputeNode(t, sClient, compIP, logger) + if sshKeyErr != nil { + return fmt.Errorf("compute node %s SSH key check failed: %v", compIP, sshKeyErr) + } + } + + return nil +} + +// CheckLSFVersion verifies if the IBM Spectrum LSF version on the cluster matches the expected version. +// It executes the 'lsid' command, retrieves the cluster ID, and compares it with the expected version. +func CheckLSFVersion(t *testing.T, sClient *ssh.Client, expectedVersion string, logger *utils.AggregatedLogger) error { + + // Execute the 'lsid' command to get the cluster ID + command := "lsid" + + output, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + // Handle the error when executing the 'lsid' command + return fmt.Errorf("failed to execute 'lsid' command: %w", err) + } + + // Verify if the expected cluster ID is present in the output + if !utils.VerifyDataContains(t, output, "IBM Spectrum LSF Standard "+expectedVersion, logger) { + // Extract actual cluster version from the output for better error reporting + actualValue := strings.TrimSpace(strings.Split(strings.Split(output, "IBM Spectrum LSF Standard")[1], ", ")[0]) + return fmt.Errorf("expected cluster Version %s, but found %s", expectedVersion, actualValue) + } + + // Log information when the cluster version is set as expected + logger.Info(t, fmt.Sprintf("Cluster Version is set as expected: %s", expectedVersion)) + // No errors occurred + return nil +} + +// IsDynamicNodeAvailable checks if a dynamic node is available by running +// the 'bhosts -w' command on the remote SSH server and verifying the output. +// It returns true if the output contains the string 'ok', indicating the node is available. +func IsDynamicNodeAvailable(t *testing.T, sClient *ssh.Client, logger *utils.AggregatedLogger) (bool, error) { + + // Run 'bhosts -w' command on the remote SSH server + command := "bhosts -w" + output, err := utils.RunCommandInSSHSession(sClient, command) + + if err != nil { + //return if the command execution fails + return false, fmt.Errorf("failed to run SSH command '%s': %w", command, err) + } + + // Return true if the output contains 'ok' + if utils.VerifyDataContains(t, output, "ok", logger) { + return true, nil + } + + // Return false if the output not contains 'ok' + return false, nil +} + +// GetOSNameOfNode retrieves the OS name of a remote node using SSH. +// It takes a testing.T instance for error reporting, an SSH client, the IP address of the remote node, +// and a logger for additional logging. It returns the OS name and an error if any. +func GetOSNameOfNode(t *testing.T, sClient *ssh.Client, hostIP string, logger *utils.AggregatedLogger) (string, error) { + // Command to retrieve the content of /etc/os-release on the remote server + //catOsReleaseCmd := "sudo su -l root -c 'cat /etc/os-release'" + catOsReleaseCmd := "cat /etc/os-release" + // Construct the SSH command + OsReleaseCmd := fmt.Sprintf("ssh %s \"%s\"", hostIP, catOsReleaseCmd) + output, err := utils.RunCommandInSSHSession(sClient, OsReleaseCmd) + if err != nil { + // Report an error and fail the test + t.Fatal("Error executing SSH command:", err) + } + + // Parse the OS name from the /etc/os-release content + osName, parseErr := utils.ParsePropertyValue(strings.TrimSpace(string(output)), "NAME") + if parseErr != nil { + // Log information about the OS installation on the specified node. + logger.Info(t, fmt.Sprintf("Operating System: %s, Installed on Node: %s", osName, hostIP)) + + // Return the parsed OS name on success + return osName, parseErr + } + + // If parsing fails, return the error + return "", parseErr +} + +// HPCCheckFileMount checks if essential LSF directories (conf, config_done, das_staging_area, gui-conf, gui-logs, log, repository-path and work) exist +// on remote machines identified by the provided list of IP addresses. It utilizes SSH to +// query and validate the directories. Any missing directory triggers an error, and the +// function logs the success message if all directories are found. +func HPCCheckFileMount(t *testing.T, sClient *ssh.Client, ipsList []string, nodeType string, logger *utils.AggregatedLogger) error { + // Define constants + const ( + sampleText = "Welcome to the ibm cloud HPC" + SampleFileName = "testOne.txt" + ) + + // Check if the node list is empty + if len(ipsList) == 0 { + return fmt.Errorf("ERROR: ips cannot be empty") + } + + // Iterate over each IP in the list + for _, ip := range ipsList { + // Run SSH command to get file system information + commandOne := fmt.Sprintf("ssh %s 'df -h'", ip) + outputOne, err := utils.RunCommandInSSHSession(sClient, commandOne) + if err != nil { + return fmt.Errorf("failed to run %s command on machine IP %s: %w", commandOne, ip, err) + } + actualMount := strings.TrimSpace(string(outputOne)) + + // Check if it's not a login node + if !(strings.Contains(strings.ToLower(nodeType), "login")) { + // Define expected file system mounts + expectedMount := []string{"/mnt/lsf", "/mnt/vpcstorage/tools", "/mnt/vpcstorage/data"} + + // Check if all expected mounts exist + for _, mount := range expectedMount { + if !utils.VerifyDataContains(t, actualMount, mount, logger) { + return fmt.Errorf("actual filesystem '%v' does not match the expected filesystem '%v' for node IP '%s'", actualMount, expectedMount, ip) + } + } + + // Log filesystem existence + logger.Info(t, fmt.Sprintf("Filesystems [/mnt/lsf, /mnt/vpcstorage/tools,/mnt/vpcstorage/data] exist on the node %s", ip)) + + // Verify essential directories existence + if err := verifyDirectories(t, sClient, ip, logger); err != nil { + return err + } + + // Create, read, verify and delete sample files in each mount + for i := 1; i < len(expectedMount); i++ { + // Create file + _, fileCreationErr := utils.ToCreateFileWithContent(t, sClient, expectedMount[i], SampleFileName, sampleText, logger) + if fileCreationErr != nil { + return fmt.Errorf("failed to create file on %s for machine IP %s: %w", expectedMount[i], ip, fileCreationErr) + } + + // Read file + actualText, fileReadErr := utils.ReadRemoteFileContents(t, sClient, expectedMount[i], SampleFileName, logger) + if fileReadErr != nil { + // Delete file if reading fails + _, fileDeletionErr := utils.ToDeleteFile(t, sClient, expectedMount[i], SampleFileName, logger) + if fileDeletionErr != nil { + return fmt.Errorf("failed to delete %s file on machine IP %s: %w", SampleFileName, ip, fileDeletionErr) + } + return fmt.Errorf("failed to read %s file content on %s machine IP %s: %w", SampleFileName, expectedMount[i], ip, fileReadErr) + } + + // Verify file content + if !utils.VerifyDataContains(t, actualText, sampleText, logger) { + return fmt.Errorf("%s actual file content '%v' does not match the file content '%v' for node IP '%s'", SampleFileName, actualText, sampleText, ip) + } + + // Delete file after verification + _, fileDeletionErr := utils.ToDeleteFile(t, sClient, expectedMount[i], SampleFileName, logger) + if fileDeletionErr != nil { + return fmt.Errorf("failed to delete %s file on machine IP %s: %w", SampleFileName, ip, fileDeletionErr) + } + } + } else { + // For login nodes, only /mnt/lsf is checked + expectedMount := "/mnt/lsf" + + // Verify /mnt/lsf existence + if !utils.VerifyDataContains(t, actualMount, expectedMount, logger) { + return fmt.Errorf("actual filesystem '%v' does not match the expected filesystem '%v' for node IP '%s'", actualMount, expectedMount, ip) + } + + // Log /mnt/lsf existence + logger.Info(t, fmt.Sprintf("Filesystems /mnt/lsf exist on the node %s", ip)) + + // Verify essential directories existence + if err := verifyDirectories(t, sClient, ip, logger); err != nil { + return err + } + } + } + // Log success if no errors occurred + logger.Info(t, fmt.Sprintf("File mount check has been successfully completed for %s", nodeType)) + // No errors occurred + return nil +} + +// verifyDirectories verifies the existence of essential directories in /mnt/lsf on the remote machine. +func verifyDirectories(t *testing.T, sClient *ssh.Client, ip string, logger *utils.AggregatedLogger) error { + // Run SSH command to list directories in /mnt/lsf + commandTwo := fmt.Sprintf("ssh %s 'cd /mnt/lsf && ls'", ip) + outputTwo, err := utils.RunCommandInSSHSession(sClient, commandTwo) + if err != nil { + return fmt.Errorf("failed to run %s command on machine IP %s: %w", commandTwo, ip, err) + } + // Split the output into directory names + actualDirs := strings.Fields(strings.TrimSpace(string(outputTwo))) + // Define expected directories + expectedDirs := []string{"conf", "config_done", "das_staging_area", "gui-conf", "gui-logs", "log", "repository-path", "work"} + + // Verify if all expected directories exist + if !utils.VerifyDataContains(t, actualDirs, expectedDirs, logger) { + return fmt.Errorf("actual directory '%v' does not match the expected directory '%v' for node IP '%s'", actualDirs, expectedDirs, ip) + } + + // Log directories existence + logger.Info(t, fmt.Sprintf("Directories [conf, config_done, das_staging_area, gui-conf, gui-logs, log, repository-path and work] exist on %s", ip)) + return nil +} + +// VerifyTerraformOutputs verifies specific fields in the Terraform outputs and ensures they are not empty based on the provided LastTestTerraformOutputs. +// Additional checks are performed for the application center and LDAP server based on the isAPPCenterEnabled and ldapServerEnabled flags. +// Any missing essential field results in an error being returned with detailed information. +func VerifyTerraformOutputs(t *testing.T, LastTestTerraformOutputs map[string]interface{}, isAPPCenterEnabled, ldapServerEnabled bool, logger *utils.AggregatedLogger) error { + + fields := []string{"ssh_to_management_node", "ssh_to_login_node", "vpc_name", "region_name"} + actualOutput := make(map[string]interface{}) + maps.Copy(actualOutput, LastTestTerraformOutputs["cluster_info"].(map[string]interface{})) + for _, field := range fields { + value := actualOutput[field].(string) + logger.Info(t, field+" = "+value) + if len(value) == 0 { + return fmt.Errorf("%s is missing terraform output", field) + } + } + + if isAPPCenterEnabled { + if len(strings.TrimSpace(actualOutput["application_center"].(string))) == 0 { + return errors.New("application_center is missing from terraform output") + + } + if len(strings.TrimSpace(actualOutput["application_center_url"].(string))) == 0 { + return errors.New("application_center_url is missing from terraform output") + } + } + + if ldapServerEnabled { + if len(strings.TrimSpace(actualOutput["ssh_to_ldap_node"].(string))) == 0 { + return fmt.Errorf("ssh_to_ldap_node is missing from terraform output") + + } + } + // Log success if no errors occurred + logger.Info(t, "Terraform output check has been successfully completed") + // No errors occurred + return nil + +} + +// LSFCheckSSHConnectivityToNodesFromLogin checks SSH connectivity from the login node to other nodes. +func LSFCheckSSHConnectivityToNodesFromLogin(t *testing.T, sshLoginClient *ssh.Client, managementNodeIPList, computeNodeIPList []string, logger *utils.AggregatedLogger) error { + + // Check if management node IP list is empty + if len(managementNodeIPList) == 0 { + return fmt.Errorf("ERROR: management node IPs cannot be empty") + } + + // Iterate over each management node IP in the list + for _, managementNodeIP := range managementNodeIPList { + // Run SSH command to get the hostname of the management node + command := fmt.Sprintf("ssh %s 'hostname'", managementNodeIP) + actualOutput, err := utils.RunCommandInSSHSession(sshLoginClient, command) + if err != nil { + return fmt.Errorf("failed to run SSH command on management node IP %s: %w", managementNodeIP, err) + } + // Check if the hostname contains "mgmt" substring + if !utils.VerifyDataContains(t, actualOutput, "mgmt", logger) { + return fmt.Errorf("compute node '%v' does not contain 'mgmt' substring for node IP '%s'", actualOutput, managementNodeIP) + } + } + + // Check if compute node IP list is empty + if len(computeNodeIPList) == 0 { + return fmt.Errorf("ERROR: compute node IPs cannot be empty") + } + // Iterate over each compute node IP in the list + for _, computeNodeIP := range computeNodeIPList { + // Run a simple SSH command to check connectivity + command := fmt.Sprintf("ssh -o ConnectTimeout=12 -q %s exit", computeNodeIP) + _, err := utils.RunCommandInSSHSession(sshLoginClient, command) + if err != nil { + return fmt.Errorf("failed to run SSH command on compute node IP %s: %w", computeNodeIP, err) + } + + } + // Log success if no errors occurred + logger.Info(t, "SSH connectivity check from login node to other nodes completed successfully") + // No errors occurred + return nil +} + +// HPCCheckNoVNC checks if NO VNC is properly configured on a remote machine. +// It executes a series of commands via SSH and verifies the expected output. +func HPCCheckNoVNC(t *testing.T, sClient *ssh.Client, logger *utils.AggregatedLogger) error { + // Define commands to be executed and their expected outputs + commands := map[string]string{ + "rpm -qa | grep xterm": "xterm", + "rpm -qa | grep tigervnc": "tigervnc", + "ps aux | grep -i novnc": "-Ddefault.novnc.port=6080", + "netstat -tuln | grep 6080": "0.0.0.0:6080", + } + + // Iterate over commands + for command, expectedOutput := range commands { + // Execute command on SSH session + output, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + return fmt.Errorf("failed to execute command '%s' via SSH: %v", command, err) + } + + // Verify if the expected output is present in the actual output + if !utils.VerifyDataContains(t, output, expectedOutput, logger) { + return fmt.Errorf("NO VNC is not set as expected for command '%s'. Expected: '%s'. Actual: '%s'", command, expectedOutput, output) + + } + } + + // Log success if no errors occurred + logger.Info(t, "NO VNC configuration set as expected") + + // No errors occurred + return nil +} + +// GetJobCommand returns the appropriate job command based on the provided job type and zone. +// zones: a string indicating the zone (e.g., "us-south"). +// jobType: a string indicating the type of job (e.g., "low", "med", "high"). +// Returns: a string representing the job command. +func GetJobCommand(zone, jobType string) string { + // Define job command constants + var lowMem, medMem, highMem string + if strings.Contains(zone, "us-south") { + lowMem = JOB_COMMAND_LOW_MEM_SOUTH + medMem = JOB_COMMAND_MED_MEM_SOUTH + highMem = JOB_COMMAND_HIGH_MEM_SOUTH + } else { + lowMem = JOB_COMMAND_LOW_MEM + medMem = JOB_COMMAND_MED_MEM + highMem = JOB_COMMAND_HIGH_MEM + } + + // Select appropriate job command based on job type + switch strings.ToLower(jobType) { + case "low": + return lowMem + case "med": + return medMem + case "high": + return highMem + default: + // Default to low job command if job type is not recognized + return lowMem + } +} + +// VerifyEncryption checks if encryption is enabled for file shares. +// It logs into IBM Cloud using the API key and VPC region, retrieves the list of file shares +// with the specified cluster prefix, and verifies encryption settings. +func VerifyEncryption(t *testing.T, apiKey, region, resourceGroup, clusterPrefix, keyManagement string, logger *utils.AggregatedLogger) error { + + // Login to IBM Cloud using the API key and VPC region + if err := utils.LoginIntoIBMCloudUsingCLI(t, apiKey, region, resourceGroup); err != nil { + return fmt.Errorf("failed to log in to IBM Cloud: %w", err) + } + + // Get the list of file shares + fileSharesCmd := fmt.Sprintf("ibmcloud is shares | grep %s | awk '{print $2}'", clusterPrefix) + cmd := exec.Command("bash", "-c", fileSharesCmd) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to retrieve file shares: %w", err) + } + + fileShareNames := strings.Fields(string(output)) + logger.Info(t, fmt.Sprintf("File share list: %s", fileShareNames)) + + for _, fileShareName := range fileShareNames { + fileShareCmd := exec.Command("ibmcloud", "is", "share", fileShareName) + output, err := fileShareCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to retrieve file share details for '%s': %w", fileShareName, err) + } + + if !utils.VerifyDataContains(t, strings.ToLower(keyManagement), "key_protect", logger) { + if !utils.VerifyDataContains(t, string(output), "provider_managed", logger) { + return fmt.Errorf("encryption in transit is unexpectedly enabled for the file shares ") + } + } else { + if !utils.VerifyDataContains(t, string(output), "user_managed", logger) { + return fmt.Errorf("encryption in transit is unexpectedly disabled for the file shares") + } + } + } + logger.Info(t, "Encryption set as expected") + return nil +} + +// ValidateRequiredEnvironmentVariables checks if the required environment variables are set and valid +func ValidateRequiredEnvironmentVariables(envVars map[string]string) error { + requiredVars := []string{"SSH_FILE_PATH", "SSH_KEY", "CLUSTER_ID", "ZONE", "RESERVATION_ID"} + for _, fieldName := range requiredVars { + fieldValue, ok := envVars[fieldName] + if !ok || fieldValue == "" { + return fmt.Errorf("missing required environment variable: %s", fieldName) + } + } + + if _, err := os.Stat(envVars["SSH_FILE_PATH"]); os.IsNotExist(err) { + return fmt.Errorf("SSH private key file '%s' does not exist", envVars["SSH_FILE_PATH"]) + } else if err != nil { + return fmt.Errorf("error checking SSH private key file: %v", err) + } + + return nil +} + +// LSFRunJobsAsLDAPUser executes an LSF job on a remote server via SSH, monitors its status, +// and ensures its completion or terminates it if it exceeds a specified timeout. +// It returns an error if any step of the process fails. +func LSFRunJobsAsLDAPUser(t *testing.T, sClient *ssh.Client, jobCmd, ldapUser string, logger *utils.AggregatedLogger) error { + // Set the maximum timeout for the job execution + var jobMaxTimeout time.Duration + + // Record the start time of the job execution + startTime := time.Now() + + // Run the LSF job command on the remote server + jobOutput, err := utils.RunCommandInSSHSession(sClient, jobCmd) + if err != nil { + return fmt.Errorf("failed to run '%s' command: %w", jobCmd, err) + } + + // Extract the job ID from the job output + jobTime := utils.SplitAndTrim(jobCmd, "sleep")[1] + min, err := utils.StringToInt(jobTime) + if err != nil { + return err + } + min = 300 + min + jobMaxTimeout = time.Duration(min) * time.Second + + // Log the job output for debugging purposes + logger.Info(t, strings.TrimSpace(string(jobOutput))) + + // Extract the job ID from the job output + jobID, err := LSFExtractJobID(jobOutput) + if err != nil { + return err + } + + // Monitor the job's status until it completes or exceeds the timeout + for time.Since(startTime) < jobMaxTimeout { + + // Run 'bjobs -a' command on the remote SSH server + command := "bjobs -a" + + // Run the 'bjobs' command to get information about all jobs + jobStatus, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + return fmt.Errorf("failed to run 'bjobs' command: %w", err) + } + + // Create a regular expression pattern to match the job ID and status + pattern := regexp.MustCompile(fmt.Sprintf(`\b%s\s+%s\s+DONE`, jobID, ldapUser)) + + // Check if the job ID appears in the 'bjobs' response with a status of 'DONE' + if pattern.MatchString(jobStatus) { + logger.Info(t, fmt.Sprintf("Job results : \n%s", jobStatus)) + logger.Info(t, fmt.Sprintf("Job %s has executed successfully", jobID)) + return nil + } + + // Sleep for a minute before checking again + logger.Info(t, fmt.Sprintf("Waiting for dynamic node creation and job completion. Elapsed time: %s", time.Since(startTime))) + time.Sleep(jobCompletionWaitTime) + } + + // If the job exceeds the specified timeout, attempt to terminate it + _, err = utils.RunCommandInSSHSession(sClient, fmt.Sprintf("bkill %s", jobID)) + if err != nil { + return fmt.Errorf("failed to run 'bkill' command: %w", err) + } + + // Return an error indicating that the job execution exceeded the specified time + return fmt.Errorf("job execution for ID %s exceeded the specified time", jobID) +} + +// HPCCheckFileMountAsLDAPUser checks if essential LSF directories (conf, config_done, das_staging_area, gui-conf, gui-logs, log, repository-path and work) exist +// on remote machines It utilizes SSH to +// query and validate the directories. Any missing directory triggers an error, and the +// function logs the success message if all directories are found. +func HPCCheckFileMountAsLDAPUser(t *testing.T, sClient *ssh.Client, nodeType string, logger *utils.AggregatedLogger) error { + // Define constants + const ( + sampleText = "Welcome to the ibm cloud HPC" + SampleFileName = "testOne.txt" + ) + + hostname, err := utils.RunCommandInSSHSession(sClient, "hostname") + if err != nil { + return fmt.Errorf("failed to run hostname command: %w", err) + } + + commandOne := "df -h" + outputOne, err := utils.RunCommandInSSHSession(sClient, commandOne) + if err != nil { + return fmt.Errorf("failed to run %s command: %w", commandOne, err) + } + actualMount := strings.TrimSpace(string(outputOne)) + + if !strings.Contains(strings.ToLower(nodeType), "login") { + expectedMount := []string{"/mnt/lsf", "/mnt/vpcstorage/tools", "/mnt/vpcstorage/data"} + for _, mount := range expectedMount { + if !utils.VerifyDataContains(t, actualMount, mount, logger) { + return fmt.Errorf("actual filesystem '%v' does not match the expected filesystem '%v' for node %s", actualMount, expectedMount, hostname) + } + } + logger.Info(t, fmt.Sprintf("Filesystems [/mnt/lsf, /mnt/vpcstorage/tools,/mnt/vpcstorage/data] exist on the node %s", hostname)) + + if err := verifyDirectoriesAsLdapUser(t, sClient, hostname, logger); err != nil { + return err + } + + for i := 1; i < len(expectedMount); i++ { + _, fileCreationErr := utils.ToCreateFileWithContent(t, sClient, expectedMount[i], SampleFileName, sampleText, logger) + if fileCreationErr != nil { + return fmt.Errorf("failed to create file on %s for machine %s: %w", expectedMount[i], hostname, fileCreationErr) + } + + actualText, fileReadErr := utils.ReadRemoteFileContents(t, sClient, expectedMount[i], SampleFileName, logger) + if fileReadErr != nil { + _, fileDeletionErr := utils.ToDeleteFile(t, sClient, expectedMount[i], SampleFileName, logger) + if fileDeletionErr != nil { + return fmt.Errorf("failed to delete %s file on machine %s: %w", SampleFileName, hostname, fileDeletionErr) + } + return fmt.Errorf("failed to read %s file content on %s machine %s: %w", SampleFileName, expectedMount[i], hostname, fileReadErr) + } + + if !utils.VerifyDataContains(t, actualText, sampleText, logger) { + return fmt.Errorf("%s actual file content '%v' does not match the file content '%v' for node %s", SampleFileName, actualText, sampleText, hostname) + } + + _, fileDeletionErr := utils.ToDeleteFile(t, sClient, expectedMount[i], SampleFileName, logger) + if fileDeletionErr != nil { + return fmt.Errorf("failed to delete %s file on machine %s: %w", SampleFileName, hostname, fileDeletionErr) + } + } + } else { + expectedMount := "/mnt/lsf" + if !utils.VerifyDataContains(t, actualMount, expectedMount, logger) { + return fmt.Errorf("actual filesystem '%v' does not match the expected filesystem '%v' for node %s", actualMount, expectedMount, hostname) + } + logger.Info(t, fmt.Sprintf("Filesystems /mnt/lsf exist on the node %s", hostname)) + + if err := verifyDirectoriesAsLdapUser(t, sClient, hostname, logger); err != nil { + return err + } + } + + logger.Info(t, fmt.Sprintf("File mount check has been successfully completed for %s", nodeType)) + return nil +} + +// verifyDirectoriesAsLdapUser verifies the existence of essential directories in /mnt/lsf on the remote machine. +func verifyDirectoriesAsLdapUser(t *testing.T, sClient *ssh.Client, hostname string, logger *utils.AggregatedLogger) error { + // Run SSH command to list directories in /mnt/lsf + commandTwo := "cd /mnt/lsf && ls" + outputTwo, err := utils.RunCommandInSSHSession(sClient, commandTwo) + if err != nil { + return fmt.Errorf("failed to run %s command on machine IP %s: %w", commandTwo, hostname, err) + } + // Split the output into directory names + actualDirs := strings.Fields(strings.TrimSpace(string(outputTwo))) + // Define expected directories + expectedDirs := []string{"conf", "config_done", "das_staging_area", "gui-conf", "gui-logs", "log", "repository-path", "work"} + + // Verify if all expected directories exist + if !utils.VerifyDataContains(t, actualDirs, expectedDirs, logger) { + return fmt.Errorf("actual directory '%v' does not match the expected directory '%v' for node IP '%s'", actualDirs, expectedDirs, hostname) + } + + // Log directories existence + logger.Info(t, fmt.Sprintf("Directories [conf, config_done, das_staging_area, gui-conf, gui-logs, log, repository-path and work] exist on %s", hostname)) + return nil +} + +// VerifyLSFCommands verifies the LSF commands on the remote machine. +func VerifyLSFCommands(t *testing.T, sClient *ssh.Client, userName string, logger *utils.AggregatedLogger) error { + // Define commands to be executed + commands := []string{ + "whoami", + "bhosts -w", + "bjobs -a", + "bhosts -w", + } + + // Iterate over commands + for _, command := range commands { + // Execute command on SSH session + output, err := utils.RunCommandInSSHSession(sClient, command) + if err != nil { + return fmt.Errorf("failed to execute command '%s' via SSH: %v", command, err) + } + if command == "whoami" { + if !utils.VerifyDataContains(t, strings.TrimSpace(output), userName, logger) { + return fmt.Errorf("unexpected user: expected '%s', got '%s'", userName, strings.TrimSpace(output)) + } + // Check if the output is not empty + } else if strings.TrimSpace(output) == "" { + return fmt.Errorf("output for command '%s' is empty", command) + } + } + + return nil +} + +// VerifyLDAPConfig verifies LDAP configuration on a remote machine by executing commands via SSH. +// It checks LDAP configuration files and performs an LDAP search to validate the configuration. +// Returns: Error if verification fails, nil otherwise. +func VerifyLDAPConfig(t *testing.T, sClient *ssh.Client, nodeType, ldapServerIP, ldapDomain, ldapUser string, logger *utils.AggregatedLogger) error { + // Perform an LDAP search to validate the configuration + ldapSearchCmd := fmt.Sprintf("ldapsearch -x -H ldap://%s -b dc=%s,dc=%s", ldapServerIP, strings.Split(ldapDomain, ".")[0], strings.Split(ldapDomain, ".")[1]) + ldapSearchActual, err := utils.RunCommandInSSHSession(sClient, ldapSearchCmd) + if err != nil { + return fmt.Errorf("failed to execute command '%s' via SSH: %v", ldapSearchCmd, err) + } + expected := fmt.Sprintf("dc=%s,dc=%s", strings.Split(ldapDomain, ".")[0], strings.Split(ldapDomain, ".")[1]) + if !utils.VerifyDataContains(t, ldapSearchActual, expected, logger) { + return fmt.Errorf("LDAP search failed: Expected '%s', got '%s'", expected, ldapSearchActual) + } + + // Verify the LDAP user exists in the search results + if !utils.VerifyDataContains(t, ldapSearchActual, "uid: "+ldapUser, logger) { + return fmt.Errorf("LDAP user %s not found in search results", ldapUser) + } + + logger.Info(t, fmt.Sprintf("%s LDAP configuration verification completed successfully.", nodeType)) + return nil +} + +// VerifyLDAPServerConfig verifies LDAP configuration on a remote machine by executing commands via SSH. +// It checks LDAP configuration files and performs an LDAP search to validate the configuration. +// Returns: Error if verification fails, nil otherwise. +func VerifyLDAPServerConfig(t *testing.T, sClient *ssh.Client, ldapAdminpassword, ldapDomain, ldapUser string, logger *utils.AggregatedLogger) error { + // Check LDAP configuration files + ldapConfigCheckCmd := "cat /etc/ldap/ldap.conf" + actual, err := utils.RunCommandInSSHSession(sClient, ldapConfigCheckCmd) + if err != nil { + return fmt.Errorf("failed to execute command '%s' via SSH: %v", ldapConfigCheckCmd, err) + } + expected := fmt.Sprintf("BASE dc=%s,dc=%s", strings.Split(ldapDomain, ".")[0], strings.Split(ldapDomain, ".")[1]) + if !utils.VerifyDataContains(t, actual, expected, logger) { + return fmt.Errorf("LDAP configuration check failed: Expected '%s', got '%s'", expected, actual) + } + + // Perform an LDAP search to validate the configuration + ldapSearchCmd := fmt.Sprintf("ldapsearch -x -D \"cn=admin,dc=%s,dc=%s\" -w %s -b \"ou=people,dc=%s,dc=%s\" -s sub \"(objectClass=*)\"", strings.Split(ldapDomain, ".")[0], strings.Split(ldapDomain, ".")[1], ldapAdminpassword, strings.Split(ldapDomain, ".")[0], strings.Split(ldapDomain, ".")[1]) + ldapSearchActual, err := utils.RunCommandInSSHSession(sClient, ldapSearchCmd) + if err != nil { + return fmt.Errorf("failed to execute command '%s' via SSH: %v", ldapSearchCmd, err) + } + expected = fmt.Sprintf("dc=%s,dc=%s", strings.Split(ldapDomain, ".")[0], strings.Split(ldapDomain, ".")[1]) + if !utils.VerifyDataContains(t, ldapSearchActual, expected, logger) { + return fmt.Errorf("LDAP search failed: Expected '%s', got '%s'", expected, ldapSearchActual) + } + + // Verify the LDAP user exists in the search results + if !utils.VerifyDataContains(t, ldapSearchActual, "uid: "+ldapUser, logger) { + return fmt.Errorf("LDAP user %s not found in search results", ldapUser) + } + + logger.Info(t, "LDAP Server configuration verification completed successfully.") + return nil +} + +// verifyPTRRecords verifies PTR records for 'mgmt' or 'login' nodes and ensures their resolution via SSH. +// It retrieves hostnames, performs nslookup to verify PTR records, and returns an error if any step fails. +func verifyPTRRecords(t *testing.T, sClient *ssh.Client, publicHostName, publicHostIP, privateHostName string, managementNodeIPList []string, loginNodeIP string, domainName string, logger *utils.AggregatedLogger) error { + // Slice to hold the list of hostnames + var hostNamesList []string + + // Check if the management node IP list is empty + if len(managementNodeIPList) == 0 { + return fmt.Errorf("management node IPs cannot be empty") + } + + // Execute the command to get the hostnames + hostNames, err := utils.RunCommandInSSHSession(sClient, "lshosts -w | awk 'NR>1' | awk '{print $1}' | grep -E 'mgmt|login'") + if err != nil { + return fmt.Errorf("failed to execute command to retrieve hostnames: %w", err) + } + + // Process the retrieved hostnames + for _, hostName := range strings.Split(strings.TrimSpace(hostNames), "\n") { + // Append domain name to hostnames if not already present + if !strings.Contains(hostName, domainName) { + hostNamesList = append(hostNamesList, hostName+"."+domainName) + } else { + hostNamesList = append(hostNamesList, hostName) + } + } + + // Function to perform nslookup and verify PTR records + verifyPTR := func(sshClient *ssh.Client, nodeDesc string) error { + for _, hostName := range hostNamesList { + // Execute nslookup command for the hostname + nsOutput, err := utils.RunCommandInSSHSession(sshClient, "nslookup "+hostName) + if err != nil { + return fmt.Errorf("failed to execute nslookup command for %s: %w", hostName, err) + } + + // Verify the PTR record existence in the search results + if utils.VerifyDataContains(t, nsOutput, "server can't find", logger) { + return fmt.Errorf("PTR record for %s not found in search results", hostName) + } + } + logger.Info(t, fmt.Sprintf("PTR Records for %s completed successfully.", nodeDesc)) + return nil + } + + // Iterate over management nodes + for _, mgmtIP := range managementNodeIPList { + // Connect to the management node via SSH + mgmtSshClient, connectionErr := utils.ConnectToHost(publicHostName, publicHostIP, privateHostName, mgmtIP) + if connectionErr != nil { + return fmt.Errorf("failed to connect to the management node %s via SSH: %v", mgmtIP, connectionErr) + } + defer mgmtSshClient.Close() + + // Verify PTR records on management node + if err := verifyPTR(mgmtSshClient, fmt.Sprintf("management node %s", mgmtIP)); err != nil { + return err + } + } + logger.Info(t, "Verify PTR Records for management nodes completed successfully.") + + // If login node IP is provided, verify PTR records on login node as well + if loginNodeIP != "" { + loginSshClient, connectionErr := utils.ConnectToHost(publicHostName, publicHostIP, privateHostName, loginNodeIP) + if connectionErr != nil { + return fmt.Errorf("failed to connect to the login node %s via SSH: %v", loginNodeIP, connectionErr) + } + defer loginSshClient.Close() + + // Verify PTR records on login node + if err := verifyPTR(loginSshClient, fmt.Sprintf("login node %s", loginNodeIP)); err != nil { + return err + } + } + + logger.Info(t, "Verify PTR Records for login node completed successfully.") + logger.Info(t, "Verify PTR Records completed successfully.") + return nil +} + +// CreateServiceInstanceAndReturnGUID creates a service instance on IBM Cloud, verifies its creation, and retrieves the service instance ID. +// It logs into IBM Cloud using the provided API key, region, and resource group, then creates the service instance +// with the specified instance name. If the creation is successful, it retrieves and returns the service instance ID. +// Returns: +// - string: service instance ID if successful +// - error: error if any step fails +func CreateServiceInstanceAndReturnGUID(t *testing.T, apiKey, region, resourceGroup, instanceName string, logger *utils.AggregatedLogger) (string, error) { + // Log in to IBM Cloud using the API key and region + if err := utils.LoginIntoIBMCloudUsingCLI(t, apiKey, region, resourceGroup); err != nil { + return "", fmt.Errorf("failed to log in to IBM Cloud: %w", err) + } + + // Create the service instance + createServiceInstanceCmd := fmt.Sprintf("ibmcloud resource service-instance-create %s kms tiered-pricing %s", instanceName, region) + cmdCreate := exec.Command("bash", "-c", createServiceInstanceCmd) + createOutput, err := cmdCreate.CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to create service instance: %w", err) + } + + // Verify that the service instance was created successfully + expectedMessage := fmt.Sprintf("Service instance %s was created", instanceName) + if !utils.VerifyDataContains(t, string(createOutput), expectedMessage, logger) { + return "", fmt.Errorf("service instance creation failed: %s", string(createOutput)) + } + + // Extract and return the service instance ID + serviceInstanceID := strings.TrimSpace(strings.Split(strings.Split(string(createOutput), "GUID:")[1], "Location:")[0]) + if len(serviceInstanceID) == 0 { + return "", fmt.Errorf("service instance ID not found") + } + + logger.Info(t, fmt.Sprintf("Service Instance '%s' created successfully. Instance ID: %s", instanceName, serviceInstanceID)) + return serviceInstanceID, nil +} + +// DeleteServiceInstance deletes a service instance on IBM Cloud and its associated keys, and verifies the deletion. +// It logs into IBM Cloud using the provided API key, region, and resource group, then deletes the service instance +// with the specified instance name. If the deletion is successful, it logs the success and returns nil; +// otherwise, it returns an error. +// Returns: +// - error: error if any step fails +func DeleteServiceInstance(t *testing.T, apiKey, region, resourceGroup, instanceName string, logger *utils.AggregatedLogger) error { + + // Log in to IBM Cloud using the API key and region + if err := utils.LoginIntoIBMCloudUsingCLI(t, apiKey, region, resourceGroup); err != nil { + return fmt.Errorf("failed to log in to IBM Cloud: %w", err) + } + + // Retrieve the service instance GUID + retrieveServiceCmd := fmt.Sprintf("ibmcloud resource service-instance --service-name %s --output", instanceName) + cmdRetrieveGUID := exec.Command("bash", "-c", retrieveServiceCmd) + retrieveOutput, err := cmdRetrieveGUID.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to retrieve service instance GUID: %w", err) + } + serviceInstanceID := strings.TrimSpace(strings.Split(strings.Split(string(retrieveOutput), "GUID:")[1], "Location:")[0]) + + if len(serviceInstanceID) == 0 { + return fmt.Errorf("service instance ID not found") + } + + logger.Info(t, fmt.Sprintf("Service instance '%s' retrieved successfully. Instance ID: %s", instanceName, serviceInstanceID)) + + // Retrieve and delete associated keys + getAssociatedKeysCmd := fmt.Sprintf("ibmcloud kp keys -i %s | awk 'NR>3' | awk '{print $1}'", serviceInstanceID) + cmdKeysID := exec.Command("bash", "-c", getAssociatedKeysCmd) + keysIDOutput, err := cmdKeysID.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to retrieve associated keys: %w", err) + } + // Extract the GUID values of the keys + keyLines := strings.Split(string(strings.TrimSpace(strings.Split(string(keysIDOutput), "kms.cloud.ibm.com")[1])), "\n") + for _, key := range keyLines { + if key != "" { + // Delete each key + deleteKeyCmd := fmt.Sprintf("ibmcloud kp key delete %s -i %s", key, serviceInstanceID) + cmdDeleteKey := exec.Command("bash", "-c", deleteKeyCmd) + deleteKeyOutput, err := cmdDeleteKey.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to delete key %s: %w", key, err) + } + if !utils.VerifyDataContains(t, string(deleteKeyOutput), "Deleted Key", logger) { + return fmt.Errorf("failed to delete key: %s", string(deleteKeyOutput)) + } + } + } + + // Delete the service instance + deleteInstanceCmd := fmt.Sprintf("ibmcloud resource service-instance-delete %s -f", instanceName) + cmdDeleteInstance := exec.Command("bash", "-c", deleteInstanceCmd) + deleteInstanceOutput, err := cmdDeleteInstance.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to delete instance %s: %w", instanceName, err) + } + + if !utils.VerifyDataContains(t, string(deleteInstanceOutput), "deleted successfully", logger) { + return fmt.Errorf("failed to delete instance: %s", string(deleteInstanceOutput)) + } + + logger.Info(t, "Service instance deleted successfully") + return nil +} + +// CreateKey creates a key in a specified service instance on IBM Cloud. +// It logs into IBM Cloud using the provided API key, region, and resource group, then creates a key +// with the specified key name in the specified service instance. If the creation is successful, it verifies the key's creation. +// Returns: +// - error: error if any step fails +func CreateKey(t *testing.T, apiKey, region, resourceGroup, instanceName, keyName string, logger *utils.AggregatedLogger) error { + + // Log in to IBM Cloud using the API key and region + if err := utils.LoginIntoIBMCloudUsingCLI(t, apiKey, region, resourceGroup); err != nil { + return fmt.Errorf("failed to log in to IBM Cloud: %w", err) + } + + // Retrieve the service instance GUID + retrieveServiceCmd := fmt.Sprintf("ibmcloud resource service-instance --service-name %s --output", instanceName) + cmdRetrieveGUID := exec.Command("bash", "-c", retrieveServiceCmd) + retrieveOutput, err := cmdRetrieveGUID.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to retrieve service instance GUID: %w", err) + } + + serviceInstanceID := strings.TrimSpace(strings.Split(strings.Split(string(retrieveOutput), "GUID:")[1], "Location:")[0]) + if len(serviceInstanceID) == 0 { + return fmt.Errorf("service instance ID not found") + } + + logger.Info(t, fmt.Sprintf("Service instance '%s' retrieved successfully. Instance ID: %s", instanceName, serviceInstanceID)) + + // Create key + + createKeyCmd := fmt.Sprintf("ibmcloud kp key create %s -i %s", keyName, serviceInstanceID) + cmdKey := exec.Command("bash", "-c", createKeyCmd) + keyOutput, err := cmdKey.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to create key: %w. Output: %s", err, string(keyOutput)) + } + if !utils.VerifyDataContains(t, string(keyOutput), "OK", logger) { + return fmt.Errorf("failed to create key: %s", string(keyOutput)) + } + + logger.Info(t, fmt.Sprintf("Key '%s' created successfully in service instance '%s'", keyName, serviceInstanceID)) + + // Retrieve and verify key + retrieveKeyCmd := fmt.Sprintf("ibmcloud kp keys -i %s", serviceInstanceID) + cmdRetrieveKey := exec.Command("bash", "-c", retrieveKeyCmd) + retrieveKeyOutput, err := cmdRetrieveKey.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to retrieve keys: %w", err) + } + if !utils.VerifyDataContains(t, string(retrieveKeyOutput), keyName, logger) { + return fmt.Errorf("key retrieval failed: %s", string(retrieveKeyOutput)) + } + + logger.Info(t, fmt.Sprintf("Key '%s' created successfully in service instance '%s'", keyName, serviceInstanceID)) + return nil +} diff --git a/tests/lsf/lsf_constants.go b/tests/lsf/lsf_constants.go new file mode 100644 index 00000000..9c1f926d --- /dev/null +++ b/tests/lsf/lsf_constants.go @@ -0,0 +1,26 @@ +package tests + +const ( + IMAGE_NAME_PATH = "modules/landing_zone_vsi/image_map.tf" + LSF_PUBLIC_HOST_NAME = "ubuntu" + LSF_PRIVATE_HOST_NAME = "lsfadmin" + LSF_LDAP_HOST_NAME = "ubuntu" + HYPERTHREADTING_TRUE = true + HYPERTHREADTING_FALSE = false + LSF_DEFAULT_RESOURCE_GROUP = "Default" + LSF_CUSTOM_RESOURCE_GROUP_VALUE_AS_NULL = "null" + EXPECTED_LSF_VERSION = "10.1.0.14" + JOB_COMMAND_LOW_MEM = `bsub -J myjob[1-2] -R "select[family=mx2] rusage[mem=10G]" sleep 60` + JOB_COMMAND_MED_MEM = `bsub -J myjob[1-2] -R "select[family=mx2] rusage[mem=30G]" sleep 60` + JOB_COMMAND_HIGH_MEM = `bsub -J myjob[1-2] -R "select[family=mx2] rusage[mem=90G]" sleep 60` + JOB_COMMAND_LOW_MEM_SOUTH = `bsub -J myjob[1-2] -R "select[family=mx3d] rusage[mem=10G]" sleep 60` + JOB_COMMAND_MED_MEM_SOUTH = `bsub -J myjob[1-2] -R "select[family=mx3d] rusage[mem=30G]" sleep 60` + JOB_COMMAND_HIGH_MEM_SOUTH = `bsub -J myjob[1-2] -R "select[family=mx3d] rusage[mem=90G]" sleep 60` + JOB_COMMAND_LOW_MEM_WITH_MORE_SLEEP = `bsub -J myjob[1-2] -R "select[family=mx2] rusage[mem=30G]" sleep 60` +) + +var ( + LSF_CUSTOM_RESOURCE_GROUP_OTHER_THAN_DEFAULT = "WES_TEST" + KMS_KEY_INSTANCE_NAME = "cicd-key-instance" + KMS_KEY_NAME = "cicd-key-name" +) diff --git a/tests/other_test.go b/tests/other_test.go new file mode 100644 index 00000000..4bfbb070 --- /dev/null +++ b/tests/other_test.go @@ -0,0 +1,671 @@ +package tests + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/common_utils" + lsf "github.com/terraform-ibm-modules/terraform-ibm-hpc/lsf" +) + +// Constants for better organization +const ( + createVpcTerraformDir = "examples/create_vpc/solutions/hpc" // Brand new VPC +) + +// TestRunBasic validates the cluster configuration and creation of an HPC cluster. +func TestRunBasic(t *testing.T) { + + // Parallelize the test + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + require.NoError(t, err, "Error setting up test options: %v", err) + + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateClusterConfiguration(t, options, testLogger) + +} + +// TestRunCustomRGAsNull validates cluster creation with a null resource group value. +func TestRunCustomRGAsNull(t *testing.T) { + // Parallelize the test + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Create test options + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, LSF_CUSTOM_RESOURCE_GROUP_VALUE_AS_NULL, ignoreDestroys) + require.NoError(t, err, "Error setting up test options: %v", err) + + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateBasicClusterConfiguration(t, options, testLogger) + +} + +// TestRunCustomRGAsNonDefault validates cluster creation with a non-default resource group value. +func TestRunCustomRGAsNonDefault(t *testing.T) { + // Parallelize the test + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.NonDefaultResourceGroup, ignoreDestroys) + require.NoError(t, err, "Error setting up test options: %v", err) + + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateBasicClusterConfiguration(t, options, testLogger) + +} + +// TestRunAppCenter validates cluster creation with the Application Center. +func TestRunAppCenter(t *testing.T) { + // Parallelize the test + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + require.NoError(t, err, "Error setting up test options: %v", err) + options.TerraformVars["enable_app_center"] = strings.ToLower(envVars.EnableAppCenter) + options.TerraformVars["app_center_gui_pwd"] = envVars.AppCenterGuiPassword //pragma: allowlist secret + + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateClusterConfigurationWithAPPCenter(t, options, testLogger) + +} + +// TestRunNoKMSAndHTOff validates cluster creation with KMS set to null and hyperthreading disabled. +func TestRunNoKMSAndHTOff(t *testing.T) { + // Parallelize the test + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + require.NoError(t, err, "Error setting up test options: %v", err) + options.TerraformVars["enable_cos_integration"] = false + options.TerraformVars["enable_vpc_flow_logs"] = false + options.TerraformVars["key_management"] = "null" + options.TerraformVars["hyperthreading_enabled"] = strings.ToLower("false") + + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateBasicClusterConfiguration(t, options, testLogger) +} + +// TestRunInUsEastRegion validates cluster creation in the US East region. +func TestRunInUsEastRegion(t *testing.T) { + // Parallelize the test to run concurrently with others + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Extract US East zone, cluster ID, and reservation ID + usEastZone := utils.SplitAndTrim(envVars.USEastZone, ",") + usEastClusterID := envVars.USEastClusterID + usEastReservationID := envVars.USEastReservationID + + // Ensure Reservation , cluster ID and zone are provided + if len(usEastClusterID) == 0 || len(usEastZone) == 0 || len(usEastReservationID) == 0 { + require.FailNow(t, "Reservation ID ,cluster ID and zone must be provided.") + } + + // Create test options, set up test environment + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + require.NoError(t, err, "Error setting up test options: %v", err) + + // Set Terraform variables + options.TerraformVars["zones"] = usEastZone + options.TerraformVars["reservation_id"] = usEastReservationID + options.TerraformVars["cluster_id"] = usEastClusterID + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateBasicClusterConfiguration(t, options, testLogger) +} + +// TestRunInEuDeRegion validates cluster creation in the Frankfurt region. +func TestRunInEuDeRegion(t *testing.T) { + // Parallelize the test to run concurrently with others + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Extract EU DE zone, cluster ID, and reservation ID + euDeZone := utils.SplitAndTrim(envVars.EUDEZone, ",") + euDeClusterID := envVars.EUDEClusterID + euDeReservationID := envVars.EUDEReservationID + + // Ensure Reservation ID ,cluster ID and zone are provided + if len(euDeClusterID) == 0 || len(euDeZone) == 0 || len(euDeReservationID) == 0 { + require.FailNow(t, "Reservation ID, cluster ID and zone must be provided.") + } + + // Create test options, set up test environment + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + require.NoError(t, err, "Error setting up test options: %v", err) + + // Set Terraform variables + options.TerraformVars["zones"] = euDeZone + options.TerraformVars["reservation_id"] = euDeReservationID + options.TerraformVars["cluster_id"] = euDeClusterID + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateBasicClusterConfiguration(t, options, testLogger) +} + +// TestRunInUSSouthRegion validates cluster creation in the US South region. +func TestRunInUSSouthRegion(t *testing.T) { + // Parallelize the test to run concurrently with others + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Extract US South zone, cluster ID, and reservation ID + usSouthZone := utils.SplitAndTrim(envVars.USSouthZone, ",") + usSouthClusterID := envVars.USSouthClusterID + usSouthReservationID := envVars.USSouthReservationID + + // Ensure cluster ID ,Reservation ID and zone are provided + if len(usSouthClusterID) == 0 || len(usSouthZone) == 0 || len(usSouthReservationID) == 0 { + require.FailNow(t, "Reservation ID ,cluster ID and zone must be provided.") + } + + // Create test options, set up test environment + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + require.NoError(t, err, "Error setting up test options: %v", err) + + // Set Terraform variables + options.TerraformVars["zones"] = usSouthZone + options.TerraformVars["reservation_id"] = usSouthReservationID + options.TerraformVars["cluster_id"] = usSouthClusterID + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateBasicClusterConfiguration(t, options, testLogger) +} + +// TestRunLDAP validates cluster creation with LDAP enabled. +func TestRunLDAP(t *testing.T) { + // Parallelize the test to run concurrently with others + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options, set up test environment + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + require.NoError(t, err, "Error setting up test options: %v", err) + + if strings.ToLower(envVars.EnableLdap) == "true" { + // Check if the Reservation ID contains 'WES' and cluster ID is not empty + if len(envVars.LdapAdminPassword) == 0 || len(envVars.LdapUserName) == 0 || len(envVars.LdapUserPassword) == 0 { + require.FailNow(t, "LDAP credentials are missing. Make sure LDAP admin password, LDAP user name, and LDAP user password are provided.") + } + } else { + require.FailNow(t, "LDAP is not enabled. Set the 'enable_ldap' environment variable to 'true' to enable LDAP.") + } + + // Set Terraform variables + options.TerraformVars["enable_ldap"] = strings.ToLower(envVars.EnableLdap) + options.TerraformVars["ldap_basedns"] = envVars.LdapBaseDns + options.TerraformVars["ldap_admin_password"] = envVars.LdapAdminPassword //pragma: allowlist secret + options.TerraformVars["ldap_user_name"] = envVars.LdapUserName + options.TerraformVars["ldap_user_password"] = envVars.LdapUserPassword //pragma: allowlist secret + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateLDAPClusterConfiguration(t, options, testLogger) +} + +// TestRunUsingExistingKMS validates cluster creation using an existing KMS. +func TestRunUsingExistingKMS(t *testing.T) { + // Parallelize the test to run concurrently with others + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // Service instance name + randomString := utils.GenerateRandomString() + kmsInstanceName := "cicd-" + randomString + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + err := lsf.CreateServiceInstanceandKmsKey(t, os.Getenv("TF_VAR_ibmcloud_api_key"), utils.GetRegion(envVars.Zone), envVars.DefaultResourceGroup, kmsInstanceName, lsf.KMS_KEY_NAME, testLogger) + require.NoError(t, err, "Service instance and KMS key creation failed") + + testLogger.Info(t, "Service instance and KMS key created successfully "+t.Name()) + + // Create test options, set up test environment + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + require.NoError(t, err, "Error setting up test options: %v", err) + + // Set Terraform variables + options.TerraformVars["key_management"] = "key_protect" + options.TerraformVars["kms_instance_name"] = kmsInstanceName + options.TerraformVars["kms_key_name"] = lsf.KMS_KEY_NAME + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer lsf.DeleteServiceInstanceAndAssociatedKeys(t, os.Getenv("TF_VAR_ibmcloud_api_key"), utils.GetRegion(envVars.Zone), envVars.DefaultResourceGroup, kmsInstanceName, testLogger) + defer options.TestTearDown() + + lsf.ValidateBasicClusterConfiguration(t, options, testLogger) +} + +// TestRunLDAPAndPac validates cluster creation with both Application Center (PAC) and LDAP enabled. +func TestRunLDAPAndPac(t *testing.T) { + // Parallelize the test to run concurrently with others + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options, set up test environment + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + require.NoError(t, err, "Error setting up test options: %v", err) + + if strings.ToLower(envVars.EnableLdap) == "true" { + // Check if the Reservation ID contains 'WES' and cluster ID is not empty + if len(envVars.LdapAdminPassword) == 0 || len(envVars.LdapUserName) == 0 || len(envVars.LdapUserPassword) == 0 { + require.FailNow(t, "LDAP credentials are missing. Make sure LDAP admin password, LDAP user name, and LDAP user password are provided.") + } + } else { + require.FailNow(t, "LDAP is not enabled. Set the 'enable_ldap' environment variable to 'true' to enable LDAP.") + } + + // Set Terraform variables + options.TerraformVars["enable_app_center"] = strings.ToLower(envVars.EnableAppCenter) + options.TerraformVars["app_center_gui_pwd"] = envVars.AppCenterGuiPassword //pragma: allowlist secret + options.TerraformVars["enable_ldap"] = strings.ToLower(envVars.EnableLdap) + options.TerraformVars["ldap_basedns"] = envVars.LdapBaseDns + options.TerraformVars["ldap_admin_password"] = envVars.LdapAdminPassword //pragma: allowlist secret + options.TerraformVars["ldap_user_name"] = envVars.LdapUserName + options.TerraformVars["ldap_user_password"] = envVars.LdapUserPassword //pragma: allowlist secret + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidatePACANDLDAPClusterConfiguration(t, options, testLogger) +} + +// TestRunCreateVpc as brand new +func TestRunCreateVpc(t *testing.T) { + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Brand new VPC creation initiated for "+t.Name()) + + // Define the HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options, set up test environment + options, err := setupOptionsVpc(t, hpcClusterPrefix, createVpcTerraformDir, envVars.DefaultResourceGroup) + require.NoError(t, err, "Error setting up test options: %v", err) + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer options.TestTearDown() + + // Run the test + output, err := options.RunTest() + require.NoError(t, err, "Error running consistency test: %v", err) + require.NotNil(t, output, "Expected non-nil output, but got nil") + + outputs := (options.LastTestTerraformOutputs) + vpcName := outputs["vpc_name"].(string) + bastionsubnetId, computesubnetIds := utils.GetSubnetIds(outputs) + + RunHpcExistingVpcSubnetId(t, vpcName, bastionsubnetId, computesubnetIds) + RunHpcExistingVpcCidr(t, vpcName) +} + +// RunHpcExistingVpcCidr with Cidr blocks +func RunHpcExistingVpcCidr(t *testing.T, vpcName string) { + // Parallelize the test to run concurrently with others + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Static values for CIDR other than default CIDR + vpcClusterPrivateSubnetsCidrBlocks := "10.241.48.0/21,10.241.120.0/22" + vpcClusterLoginPrivateSubnetsCidrBlocks := "10.241.60.0/22" + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + options.TerraformVars["vpc_name"] = vpcName + options.TerraformVars["vpc_cluster_private_subnets_cidr_blocks"] = utils.SplitAndTrim(vpcClusterPrivateSubnetsCidrBlocks, ",") + options.TerraformVars["vpc_cluster_login_private_subnets_cidr_blocks"] = utils.SplitAndTrim(vpcClusterLoginPrivateSubnetsCidrBlocks, ",") + require.NoError(t, err, "Error setting up test options: %v", err) + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateClusterConfiguration(t, options, testLogger) +} + +// RunHpcExistingVpcSubnetId with compute and login subnet id's +func RunHpcExistingVpcSubnetId(t *testing.T, vpcName string, bastionsubnetId string, computesubnetIds string) { + // Parallelize the test to run concurrently with others + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + options.TerraformVars["vpc_name"] = vpcName + options.TerraformVars["login_subnet_id"] = bastionsubnetId + options.TerraformVars["cluster_subnet_ids"] = utils.SplitAndTrim(computesubnetIds, ",") + require.NoError(t, err, "Error setting up test options: %v", err) + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateClusterConfiguration(t, options, testLogger) +} + +// TestRunCreateVpcWithCustomDns brand new VPC with DNS +func TestRunVpcWithCustomDns(t *testing.T) { + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // Define the HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options, set up test environment + options, err := setupOptionsVpc(t, hpcClusterPrefix, createVpcTerraformDir, envVars.DefaultResourceGroup) + options.TerraformVars["enable_hub"] = true + + require.NoError(t, err, "Error setting up test options: %v", err) + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer options.TestTearDown() + + // Run the test + output, err := options.RunTestConsistency() + require.NoError(t, err, "Error running consistency test: %v", err) + require.NotNil(t, output, "Expected non-nil output, but got nil") + + outputs := (options.LastTestTerraformOutputs) + vpcName := outputs["vpc_name"].(string) + instanceId, customResolverId := utils.GetDnsCustomResolverIds(outputs) + bastionsubnetId, computesubnetIds := utils.GetSubnetIds(outputs) + + RunHpcExistingVpcCustomDnsExist(t, vpcName, bastionsubnetId, computesubnetIds, instanceId, customResolverId) + RunHpcExistingVpcCustomExistDnsNew(t, vpcName, bastionsubnetId, computesubnetIds, customResolverId) + RunHpcNewVpcCustomNullExistDns(t, instanceId) + RunHpcNewVpcExistCustomDnsNull(t, customResolverId) +} + +// RunHpcExistingVpcCustomDns with existing custom_reslover_id and dns_instance_id +func RunHpcExistingVpcCustomDnsExist(t *testing.T, vpcName string, bastionsubnetId string, computesubnetIds string, instanceId string, customResolverId string) { + // Parallelize the test + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + options.TerraformVars["vpc_name"] = vpcName + options.TerraformVars["login_subnet_id"] = bastionsubnetId + options.TerraformVars["cluster_subnet_ids"] = utils.SplitAndTrim(computesubnetIds, ",") + options.TerraformVars["dns_instance_id"] = instanceId + options.TerraformVars["dns_custom_resolver_id"] = customResolverId + + require.NoError(t, err, "Error setting up test options: %v", err) + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateClusterConfiguration(t, options, testLogger) +} + +// RunHpcExistingVpcCustomExistDnsNew with existing custom_reslover_id and new dns_instance_id +func RunHpcExistingVpcCustomExistDnsNew(t *testing.T, vpcName string, bastionsubnetId string, computesubnetIds string, customResolverId string) { + // Parallelize the test + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + options.TerraformVars["vpc_name"] = vpcName + options.TerraformVars["login_subnet_id"] = bastionsubnetId + options.TerraformVars["cluster_subnet_ids"] = utils.SplitAndTrim(computesubnetIds, ",") + options.TerraformVars["dns_custom_resolver_id"] = customResolverId + + require.NoError(t, err, "Error setting up test options: %v", err) + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateClusterConfiguration(t, options, testLogger) +} + +// RunHpcNewVpcCustomNullExistDns with custom_reslover_id null and existing dns_instance_id +func RunHpcNewVpcCustomNullExistDns(t *testing.T, instanceId string) { + // Parallelize the test + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + options.TerraformVars["dns_instance_id"] = instanceId + + require.NoError(t, err, "Error setting up test options: %v", err) + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateClusterConfiguration(t, options, testLogger) +} + +// RunHpcNewVpcExistCustomDnsNull with existing custom_reslover_id and dns_instance_id null +func RunHpcNewVpcExistCustomDnsNull(t *testing.T, customResolverId string) { + // Parallelize the test + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + options.TerraformVars["dns_instance_id"] = customResolverId + + require.NoError(t, err, "Error setting up test options: %v", err) + + // Skip test teardown for further inspection + options.SkipTestTearDown = true + defer options.TestTearDown() + + lsf.ValidateClusterConfiguration(t, options, testLogger) +} diff --git a/tests/pr_test.go b/tests/pr_test.go index 7b8bf4a2..6caebb15 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -1,34 +1,288 @@ -package test +package tests import ( + "fmt" + "log" "os" + "path/filepath" + "reflect" + "strings" "testing" + "time" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testhelper" + + utils "github.com/terraform-ibm-modules/terraform-ibm-hpc/common_utils" ) +// Constants for better organization const ( - exampleBasicTerraformDir = "examples/basic" // Path of the Terraform directory + // Path of the Terraform directory + terraformDir = "solutions/hpc" ) -// TestRunHpcBasicExample -func TestRunHpcBasicExample(t *testing.T) { - t.Parallel() +// Terraform resource names to ignore during consistency checks +var ignoreDestroys = []string{ + "module.landing_zone_vsi.module.hpc.module.check_cluster_status.null_resource.remote_exec[0]", + "module.landing_zone_vsi.module.hpc.module.check_node_status.null_resource.remote_exec[1]", + "module.landing_zone_vsi.module.hpc.module.check_node_status.null_resource.remote_exec[0]", + "module.landing_zone_vsi.module.hpc.module.check_node_status.null_resource.remote_exec[2]", + "module.check_node_status.null_resource.remote_exec[0]", + "module.landing_zone_vsi.module.wait_management_vsi_booted.null_resource.remote_exec[0]", + "module.check_node_status.null_resource.remote_exec[1]", + "module.landing_zone_vsi.module.wait_management_candidate_vsi_booted.null_resource.remote_exec[0]", + "module.check_cluster_status.null_resource.remote_exec[0]", + "module.landing_zone_vsi.module.hpc.module.landing_zone_vsi.module.wait_management_candidate_vsi_booted.null_resource.remote_exec[0]", + "module.landing_zone_vsi.module.hpc.module.landing_zone_vsi.module.wait_management_vsi_booted.null_resource.remote_exec[0]", +} + +// EnvVars stores environment variable values. +type EnvVars struct { + DefaultResourceGroup string + NonDefaultResourceGroup string + Zone string + ClusterID string + ReservationID string + RemoteAllowedIPs string + SSHKey string + LoginNodeInstanceType string + LoginNodeImageName string + ManagementImageName string + ComputeImageName string + ManagementNodeInstanceType string + ManagementNodeCount string + KeyManagement string + KMSInstanceName string + KMSKeyName string + HyperthreadingEnabled string + DnsDomainName string + EnableAppCenter string + AppCenterGuiPassword string + EnableLdap string + LdapBaseDns string + LdapServer string + LdapAdminPassword string + LdapUserName string + LdapUserPassword string + USEastZone string + USEastReservationID string + USEastClusterID string + EUDEZone string + EUDEReservationID string + EUDEClusterID string + SSHFilePath string + USSouthZone string + USSouthReservationID string + USSouthClusterID string +} + +// GetEnvVars retrieves environment variables. +func GetEnvVars() EnvVars { + return EnvVars{ + DefaultResourceGroup: os.Getenv("DEFAULT_RESOURCE_GROUP"), + NonDefaultResourceGroup: os.Getenv("NON_DEFAULT_RESOURCE_GROUP"), + Zone: os.Getenv("ZONE"), + ClusterID: os.Getenv("CLUSTER_ID"), + ReservationID: os.Getenv("RESERVATION_ID"), + RemoteAllowedIPs: os.Getenv("REMOTE_ALLOWED_IPS"), + SSHKey: os.Getenv("SSH_KEY"), + LoginNodeInstanceType: os.Getenv("LOGIN_NODE_INSTANCE_TYPE"), + LoginNodeImageName: os.Getenv("LOGIN_NODE_IMAGE_NAME"), + ManagementImageName: os.Getenv("MANAGEMENT_IMAGE_NAME"), + ComputeImageName: os.Getenv("COMPUTE_IMAGE_NAME"), + ManagementNodeInstanceType: os.Getenv("MANAGEMENT_NODE_INSTANCE_TYPE"), + ManagementNodeCount: os.Getenv("MANAGEMENT_NODE_COUNT"), + KeyManagement: os.Getenv("KEY_MANAGEMENT"), + KMSInstanceName: os.Getenv("KMS_INSTANCE_NAME"), + KMSKeyName: os.Getenv("KMS_KEY_NAME"), + HyperthreadingEnabled: os.Getenv("HYPERTHREADING_ENABLED"), + DnsDomainName: os.Getenv("DNS_DOMAIN_NAME"), + EnableAppCenter: os.Getenv("ENABLE_APP_CENTER"), + AppCenterGuiPassword: os.Getenv("APP_CENTER_GUI_PASSWORD"), + EnableLdap: os.Getenv("ENABLE_LDAP"), + LdapBaseDns: os.Getenv("LDAP_BASEDNS"), + LdapServer: os.Getenv("LDAP_SERVER"), + LdapAdminPassword: os.Getenv("LDAP_ADMIN_PASSWORD"), + LdapUserName: os.Getenv("LDAP_USER_NAME"), + LdapUserPassword: os.Getenv("LDAP_USER_PASSWORD"), + USEastZone: os.Getenv("US_EAST_ZONE"), + USEastReservationID: os.Getenv("US_EAST_RESERVATION_ID"), + USEastClusterID: os.Getenv("US_EAST_CLUSTER_ID"), + EUDEZone: os.Getenv("EU_DE_ZONE"), + EUDEReservationID: os.Getenv("EU_DE_RESERVATION_ID"), + EUDEClusterID: os.Getenv("EU_DE_CLUSTER_ID"), + USSouthZone: os.Getenv("US_SOUTH_ZONE"), + USSouthReservationID: os.Getenv("US_SOUTH_RESERVATION_ID"), + USSouthClusterID: os.Getenv("US_SOUTH_CLUSTER_ID"), + SSHFilePath: os.Getenv("SSH_FILE_PATH"), + } +} + +var ( + // testLogger stores the logger instance for logging test messages. + testLogger *utils.AggregatedLogger + // loggerErr stores the error occurred during logger initialization. + loggerErr error + // testSuiteInitialized indicates whether the test suite has been initialized. + testSuiteInitialized bool +) + +// setupTestSuite initializes the test suite. +func setupTestSuite(t *testing.T) { + if !testSuiteInitialized { + fmt.Println("Started executing the test suite...") + timestamp := time.Now().Format("2006-01-02_15-04-05") + logFileName := fmt.Sprintf("log_%s.log", timestamp) + testLogger, loggerErr = utils.NewAggregatedLogger(logFileName) + if loggerErr != nil { + t.Fatalf("Error initializing logger: %v", loggerErr) + } + testSuiteInitialized = true + } +} + +// setupOptionsVpc creates a test options object with the given parameters to creating brand new vpc +func setupOptionsVpc(t *testing.T, hpcClusterPrefix, terraformDir, resourceGroup string) (*testhelper.TestOptions, error) { + // Check if TF_VAR_ibmcloud_api_key is set + if os.Getenv("TF_VAR_ibmcloud_api_key") == "" { + return nil, fmt.Errorf("TF_VAR_ibmcloud_api_key is not set") + } + + // Retrieve environment variables + envVars := GetEnvVars() + + // Validate required environment variables + requiredVars := []string{"SSHKey", "Zone"} + for _, fieldName := range requiredVars { + // Check if the field value is empty + if fieldValue := reflect.ValueOf(envVars).FieldByName(fieldName).String(); fieldValue == "" { + return nil, fmt.Errorf("missing required environment variable: %s", fieldName) + } + } + + // Generate timestamped cluster prefix + prefix := utils.GenerateTimestampedClusterPrefix(hpcClusterPrefix) + + // Create test options options := &testhelper.TestOptions{ - Testing: t, - TerraformDir: exampleBasicTerraformDir, + Testing: t, + TerraformDir: terraformDir, + IgnoreDestroys: testhelper.Exemptions{List: ignoreDestroys}, TerraformVars: map[string]interface{}{ - "ibmcloud_api_key": os.Getenv("TF_VAR_ibmcloud_api_key"), + "cluster_prefix": prefix, + "bastion_ssh_keys": utils.SplitAndTrim(envVars.SSHKey, ","), + "zones": utils.SplitAndTrim(envVars.Zone, ","), + "remote_allowed_ips": utils.SplitAndTrim(envVars.RemoteAllowedIPs, ","), + "resource_group": resourceGroup, }, } - // Run the test - output, err := options.RunTestConsistency() + return options, nil +} + +// setupOptions creates a test options object with the given parameters. +func setupOptions(t *testing.T, hpcClusterPrefix, terraformDir, resourceGroup string, ignoreDestroys []string) (*testhelper.TestOptions, error) { + + // Check if TF_VAR_ibmcloud_api_key is set + if os.Getenv("TF_VAR_ibmcloud_api_key") == "" { + return nil, fmt.Errorf("TF_VAR_ibmcloud_api_key is not set") + } + + // Retrieve environment variables + envVars := GetEnvVars() + + // Validate required environment variables + requiredVars := []string{"SSHKey", "ClusterID", "Zone", "ReservationID"} + for _, fieldName := range requiredVars { + // Check if the field value is empty + if fieldValue := reflect.ValueOf(envVars).FieldByName(fieldName).String(); fieldValue == "" { + return nil, fmt.Errorf("missing required environment variable: %s", fieldName) + } + } - // Check for errors - assert.Nil(t, err, "Expected no errors, but got: %v", err) + // Generate timestamped cluster prefix + prefix := utils.GenerateTimestampedClusterPrefix(hpcClusterPrefix) + + // Create test options + options := &testhelper.TestOptions{ + Testing: t, + TerraformDir: terraformDir, + IgnoreDestroys: testhelper.Exemptions{List: ignoreDestroys}, + TerraformVars: map[string]interface{}{ + "cluster_prefix": prefix, + "bastion_ssh_keys": utils.SplitAndTrim(envVars.SSHKey, ","), + "compute_ssh_keys": utils.SplitAndTrim(envVars.SSHKey, ","), + "zones": utils.SplitAndTrim(envVars.Zone, ","), + "remote_allowed_ips": utils.SplitAndTrim(envVars.RemoteAllowedIPs, ","), + "cluster_id": envVars.ClusterID, + "reservation_id": envVars.ReservationID, + "resource_group": resourceGroup, + "login_node_instance_type": envVars.LoginNodeInstanceType, + "login_image_name": envVars.LoginNodeImageName, + "management_image_name": envVars.ManagementImageName, + "management_node_instance_type": envVars.ManagementNodeInstanceType, + "management_node_count": envVars.ManagementNodeCount, + "compute_image_name": envVars.ComputeImageName, + "key_management": envVars.KeyManagement, + "hyperthreading_enabled": strings.ToLower(envVars.HyperthreadingEnabled), + "app_center_high_availability": false, + "observability_atracker_on_cos_enable": false, + "dns_domain_name": map[string]string{"compute": envVars.DnsDomainName}, + }, + } + + // Remove parameters with empty values + for key, value := range options.TerraformVars { + if value == "" { + delete(options.TerraformVars, key) + } + } + + return options, nil +} + +func TestMain(m *testing.M) { + + absPath, err := filepath.Abs("test_config.yml") + if err != nil { + log.Fatalf("error getting absolute path: %v", err) + } + + // Read configuration from yaml file + _, err = utils.GetConfigFromYAML(absPath) + if err != nil { + log.Fatalf("error reading configuration from yaml: %v", err) + } + + os.Exit(m.Run()) + +} + +// TestRunDefault create basic cluster of an HPC cluster. +func TestRunDefault(t *testing.T) { + + // Parallelize the test + t.Parallel() + + // Setup test suite + setupTestSuite(t) + + testLogger.Info(t, "Cluster creation process initiated for "+t.Name()) + + // HPC cluster prefix + hpcClusterPrefix := utils.GenerateRandomString() + + // Retrieve cluster information from environment variables + envVars := GetEnvVars() + + // Create test options + options, err := setupOptions(t, hpcClusterPrefix, terraformDir, envVars.DefaultResourceGroup, ignoreDestroys) + require.NoError(t, err, "Error setting up test options: %v", err) + + // Run the test and handle errors + output, err := options.RunTestConsistency() + require.NoError(t, err, "Error running consistency test: %v", err) + require.NotNil(t, output, "Expected non-nil output, but got nil") - // Check the output for specific strings - assert.NotNil(t, output, "Expected non-nil output, but got nil") } diff --git a/tests/test_config.yml b/tests/test_config.yml new file mode 100644 index 00000000..812dca9e --- /dev/null +++ b/tests/test_config.yml @@ -0,0 +1,36 @@ +default_resource_group: Default +non_default_resource_group: WES_TEST +zone: us-east-3 +cluster_id: HPC-LSF-2 +reservation_id: +remote_allowed_ips: +ssh_key: geretain-hpc +login_node_instance_type: bx2-2x8 +login_image_name: hpcaas-lsf10-rhel88-compute-v5 +management_image_name: hpcaas-lsf10-rhel88-v6 +compute_image_name: hpcaas-lsf10-rhel88-compute-v5 +management_node_instance_type: bx2-16x64 +management_node_count: 2 +enable_vpc_flow_logs: false +key_management: key_protect +kms_instance_name: +kms_key_name: +hyperthreading_enabled: true +dns_domain_name : wes.com +enable_app_center: true +app_center_gui_pwd: Pass@123 # pragma: allowlist secret +enable_ldap: true +ldap_basedns: cicd.com +ldap_admin_password: Pass@123 # pragma: allowlist secret +ldap_user_name: tester +ldap_user_password: Pass@123 # pragma: allowlist secret +us_east_zone: us-east-3 +us_east_reservation_id: +us_east_cluster_id: HPC-LSF-2 +eu_de_zone: eu-de-3 +eu_de_reservation_id: +eu_de_cluster_id: HPC-LSF-1 +us_south_zone: us-south-1 +us_south_reservation_id: +us_south_cluster_id: HPC-LSF-1 +ssh_file_path: /artifacts/.ssh/id_rsa diff --git a/tests/test_output/output.txt b/tests/test_output/output.txt new file mode 100644 index 00000000..1fe89236 --- /dev/null +++ b/tests/test_output/output.txt @@ -0,0 +1,4 @@ +### Test Output Logs + +- **Console Output**: Check the console for detailed test results. +- **Log Files**: Review `test_output.log` and custom logs in the `/tests/test_output` folder with a timestamp for detailed analysis and troubleshooting. For example, a log file might be named `log_20XX-MM-DD_HH-MM-SS.log`. diff --git a/tools/image-builder/compute-rhel.sh b/tools/image-builder/compute-rhel.sh new file mode 100644 index 00000000..a842db2d --- /dev/null +++ b/tools/image-builder/compute-rhel.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# LSF prerequisites +LSF_TOP="/opt/ibm/lsf" +LSF_CONF_PATH="${LSF_TOP}/conf" +LSF_PACKAGES_PATH="provide the path to where the lsf latest packges are present" +sleep 180 +LSF_PREREQS="python38 nfs-utils ed wget gcc-c++ elfutils-libelf-devel kernel-devel-$(uname -r) gcc-gfortran libgfortran libquadmath libmpc libquadmath-devel mpfr perl libnsl" +dnf install -y libnsl libnsl2 openldap-clients nss-pam-ldapd authselect sssd oddjob oddjob-mkhomedir +# shellcheck disable=SC2086 +yum install -y ${LSF_PREREQS} +useradd lsfadmin +# The IBM tool https://github.com/ibm/detect-secrets detects a potential secret keywork. +# However, the code is safe, it only allows lsfadmin to become sudoers. +echo 'lsfadmin ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers # pragma: allowlist secret +mkdir -p ${LSF_TOP} +chmod -R 755 /opt +rm -f /usr/bin/python3 +ln -s /usr/bin/python3.8 /usr/bin/python3 +curl -fsSL https://clis.cloud.ibm.com/install/linux | sh +pip3 install ibm-vpc==0.10.0 +pip3 install ibm-cloud-networking-services ibm-cloud-sdk-core selinux +ibmcloud plugin install vpc-infrastructure +ibmcloud plugin install DNS +chmod 755 -R /usr/local/lib/python3.8 +chmod 755 -R /usr/local/lib64/python3.8 +hostname lsfservers +echo 'provide the value of the entitlement check that is available on the lsf entitlement check file' > "${LSF_PACKAGES_PATH}/ls.entitlement" +echo 'provide the value of the entitlement check that is available on the lsf entitlement check file' > "${LSF_PACKAGES_PATH}/lsf.entitlement" + +# Need a appropriate LSF latest packages, using the packages the below configures the lsf core installation +cd "${LSF_PACKAGES_PATH}" || exit +zcat lsf*lsfinstall_linux_x86_64.tar.Z | tar xvf - +cd lsf*_lsfinstall || exit +sed -e '/show_copyright/ s/^#*/#/' -i lsfinstall +cat <> install.config +LSF_TOP="/opt/ibm/lsf" +LSF_ADMINS="lsfadmin" +LSF_CLUSTER_NAME="HPCCluster" +LSF_MASTER_LIST="lsfservers" +LSF_ENTITLEMENT_FILE="\${LSF_PACKAGES_PATH}/lsf.entitlement" +CONFIGURATION_TEMPLATE="DEFAULT" +ENABLE_DYNAMIC_HOSTS="Y" +ENABLE_EGO="N" +ACCEPT_LICENSE="Y" +SILENT_INSTALL="Y" +LSF_SILENT_INSTALL_TARLIST="ALL" +EOT +bash lsfinstall -f install.config +echo $? +cat Install.log +echo "====================== LSF Setup Done=====================" + +# The IBM tool https://github.com/ibm/detect-secrets detects a potential secret keywork. +# However, the code is safe, it only detected the word API_KEY but no secrets are exposed. +# Removal of API keys +sed -i "s/^VPC_APIKEY=.*/VPC_APIKEY=/g" "${LSF_CONF_PATH}/resource_connector/ibmcloudgen2/credentials" # pragma: allowlist secret +sed -i "s/^RESOURCE_RECORDS_APIKEY=.*/RESOURCE_RECORDS_APIKEY=/g" "${LSF_CONF_PATH}/resource_connector/ibmcloudgen2/credentials" # pragma: allowlist secret + +hostname lsfservers + +# LSF worker (Resource connector) installation +cd "${LSF_PACKAGES_PATH}" || exit +cd lsf*_lsfinstall || exit +cat <> server.config +LSF_TOP="/opt/ibm/lsf_worker" +LSF_ADMINS="lsfadmin" +LSF_ENTITLEMENT_FILE="\${LSF_PACKAGES_PATH}/lsf.entitlement" +LSF_SERVER_HOSTS="lsfservers" +LSF_LOCAL_RESOURCES="[resource cloudhpchost]" +ACCEPT_LICENSE="Y" +SILENT_INSTALL="Y" +EOT +bash lsfinstall -s -f server.config +echo $? +cat Install.log +echo "====================== WORKER Setup Done=====================" +rm -rf /opt/ibm/lsf_worker/10.1 +ln -s /opt/ibm/lsf/10.1 /opt/ibm/lsf_worker +rm -rf /opt/ibm/lsf_worker/10.1 +ln -s /opt/ibm/lsf/10.1 /opt/ibm/lsf_worker + + +# OpenMPI installation +cd "${LSF_PACKAGES_PATH}" || exit +wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.0.tar.gz +tar -xvf openmpi-4.1.0.tar.gz +cd openmpi-4.1.0 || exit +ln -s /usr/lib64/libnsl.so.2.0.0 /usr/lib64/libnsl.so +export LANG=C +./configure --prefix='/usr/local/openmpi-4.1.0' --enable-mpi-thread-multiple --enable-shared --disable-static --enable-mpi-fortran=usempi --disable-libompitrace --enable-script-wrapper-compilers --enable-wrapper-rpath --enable-orterun-prefix-by-default --with-io-romio-flags=--with-file-system=nfs --with-lsf=/opt/ibm/lsf/10.1 --with-lsf-libdir=/opt/ibm/lsf/10.1/linux3.10-glibc2.17-x86_64/lib +make -j 32 +make install +find /usr/local/openmpi-4.1.0/ -type d -exec chmod 775 {} \; +echo "====================== SetUp of oneMPI completed=====================" + +# Intel One API (hpckit) installation +tee > /etc/yum.repos.d/oneAPI.repo << EOF +[oneAPI] +name=Intel® oneAPI repository +baseurl=https://yum.repos.intel.com/oneapi +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB +EOF +ls /etc/yum.repos.d +yum install -y intel-basekit intel-hpckit +rpm -qa | grep "intel-hpckit\|intel-basekit" +rm -rf /etc/yum.repos.d/oneAPI.repo +ls /etc/yum.repos.d +echo "====================== SetUp of one API completed=====================" + +# Setting up access for the appropriate path +# shellcheck disable=SC2086 +mv -f ${LSF_PACKAGES_PATH}/*.entitlement /opt/ibm/lsf/conf +chown -R lsfadmin:root ${LSF_CONF_PATH} + +# Updating security check +yum update --security -y + +# Sysdig Agent installation +echo "Installing Sysdig Agent" +curl -sL https://ibm.biz/install-sysdig-agent | sudo bash -s -- --access_key ==ACCESSKEY== --collector ==COLLECTOR== --collector_port 6443 --secure true --check_certificate false --additional_conf 'sysdig_capture_enabled: false\nremotefs: true\nfeature:\n mode: monitor_light' +systemctl stop dragent +systemctl disable dragent + +# Cleanup of all the folders and unwanted ssh keys as part of security +rm -rf "${LSF_PACKAGES_PATH}" +rm -rf /home/vpcuser/.ssh/authorized_keys +rm -rf /home/vpcuser/.ssh/known_hosts +rm -rf /home/vpcuser/.ssh/id_rsa* +rm -rf /home/lsfadmin/.ssh/authorized_keys +rm -rf /home/lsfadmin/.ssh/known_hosts +rm -rf /home/lsfadmin/.ssh/id_rsa* +rm -rf /root/.ssh/authorized_keys +rm -rf /root/.ssh/known_hosts +rm -rf /root/.ssh/id_rsa* +systemctl stop syslog +rm -rf /var/log/messages +rm -rf /root/.bash_history +history -c diff --git a/variables.tf b/variables.tf deleted file mode 100644 index 76fef18a..00000000 --- a/variables.tf +++ /dev/null @@ -1,452 +0,0 @@ -############################################################################## -# Offering Variations -############################################################################## -# Future use -/* -variable "scheduler" { - type = string - default = "LSF" - description = "Select one of the scheduler (LSF/Symphony/Slurm/None)" -} - -variable "storage_type" { - type = string - default = "scratch" - description = "Select the required storage type(scratch/persistent/eval)." -} - -variable "ibm_customer_number" { - type = string - sensitive = true - default = "" - description = "Comma-separated list of the IBM Customer Number(s) (ICN) that is used for the Bring Your Own License (BYOL) entitlement check. For more information on how to find your ICN, see [What is my IBM Customer Number (ICN)?](https://www.ibm.com/support/pages/what-my-ibm-customer-number-icn)." - validation { - # regex(...) fails if the IBM customer number has special characters. - condition = can(regex("^[0-9A-Za-z]*([0-9A-Za-z]+,[0-9A-Za-z]+)*$", var.ibm_customer_number)) - error_message = "The IBM customer number input value cannot have special characters." - } -} -*/ -############################################################################## -# Account Variables -############################################################################## - -variable "ibmcloud_api_key" { - description = "IBM Cloud API Key that will be used for authentication in scripts run in this module. Only required if certain options are required." - type = string - sensitive = true - default = null -} - -############################################################################## -# Resource Groups Variables -############################################################################## - -variable "resource_group" { - description = "String describing resource groups to create or reference" - type = string - default = null -} - -############################################################################## -# Module Level Variables -############################################################################## - -variable "prefix" { - description = "A unique identifier for resources. Must begin with a letter and end with a letter or number. This prefix will be prepended to any resources provisioned by this template. Prefixes must be 16 or fewer characters." - type = string - - validation { - error_message = "Prefix must begin and end with a letter and contain only letters, numbers, and - characters." - condition = can(regex("^([A-z]|[a-z][-a-z0-9]*[a-z0-9])$", var.prefix)) - } -} - -variable "zones" { - description = "Region where VPC will be created. To find your VPC region, use `ibmcloud is regions` command to find available regions." - type = list(string) -} - -############################################################################## -# VPC Variables -############################################################################## - -variable "vpc" { - type = string - description = "Name of an existing VPC in which the cluster resources will be deployed. If no value is given, then a new VPC will be provisioned for the cluster. [Learn more](https://cloud.ibm.com/docs/vpc)" - default = null -} - -variable "network_cidr" { - description = "Network CIDR for the VPC. This is used to manage network ACL rules for cluster provisioning." - type = string - default = "10.0.0.0/8" -} - -variable "placement_strategy" { - type = string - default = null - description = "VPC placement groups to create (null / host_spread / power_spread)" -} - -############################################################################## -# Access Variables -############################################################################## -variable "enable_bastion" { - type = bool - default = true - description = "The solution supports multiple ways to connect to your HPC cluster for example, using bastion node, via VPN or direct connection. If connecting to the HPC cluster via VPN or direct connection, set this value to false." -} - -variable "enable_bootstrap" { - type = bool - default = false - description = "Bootstrap should be only used for better deployment performance" -} - -variable "bootstrap_instance_profile" { - type = string - default = "mx2-4x32" - description = "Bootstrap should be only used for better deployment performance" -} - -variable "bastion_ssh_keys" { - type = list(string) - description = "The key pair to use to access the bastion host." -} - -variable "bastion_subnets_cidr" { - type = list(string) - default = ["10.0.0.0/24"] - description = "Subnet CIDR block to launch the bastion host." -} - -variable "enable_vpn" { - type = bool - default = false - description = "The solution supports multiple ways to connect to your HPC cluster for example, using bastion node, via VPN or direct connection. If connecting to the HPC cluster via VPN, set this value to true." -} - -variable "vpn_peer_cidr" { - type = list(string) - default = null - description = "The peer CIDRs (e.g., 192.168.0.0/24) to which the VPN will be connected." -} - -variable "vpn_peer_address" { - type = string - default = null - description = "The peer public IP address to which the VPN will be connected." -} - -variable "vpn_preshared_key" { - type = string - default = null - description = "The pre-shared key for the VPN." -} - -variable "allowed_cidr" { - description = "Network CIDR to access the VPC. This is used to manage network ACL rules for accessing the cluster." - type = list(string) - default = ["10.0.0.0/8"] -} - -############################################################################## -# Compute Variables -############################################################################## -# Future use -/* -variable "login_subnets_cidr" { - type = list(string) - default = ["10.10.10.0/24", "10.20.10.0/24", "10.30.10.0/24"] - description = "Subnet CIDR block to launch the login host." -} -*/ - -variable "login_ssh_keys" { - type = list(string) - description = "The key pair to use to launch the login host." -} - -variable "login_image_name" { - type = string - default = "ibm-redhat-8-6-minimal-amd64-5" - description = "Image name to use for provisioning the login instances." -} - -variable "login_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 1 - }] - description = "Number of instances to be launched for login." -} - -variable "compute_subnets_cidr" { - type = list(string) - default = ["10.10.20.0/24", "10.20.20.0/24", "10.30.20.0/24"] - description = "Subnet CIDR block to launch the compute cluster host." -} - -variable "compute_ssh_keys" { - type = list(string) - description = "The key pair to use to launch the compute host." -} - -variable "management_image_name" { - type = string - default = "ibm-redhat-8-6-minimal-amd64-5" - description = "Image name to use for provisioning the management cluster instances." -} - -variable "management_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 3 - }] - description = "Number of instances to be launched for management." -} - -variable "static_compute_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 0 - }] - description = "Min Number of instances to be launched for compute cluster." -} - -variable "dynamic_compute_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "cx2-2x4" - count = 250 - }] - description = "MaxNumber of instances to be launched for compute cluster." -} - -variable "compute_image_name" { - type = string - default = "ibm-redhat-8-6-minimal-amd64-5" - description = "Image name to use for provisioning the compute cluster instances." -} -# Future use -/* -variable "compute_gui_username" { - type = string - default = "admin" - sensitive = true - description = "GUI user to perform system management and monitoring tasks on compute cluster." -} - -variable "compute_gui_password" { - type = string - sensitive = true - description = "Password for compute cluster GUI" -} -*/ -############################################################################## -# Scale Storage Variables -############################################################################## - -variable "storage_subnets_cidr" { - type = list(string) - default = ["10.10.30.0/24", "10.20.30.0/24", "10.30.30.0/24"] - description = "Subnet CIDR block to launch the storage cluster host." -} - -variable "storage_ssh_keys" { - type = list(string) - description = "The key pair to use to launch the storage cluster host." -} - -variable "storage_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "bx2-2x8" - count = 3 - }] - description = "Number of instances to be launched for storage cluster." -} - -variable "storage_image_name" { - type = string - default = "ibm-redhat-8-6-minimal-amd64-5" - description = "Image name to use for provisioning the storage cluster instances." -} - -variable "protocol_subnets_cidr" { - type = list(string) - default = ["10.10.40.0/24", "10.20.40.0/24", "10.30.40.0/24"] - description = "Subnet CIDR block to launch the storage cluster host." -} - -variable "protocol_instances" { - type = list( - object({ - profile = string - count = number - }) - ) - default = [{ - profile = "bx2-2x8" - count = 2 - }] - description = "Number of instances to be launched for protocol hosts." -} -# Future use -/* -variable "storage_gui_username" { - type = string - default = "admin" - sensitive = true - description = "GUI user to perform system management and monitoring tasks on storage cluster." -} - -variable "storage_gui_password" { - type = string - sensitive = true - description = "Password for storage cluster GUI" -} -*/ - -variable "file_shares" { - type = list( - object({ - mount_path = string, - size = number, - iops = number - }) - ) - default = [{ - mount_path = "/mnt/binaries" - size = 100 - iops = 1000 - }, { - mount_path = "/mnt/data" - size = 100 - iops = 1000 - }] - description = "Custom file shares to access shared storage" -} - -variable "nsd_details" { - type = list( - object({ - profile = string - capacity = optional(number) - iops = optional(number) - }) - ) - default = [{ - profile = "custom" - size = 100 - iops = 100 - }] - description = "Storage scale NSD details" -} - -############################################################################## -# DNS Template Variables -############################################################################## - -variable "dns_instance_id" { - type = string - default = null - description = "IBM Cloud HPC DNS service instance id." -} - -variable "dns_custom_resolver_id" { - type = string - default = null - description = "IBM Cloud DNS custom resolver id." -} - -variable "dns_domain_names" { - type = object({ - compute = string - storage = string - protocol = string - }) - default = { - compute = "comp.com" - storage = "strg.com" - protocol = "ces.com" - } - description = "IBM Cloud HPC DNS domain names." -} - -############################################################################## -# Observability Variables -############################################################################## - -variable "enable_cos_integration" { - type = bool - default = true - description = "Integrate COS with HPC solution" -} - -variable "cos_instance_name" { - type = string - default = null - description = "Exiting COS instance name" -} - -variable "enable_atracker" { - type = bool - default = true - description = "Enable Activity tracker" -} - -variable "enable_vpc_flow_logs" { - type = bool - default = true - description = "Enable Activity tracker" -} - -############################################################################## -# Encryption Variables -############################################################################## - -variable "key_management" { - type = string - default = "key_protect" - description = "null/key_protect/hs_crypto" -} - -variable "hpcs_instance_name" { - type = string - default = null - description = "Hyper Protect Crypto Service instance" -} - -############################################################################## -# TODO: Auth Server (LDAP/AD) Variables -############################################################################## diff --git a/version.tf b/version.tf deleted file mode 100644 index d5cac67d..00000000 --- a/version.tf +++ /dev/null @@ -1,12 +0,0 @@ -terraform { - required_version = ">= 1.3.0, <1.7.0" - # If your module requires any terraform providers, uncomment the "required_providers" section below and add all required providers. - # Each required provider's version should be a flexible range to future proof the module's usage with upcoming minor and patch versions. - - # required_providers { - # ibm = { - # source = "IBM-Cloud/ibm" - # version = ">= 1.49.0, < 2.0.0" - # } - # } -}