From 48a3dd55ee41451645f82839013f2cb2f9e0ae19 Mon Sep 17 00:00:00 2001 From: Sudharshan Muralidharan <157102595+sudharshanibm@users.noreply.github.com> Date: Thu, 3 Apr 2025 15:06:10 +0530 Subject: [PATCH 1/2] Add terraform for provisioning s390x build cluster on ibmcloud Signed-off-by: Sudharshan Muralidharan --- .../k8s-s390x-build-cluster/bastion.tf | 129 +++++++ .../terraform/k8s-s390x-build-cluster/data.tf | 55 +++ .../k8s-s390x-build-cluster/load_balancer.tf | 50 +++ .../k8s-s390x-build-cluster/nodes.tf | 84 +++++ .../k8s-s390x-build-cluster/outputs.tf | 43 +++ .../k8s-s390x-build-cluster/providers.tf | 32 ++ .../k8s-s390x-build-cluster/variables.tf | 136 +++++++ .../k8s-s390x-build-cluster/versions.tf | 37 ++ .../terraform/k8s-s390x-infra-setup/README.md | 47 +++ .../terraform/k8s-s390x-infra-setup/main.tf | 32 ++ .../modules/resource_group/outputs.tf | 18 + .../modules/resource_group/resource_group.tf | 18 + .../modules/resource_group/versions.tf | 22 ++ .../modules/secrets_manager/outputs.tf | 22 ++ .../modules/secrets_manager/secret_manager.tf | 76 ++++ .../modules/secrets_manager/variables.tf | 16 + .../modules/secrets_manager/versions.tf | 23 ++ .../modules/vpc/outputs.tf | 40 ++ .../modules/vpc/variables.tf | 17 + .../modules/vpc/versions.tf | 23 ++ .../k8s-s390x-infra-setup/modules/vpc/vpc.tf | 343 ++++++++++++++++++ .../k8s-s390x-infra-setup/outputs.tf | 19 + .../k8s-s390x-infra-setup/providers.tf | 37 ++ .../k8s-s390x-infra-setup/var.tfvars | 1 + .../k8s-s390x-infra-setup/variables.tf | 31 ++ .../k8s-s390x-infra-setup/versions.tf | 37 ++ 26 files changed, 1388 insertions(+) create mode 100644 infra/ibmcloud/terraform/k8s-s390x-build-cluster/bastion.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-build-cluster/data.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-build-cluster/load_balancer.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-build-cluster/nodes.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-build-cluster/outputs.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-build-cluster/providers.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-build-cluster/variables.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-build-cluster/versions.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/README.md create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/main.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/outputs.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/resource_group.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/versions.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/outputs.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/secret_manager.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/variables.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/versions.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/outputs.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/variables.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/versions.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/vpc.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/outputs.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/providers.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/var.tfvars create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/variables.tf create mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/versions.tf diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/bastion.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/bastion.tf new file mode 100644 index 00000000000..8d678f29bcb --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/bastion.tf @@ -0,0 +1,129 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +resource "ibm_is_instance" "bastion" { + count = var.bastion.count + name = "bastion-s390x-${count.index + 1}" + vpc = data.ibm_is_vpc.vpc.id + zone = var.zone + profile = var.bastion.profile + image = var.image_id + keys = [ibm_is_ssh_key.k8s_ssh_key.id] + resource_group = data.ibm_resource_group.resource_group.id + primary_network_interface { + name = "public-nic" + subnet = data.ibm_is_subnet.subnet.id + security_groups = [data.ibm_is_security_group.bastion_sg.id] + } + + boot_volume { + name = "boot-vol-bastion-${count.index}" + size = var.bastion.boot_volume.size + } + + user_data = <<-EOF + #cloud-config + package_update: true + package_upgrade: true + packages: + - tcpdump + - net-tools + - iptables-persistent + write_files: + - path: /etc/ssh/sshd_config.d/99-bastion.conf + content: | + AllowTcpForwarding yes + GatewayPorts yes + PermitTunnel yes + PermitRootLogin prohibit-password + PasswordAuthentication no + ClientAliveInterval 120 + ClientAliveCountMax 3 + MaxSessions 50 + MaxStartups 50:30:100 + - path: /etc/systemd/network/10-eth1.network + content: | + [Match] + Name=eth1 + [Network] + Address=${data.ibm_is_subnet.subnet.ipv4_cidr_block} + DNS=8.8.8.8 + DNS=8.8.4.4 + runcmd: + - [sysctl, -w, net.ipv4.ip_forward=1] + - [echo, "net.ipv4.ip_forward = 1", ">>", /etc/sysctl.conf] + - [iptables, -t, nat, -A, POSTROUTING, -o, eth0, -j, MASQUERADE] + - [iptables, -A, FORWARD, -i, eth1, -o, eth0, -j, ACCEPT] + - [iptables, -A, FORWARD, -i, eth0, -o, eth1, -m, state, --state, RELATED,ESTABLISHED, -j, ACCEPT] + - [netfilter-persistent, save] + - [systemctl, restart, systemd-networkd] + - [systemctl, restart, sshd] + - [hostnamectl, set-hostname, "bastion-s390x-${count.index + 1}.s390x-vpc.cloud.ibm.com"] + - [echo, "bastion-s390x-${count.index + 1}.s390x-vpc.cloud.ibm.com", ">", /etc/hostname] + - [sed, -i, "s/^127.0.1.1.*/127.0.1.1\tbastion-s390x-${count.index + 1}.s390x-vpc.cloud.ibm.com/", /etc/hosts] + EOF +} + +resource "ibm_is_floating_ip" "bastion_fip" { + count = var.bastion.count + name = "bastion-fip-${count.index}" + target = ibm_is_instance.bastion[count.index].primary_network_interface[0].id + resource_group = data.ibm_resource_group.resource_group.id +} + +resource "time_sleep" "wait_for_bastion" { + count = var.bastion.count + depends_on = [ibm_is_floating_ip.bastion_fip] + + create_duration = "180s" # Wait 3 minutes for full initialization +} + +resource "null_resource" "bastion_setup" { + count = var.bastion.count + depends_on = [time_sleep.wait_for_bastion] + + connection { + type = "ssh" + user = "root" + host = ibm_is_floating_ip.bastion_fip[count.index].address + private_key = data.ibm_sm_arbitrary_secret.ssh_private_key.payload + timeout = "5m" + } + + provisioner "remote-exec" { + inline = [ + # Network verification + "echo '=== Network Interfaces ==='", + "ip -4 addr show", + "echo '=== Routing Table ==='", + "ip route", + "echo '=== NAT Configuration ==='", + "iptables -t nat -L -n -v", + "echo '=== IP Forwarding ==='", + "sysctl net.ipv4.ip_forward", + + # Hostname verification + "echo '=== Hostname ==='", + "hostname", + "hostnamectl", + "cat /etc/hostname", + + # Final security updates + "command -v apt-get >/dev/null && apt-get update -y && apt-get upgrade -y --security || true", + "command -v yum >/dev/null && yum update -y --security || true", + "command -v dnf >/dev/null && dnf update -y --security || true" + ] + } +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/data.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/data.tf new file mode 100644 index 00000000000..0bbedc7f9bd --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/data.tf @@ -0,0 +1,55 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +data "ibm_resource_group" "resource_group" { + name = "rg-build-cluster" +} + +data "ibm_is_vpc" "vpc" { + name = "k8s-s390x-vpc" +} + +data "ibm_is_subnet" "subnet" { + name = "k8s-s390x-subnet" +} + +data "ibm_is_security_group" "bastion_sg" { + name = "k8s-vpc-s390x-bastion-sg" + vpc = data.ibm_is_vpc.vpc.id +} + +data "ibm_is_security_group" "master_sg" { + name = "k8s-vpc-s390x-master-sg" + vpc = data.ibm_is_vpc.vpc.id +} + +data "ibm_is_security_group" "worker_sg" { + name = "k8s-vpc-s390x-worker-sg" + vpc = data.ibm_is_vpc.vpc.id +} + +data "ibm_sm_arbitrary_secret" "ssh_private_key" { + instance_id = var.secrets_manager_id + region = var.region + name = "zvsi-ssh-private-key" + secret_group_name = "default" +} + +data "ibm_sm_arbitrary_secret" "ssh_public_key" { + instance_id = var.secrets_manager_id + region = var.region + name = "zvsi-ssh-public-key" + secret_group_name = "default" +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/load_balancer.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/load_balancer.tf new file mode 100644 index 00000000000..fe26c35bede --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/load_balancer.tf @@ -0,0 +1,50 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +resource "ibm_is_lb" "k8s_load_balancer" { + name = "k8s-s390x-lb" + type = "public" + subnets = [data.ibm_is_subnet.subnet.id] + resource_group = data.ibm_resource_group.resource_group.id + security_groups = [data.ibm_is_security_group.master_sg.id] +} + +resource "ibm_is_lb_pool" "k8s_api_pool" { + name = "k8s-api-server-pool" + lb = ibm_is_lb.k8s_load_balancer.id + protocol = "tcp" + algorithm = "round_robin" + health_delay = 5 + health_retries = 2 + health_timeout = 2 + health_type = "tcp" + health_monitor_url = "/" + health_monitor_port = 6443 +} + +resource "ibm_is_lb_listener" "k8s_api_listener" { + lb = ibm_is_lb.k8s_load_balancer.id + protocol = "tcp" + port = 6443 + default_pool = ibm_is_lb_pool.k8s_api_pool.pool_id +} + +resource "ibm_is_lb_pool_member" "k8s_api_members" { + count = var.control_plane.count + lb = ibm_is_lb.k8s_load_balancer.id + pool = ibm_is_lb_pool.k8s_api_pool.pool_id + port = 6443 + target_address = ibm_is_instance.control_plane[count.index].primary_network_interface[0].primary_ipv4_address +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/nodes.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/nodes.tf new file mode 100644 index 00000000000..2a0323fae64 --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/nodes.tf @@ -0,0 +1,84 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +resource "ibm_is_ssh_key" "k8s_ssh_key" { + name = "k8s-s390x-ssh-key" + public_key = data.ibm_sm_arbitrary_secret.ssh_public_key.payload + resource_group = data.ibm_resource_group.resource_group.id +} + +resource "ibm_is_instance" "control_plane" { + count = var.control_plane.count + name = "master-s390x-${count.index + 1}" + vpc = data.ibm_is_vpc.vpc.id + zone = var.zone + profile = var.control_plane.profile + image = var.image_id + keys = [ibm_is_ssh_key.k8s_ssh_key.id] + resource_group = data.ibm_resource_group.resource_group.id + + primary_network_interface { + subnet = data.ibm_is_subnet.subnet.id + security_groups = [data.ibm_is_security_group.master_sg.id] + } + + boot_volume { + name = "boot-vol-master-${count.index + 1}" + size = var.control_plane.boot_volume.size + } +} + +resource "ibm_is_instance" "compute" { + count = var.compute.count + name = "worker-s390x-${count.index + 1}" + vpc = data.ibm_is_vpc.vpc.id + zone = var.zone + profile = var.compute.profile + image = var.image_id + keys = [ibm_is_ssh_key.k8s_ssh_key.id] + resource_group = data.ibm_resource_group.resource_group.id + + primary_network_interface { + subnet = data.ibm_is_subnet.subnet.id + security_groups = [data.ibm_is_security_group.worker_sg.id] + } + + boot_volume { + name = "boot-vol-worker-${count.index + 1}" + size = var.compute.boot_volume.size + } +} + +resource "null_resource" "node_setup" { + for_each = merge( + { for idx, instance in ibm_is_instance.control_plane : "master-${idx}" => instance }, + { for idx, instance in ibm_is_instance.compute : "worker-${idx}" => instance } + ) + depends_on = [time_sleep.wait_for_bastion] + triggers = { + instance_id = each.value.id + retry_count = 0 + } + connection { + type = "ssh" + user = "root" + host = each.value.primary_network_interface[0].primary_ipv4_address + private_key = data.ibm_sm_arbitrary_secret.ssh_private_key.payload + bastion_host = ibm_is_floating_ip.bastion_fip[0].address + timeout = "5m" + agent = false + + } +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/outputs.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/outputs.tf new file mode 100644 index 00000000000..0b43c09eade --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/outputs.tf @@ -0,0 +1,43 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +output "bastion_public_ips" { + description = "Public Floating IP(s) assigned to the bastion" + value = ibm_is_floating_ip.bastion_fip[*].address +} + +output "bastion_private_ips" { + description = "Private IP address(es) of the bastion host(s)" + value = ibm_is_instance.bastion[*].primary_network_interface[0].primary_ipv4_address +} + +output "master_node_ips" { + description = "Private IP addresses of control plane nodes" + value = ibm_is_instance.control_plane[*].primary_network_interface[0].primary_ipv4_address +} + +output "worker_node_ips" { + description = "Private IP addresses of worker nodes" + value = ibm_is_instance.compute[*].primary_network_interface[0].primary_ipv4_address +} + +output "api_load_balancer_hostname" { + description = "Hostname of the Kubernetes API load balancer" + value = ibm_is_lb.k8s_load_balancer.hostname +} +output "subnet_cidr" { + description = "CIDR block of the public subnet" + value = data.ibm_is_subnet.subnet.ipv4_cidr_block +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/providers.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/providers.tf new file mode 100644 index 00000000000..aca0b026bf7 --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/providers.tf @@ -0,0 +1,32 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +locals { + key = var.ibmcloud_api_key + region = "eu-de" + zone = "eu-de-1" +} + +provider "ibm" { + ibmcloud_api_key = local.key + region = local.region + zone = local.zone +} + +provider "ibm" { + alias = "vpc" + ibmcloud_api_key = local.key + region = "eu-de" +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/variables.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/variables.tf new file mode 100644 index 00000000000..98d458cae11 --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/variables.tf @@ -0,0 +1,136 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +variable "ibmcloud_api_key" { + type = string + description = "IBM Cloud API key" + sensitive = true +} + +variable "region" { + type = string + description = "IBM Cloud region" + default = "eu-de" +} + +variable "zone" { + type = string + description = "IBM Cloud zone" + default = "eu-de-1" +} + +variable "resource_group_name" { + type = string + description = "Resource group name" + default = "rg-build-cluster" +} + +variable "image_id" { + type = string + description = "Image ID for instances" + default = "r010-1476c687-824d-4b10-b604-7538defb3c73" +} + +variable "keypair_name" { + type = string + description = "SSH key pair name" + default = "k8s-sshkey" +} + +variable "secrets_manager_id" { + type = string + description = "The instance ID of your secrets manager" + default = "" + + validation { + condition = var.secrets_manager_id != "" + error_message = "The secrets_manager_id is required and cannot be empty." + } +} + +variable "bastion" { + type = object({ + count = number + profile = string + boot_volume = object({ + size = number + }) + }) + default = { + count = 1 + profile = "bz2-8x32" + boot_volume = { + size = 100 + } + } +} + +variable "control_plane" { + type = object({ + count = number + profile = string + boot_volume = object({ + size = number + }) + }) + default = { + count = 5 + profile = "bz2-8x32" + boot_volume = { + size = 100 + } + } +} + +variable "compute" { + type = object({ + count = number + profile = string + boot_volume = object({ + size = number + }) + }) + default = { + count = 10 + profile = "bz2-8x32" + boot_volume = { + size = 100 + } + } +} + +variable "connection_timeout" { + description = "Timeout in minutes for SSH connections" + type = number + default = 2 +} + +variable "bastion_boot_volume_size" { + description = "Size of the bastion boot volume in GB" + type = number + default = 100 +} + +variable "bastion_private_ip" { + description = "Private IP address for the bastion's secondary interface" + type = string + default = "192.168.100.10" +} + +variable "bastion_profile" { + description = "Instance profile for the bastion host" + type = string + default = "bz2-8x32" +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/versions.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/versions.tf new file mode 100644 index 00000000000..71b95a24213 --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/versions.tf @@ -0,0 +1,37 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +terraform { + backend "s3" { + bucket = "k8s-infra-z-eu-cos-bucket" + key = "k8s-s390x-build-cluster/terraform.tfstate" + region = "eu-geo" + skip_region_validation = true + skip_requesting_account_id = true + skip_credentials_validation = true + skip_s3_checksum = true + endpoints = { + s3 = "https://s3.eu.cloud-object-storage.appdomain.cloud" + } + secret_key = "" + access_key = "" + } + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + } + } +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/README.md b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/README.md new file mode 100644 index 00000000000..ff00f5773bd --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/README.md @@ -0,0 +1,47 @@ +# _TF: IBM K8s Account Infrastructure_ +This Terraform configuration sets up an organized structure for deploying various IBM Cloud resources using modules. + +--- +# To run the automation, follow these steps in order: + +**1. Navigate to the correct directory** +
You need to be in the `k8s-s390x-infra-setup` directory to run the automation. + +**2. Check the `versions.tf` file** +
Set `secret_key` and `access_key` in `versions.tf` to configure the remote S3 backend (IBM Cloud COS). + +**3. Initialize Terraform** +
Execute the following command to initialize Terraform in your project directory. This command will download the necessary provider plugins and prepare the working environment. +``` +terraform init -reconfigure +``` + +**4. Check the `variables.tf` file** +
Open the `variables.tf` file to review all the available variables. This file lists all customizable inputs for your Terraform configuration. + +`ibmcloud_api_key` is the only required variable that you must set in order to proceed. You can set this key either by adding it to your `var.tfvars` file or by exporting it as an environment variable. + +**Option 1:** Set in `var.tfvars` file +Add the following line to the `var.tfvars` file: +``` +ibmcloud_api_key = "" +``` + +**Option 2:** Export as an environment variable +Alternatively, you can export the ibmcloud_api_key as an environment variable before running Terraform: +``` +export TF_VAR_ibmcloud_api_key="" +``` + +**5. Run Terraform Apply** +
After setting the necessary variables (particularly the API_KEY), execute the following command to apply the Terraform configuration and provision the infrastructure: +``` +terraform apply -var-file var.tfvars +``` +Terraform will display a plan of the actions it will take, and you'll be prompted to confirm the execution. Type `yes` to proceed. + +**6 .Get Output Information** +
Once the infrastructure has been provisioned, use the terraform output command to list details about the provisioned resources. +``` +terraform output +``` diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/main.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/main.tf new file mode 100644 index 00000000000..2508a0b540d --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/main.tf @@ -0,0 +1,32 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +module "resource_group" { + source = "./modules/resource_group" +} + +module "secrets_manager" { + source = "./modules/secrets_manager" + resource_group_id = module.resource_group.build_resource_group_id +} +module "vpc" { + providers = { + ibm = ibm.vpc + } + source = "./modules/vpc" + zone = var.zone + resource_group_id = module.resource_group.build_resource_group_id +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/outputs.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/outputs.tf new file mode 100644 index 00000000000..bbcfc503e87 --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/outputs.tf @@ -0,0 +1,18 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +output "build_resource_group_id" { + value = ibm_resource_group.build_resource_group.id +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/resource_group.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/resource_group.tf new file mode 100644 index 00000000000..78595246a53 --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/resource_group.tf @@ -0,0 +1,18 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +resource "ibm_resource_group" "build_resource_group" { + name = "rg-build-cluster" +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/versions.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/versions.tf new file mode 100644 index 00000000000..bd694dea3a8 --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/versions.tf @@ -0,0 +1,22 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +terraform { + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + } + } +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/outputs.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/outputs.tf new file mode 100644 index 00000000000..21f12242cb4 --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/outputs.tf @@ -0,0 +1,22 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +output "k8s_secrets_manager_id" { + value = data.ibm_resource_instance.existing_sm.id != "" ? data.ibm_resource_instance.existing_sm.guid : ibm_resource_instance.secrets_manager[0].guid +} + +output "k8s_z_ssh_public_key" { + value = tls_private_key.private_key.public_key_openssh +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/secret_manager.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/secret_manager.tf new file mode 100644 index 00000000000..95fca7fbba4 --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/secret_manager.tf @@ -0,0 +1,76 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +locals { + secrets_manager_region = "eu-de" + secrets_manager_name = "k8s-s390x-secrets-manager" + z_service_cred_secret_name = "k8s-s390x-sm-service-credentials-secret" +} + +# Try to find existing Secrets Manager by name +data "ibm_resource_instance" "existing_sm" { + name = local.secrets_manager_name + location = local.secrets_manager_region + service = "secrets-manager" +} + +# Create Secrets Manager only if not found +resource "ibm_resource_instance" "secrets_manager" { + count = data.ibm_resource_instance.existing_sm.id != "" ? 0 : 1 + name = local.secrets_manager_name + resource_group_id = var.resource_group_id + service = "secrets-manager" + plan = "standard" + location = local.secrets_manager_region + service_endpoints = "public-and-private" + + timeouts { + create = "15m" + update = "15m" + delete = "15m" + } +} + +# Get correct Secrets Manager ID (existing or newly created) +locals { + secrets_manager_id = ( + length(data.ibm_resource_instance.existing_sm) > 0 ? + data.ibm_resource_instance.existing_sm.guid : + ibm_resource_instance.secrets_manager[0].guid + ) +} + + +# Generate RSA key +resource "tls_private_key" "private_key" { + algorithm = "RSA" + rsa_bits = 4096 +} +# Arbitrary secrets +resource "ibm_sm_arbitrary_secret" "z_ssh_private_key" { + name = "zvsi-ssh-private-key" + instance_id = local.secrets_manager_id + region = local.secrets_manager_region + labels = ["zvsi-ssh-private-key"] + payload = tls_private_key.private_key.private_key_openssh +} + +resource "ibm_sm_arbitrary_secret" "z_ssh_public_key" { + name = "zvsi-ssh-public-key" + instance_id = local.secrets_manager_id + region = local.secrets_manager_region + labels = ["zvsi-ssh-public-key"] + payload = tls_private_key.private_key.public_key_openssh +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/variables.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/variables.tf new file mode 100644 index 00000000000..e09671f03cd --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/variables.tf @@ -0,0 +1,16 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +variable "resource_group_id" {} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/versions.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/versions.tf new file mode 100644 index 00000000000..e51f5b24c6b --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/versions.tf @@ -0,0 +1,23 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +terraform { + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + } + } +} + diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/outputs.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/outputs.tf new file mode 100644 index 00000000000..9137e64970c --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/outputs.tf @@ -0,0 +1,40 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +output "vpc_id" { + value = ibm_is_vpc.vpc.id +} +output "subnet_id" { + value = ibm_is_subnet.subnet.id +} + +output "bastion_sg_id" { + value = ibm_is_security_group.bastion_sg.id +} + +output "crn" { + value = ibm_is_vpc.vpc.crn +} +output "bastion_sg_name" { + value = ibm_is_security_group.bastion_sg.name +} + +output "master_sg_name" { + value = ibm_is_security_group.master_sg.name +} + +output "worker_sg_name" { + value = ibm_is_security_group.worker_sg.name +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/variables.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/variables.tf new file mode 100644 index 00000000000..7033b8873a0 --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/variables.tf @@ -0,0 +1,17 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +variable "resource_group_id" {} +variable "zone" {} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/versions.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/versions.tf new file mode 100644 index 00000000000..3987f68e95a --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/versions.tf @@ -0,0 +1,23 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +terraform { + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + } + } +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/vpc.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/vpc.tf new file mode 100644 index 00000000000..3fb48cbebf9 --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/vpc.tf @@ -0,0 +1,343 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +resource "ibm_is_vpc" "vpc" { + name = "k8s-s390x-vpc" + resource_group = var.resource_group_id +} + +# VPC +resource "ibm_is_public_gateway" "public_gw" { + name = "k8s-s390x-public-gw" + vpc = ibm_is_vpc.vpc.id + zone = var.zone + resource_group = var.resource_group_id +} + +# Subnet +resource "ibm_is_subnet" "subnet" { + name = "k8s-s390x-subnet" + vpc = ibm_is_vpc.vpc.id + zone = var.zone + resource_group = var.resource_group_id + total_ipv4_address_count = 256 + public_gateway = ibm_is_public_gateway.public_gw.id +} + +# Security Groups +resource "ibm_is_security_group" "bastion_sg" { + name = "k8s-vpc-s390x-bastion-sg" + vpc = ibm_is_vpc.vpc.id + resource_group = var.resource_group_id +} + +resource "ibm_is_security_group" "master_sg" { + name = "k8s-vpc-s390x-master-sg" + vpc = ibm_is_vpc.vpc.id + resource_group = var.resource_group_id +} + +resource "ibm_is_security_group" "worker_sg" { + name = "k8s-vpc-s390x-worker-sg" + vpc = ibm_is_vpc.vpc.id + resource_group = var.resource_group_id +} + +# Security Group Rules +resource "ibm_is_security_group_rule" "bastion_inbound_ssh" { + group = ibm_is_security_group.bastion_sg.id + direction = "inbound" + remote = "0.0.0.0/0" + tcp { + port_min = 22 + port_max = 22 + } +} + +resource "ibm_is_security_group_rule" "bastion_outbound_all" { + group = ibm_is_security_group.bastion_sg.id + direction = "outbound" + remote = "0.0.0.0/0" +} + +## Master Rules + +resource "ibm_is_security_group_rule" "worker_inbound_ssh_from_bastion" { + group = ibm_is_security_group.worker_sg.id + direction = "inbound" + remote = ibm_is_security_group.bastion_sg.id + tcp { + port_min = 22 + port_max = 22 + } +} + + +resource "ibm_is_security_group_rule" "worker_internal" { + group = ibm_is_security_group.worker_sg.id + direction = "inbound" + remote = ibm_is_subnet.subnet.ipv4_cidr_block +} + +resource "ibm_is_security_group_rule" "worker_master_all" { + group = ibm_is_security_group.worker_sg.id + direction = "inbound" + remote = ibm_is_security_group.master_sg.id +} + +resource "ibm_is_security_group_rule" "bastion_private_inbound" { + group = ibm_is_security_group.bastion_sg.id + direction = "inbound" + remote = ibm_is_subnet.subnet.ipv4_cidr_block +} + +resource "ibm_is_security_group_rule" "bastion_private_outbound" { + group = ibm_is_security_group.bastion_sg.id + direction = "outbound" + remote = ibm_is_subnet.subnet.ipv4_cidr_block +} + + +resource "ibm_is_security_group_rule" "master_to_worker_kubelet_api" { + group = ibm_is_security_group.worker_sg.id + direction = "inbound" + remote = ibm_is_security_group.master_sg.id + tcp { + port_min = 10250 + port_max = 10250 + } +} +resource "ibm_is_security_group_rule" "allow_master_to_worker_all" { + group = ibm_is_security_group.worker_sg.id + direction = "inbound" + remote = ibm_is_security_group.master_sg.id + tcp { + port_min = 1 + port_max = 65535 + } +} + +resource "ibm_is_security_group_rule" "allow_bastion_ssh_to_worker" { + group = ibm_is_security_group.worker_sg.id + direction = "inbound" + remote = ibm_is_security_group.bastion_sg.id + tcp { + port_min = 22 + port_max = 22 + } +} + +resource "ibm_is_security_group_rule" "allow_vpc_cidr_to_worker" { + group = ibm_is_security_group.worker_sg.id + direction = "inbound" + remote = "10.243.0.0/24" + tcp { + port_min = 1 + port_max = 65535 + } +} +resource "ibm_is_security_group_rule" "outbound_http" { + group = ibm_is_security_group.worker_sg.id + direction = "outbound" + remote = "0.0.0.0/0" + tcp { + port_min = 80 + port_max = 80 + } +} + +resource "ibm_is_security_group_rule" "outbound_https" { + group = ibm_is_security_group.worker_sg.id + direction = "outbound" + remote = "0.0.0.0/0" + tcp { + port_min = 443 + port_max = 443 + } +} + +resource "ibm_is_security_group_rule" "outbound_dns_tcp" { + group = ibm_is_security_group.worker_sg.id + direction = "outbound" + remote = "0.0.0.0/0" + tcp { + port_min = 53 + port_max = 53 + } +} + +resource "ibm_is_security_group_rule" "outbound_dns_udp" { + group = ibm_is_security_group.worker_sg.id + direction = "outbound" + remote = "0.0.0.0/0" + udp { + port_min = 53 + port_max = 53 + } +} + +resource "ibm_is_security_group_rule" "outbound_k8s_api" { + group = ibm_is_security_group.worker_sg.id + direction = "outbound" + remote = "0.0.0.0/0" + tcp { + port_min = 6443 + port_max = 6443 + } +} + +resource "ibm_is_security_group_rule" "worker_outbound_to_all" { + group = ibm_is_security_group.worker_sg.id + direction = "outbound" + remote = "0.0.0.0/0" +} + +resource "ibm_is_security_group_rule" "worker_pod_inbound" { + group = ibm_is_security_group.worker_sg.id + direction = "inbound" + remote = ibm_is_subnet.subnet.ipv4_cidr_block + tcp { + port_min = 10250 + port_max = 10250 + } +} + +resource "ibm_is_security_group_rule" "worker_pod_outbound" { + group = ibm_is_security_group.worker_sg.id + direction = "outbound" + remote = ibm_is_subnet.subnet.ipv4_cidr_block + tcp { + port_min = 10250 + port_max = 10250 + } +} + +resource "ibm_is_security_group_rule" "master_inbound_from_workers" { + group = ibm_is_security_group.master_sg.id + direction = "inbound" + remote = ibm_is_security_group.worker_sg.id +} + +resource "ibm_is_security_group_rule" "master_inbound_from_bastion_ssh" { + group = ibm_is_security_group.master_sg.id + direction = "inbound" + remote = ibm_is_security_group.bastion_sg.id + tcp { + port_min = 22 + port_max = 22 + } +} + +resource "ibm_is_security_group_rule" "master_inbound_from_internal_cidr" { + group = ibm_is_security_group.master_sg.id + direction = "inbound" + remote = "10.243.0.0/24" # adjust based on your CIDR +} + +resource "ibm_is_security_group_rule" "master_inbound_from_self" { + group = ibm_is_security_group.master_sg.id + direction = "inbound" + remote = ibm_is_security_group.master_sg.id +} + +resource "ibm_is_security_group_rule" "master_inbound_api" { + group = ibm_is_security_group.master_sg.id + direction = "inbound" + remote = "0.0.0.0/0" + tcp { + port_min = 6443 + port_max = 6443 + } +} +resource "ibm_is_security_group_rule" "master_outbound_http" { + group = ibm_is_security_group.master_sg.id + direction = "outbound" + tcp { + port_min = 80 + port_max = 80 + } + remote = "0.0.0.0/0" +} + +resource "ibm_is_security_group_rule" "master_outbound_https" { + group = ibm_is_security_group.master_sg.id + direction = "outbound" + tcp { + port_min = 443 + port_max = 443 + } + remote = "0.0.0.0/0" +} + +resource "ibm_is_security_group_rule" "master_outbound_dns_tcp" { + group = ibm_is_security_group.master_sg.id + direction = "outbound" + tcp { + port_min = 53 + port_max = 53 + } + remote = "0.0.0.0/0" +} + +resource "ibm_is_security_group_rule" "master_outbound_dns_udp" { + group = ibm_is_security_group.master_sg.id + direction = "outbound" + udp { + port_min = 53 + port_max = 53 + } + remote = "0.0.0.0/0" +} + +resource "ibm_is_security_group_rule" "master_outbound_api" { + group = ibm_is_security_group.master_sg.id + direction = "outbound" + tcp { + port_min = 6443 + port_max = 6443 + } + remote = "0.0.0.0/0" +} + +resource "ibm_is_security_group_rule" "master_outbound_to_workers" { + group = ibm_is_security_group.master_sg.id + direction = "outbound" + remote = ibm_is_security_group.master_sg.id +} + +resource "ibm_is_security_group_rule" "master_outbound_to_all" { + group = ibm_is_security_group.master_sg.id + direction = "outbound" + remote = "0.0.0.0/0" +} +resource "ibm_is_security_group_rule" "master_pod_inbound" { + group = ibm_is_security_group.master_sg.id + direction = "inbound" + remote = ibm_is_subnet.subnet.ipv4_cidr_block + tcp { + port_min = 10250 + port_max = 10250 + } +} + +resource "ibm_is_security_group_rule" "master_pod_outbound" { + group = ibm_is_security_group.master_sg.id + direction = "outbound" + remote = ibm_is_subnet.subnet.ipv4_cidr_block + tcp { + port_min = 10250 + port_max = 10250 + } +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/outputs.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/outputs.tf new file mode 100644 index 00000000000..7d7d844746d --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/outputs.tf @@ -0,0 +1,19 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +output "secrets_manager_id" { + value = module.secrets_manager.k8s_secrets_manager_id +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/providers.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/providers.tf new file mode 100644 index 00000000000..6a48d3a19db --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/providers.tf @@ -0,0 +1,37 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +locals { + key = var.ibmcloud_api_key + region = "eu-de" + zone = "eu-de-1" +} + +provider "ibm" { + ibmcloud_api_key = local.key + region = local.region + zone = local.zone +} +provider "ibm" { + alias = "vpc" + ibmcloud_api_key = local.key + region = local.region +} +provider "ibm" { + alias = "vpcs" + ibmcloud_api_key = local.key + region = local.region +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/var.tfvars b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/var.tfvars new file mode 100644 index 00000000000..01d700615fb --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/var.tfvars @@ -0,0 +1 @@ +ibmcloud_api_key = "" diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/variables.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/variables.tf new file mode 100644 index 00000000000..a14f0e9d580 --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/variables.tf @@ -0,0 +1,31 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +variable "ibmcloud_api_key" { + type = string + description = "IBM Cloud API key associated with user's identity" + sensitive = true + + validation { + condition = var.ibmcloud_api_key != "" + error_message = "The ibmcloud_api_key is required and cannot be empty." + } +} +variable "zone" { + description = "IBM Cloud zone for resources" + type = string + default = "eu-de-1" +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/versions.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/versions.tf new file mode 100644 index 00000000000..12b72c5229b --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/versions.tf @@ -0,0 +1,37 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +terraform { + backend "s3" { + bucket = "k8s-infra-z-eu-cos-bucket" + key = "k8s-s390x-infra-setup/terraform.tfstate" + region = "eu-geo" + skip_region_validation = true + skip_requesting_account_id = true + skip_credentials_validation = true + skip_s3_checksum = true + endpoints = { + s3 = "https://s3.eu.cloud-object-storage.appdomain.cloud" + } + secret_key = "" + access_key = "" + } + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + } + } +} From dd304194b7a621f4511098178e5d12bb2d2b8fa8 Mon Sep 17 00:00:00 2001 From: Sudharshan Muralidharan <157102595+sudharshanibm@users.noreply.github.com> Date: Mon, 15 Sep 2025 22:34:17 +0530 Subject: [PATCH 2/2] Updated Reviewed Changes --- .../k8s-s390x-build-cluster/README.md | 127 ++++++++++++++++++ .../k8s-s390x-build-cluster/bastion.tf | 85 ++++-------- .../terraform/k8s-s390x-build-cluster/data.tf | 6 +- .../k8s-s390x-build-cluster/load_balancer.tf | 23 ++-- .../k8s-s390x-build-cluster/nodes.tf | 67 ++++----- .../k8s-s390x-build-cluster/outputs.tf | 26 ++-- .../k8s-s390x-build-cluster/variables.tf | 89 +++++------- .../k8s-s390x-build-cluster/versions.tf | 17 ++- .../terraform/k8s-s390x-infra-setup/README.md | 18 ++- .../terraform/k8s-s390x-infra-setup/main.tf | 9 +- .../modules/resource_group/outputs.tf | 18 --- .../modules/resource_group/resource_group.tf | 18 --- .../modules/resource_group/versions.tf | 22 --- .../modules/secrets_manager/outputs.tf | 2 +- .../modules/secrets_manager/secret_manager.tf | 27 +--- .../modules/secrets_manager/variables.tf | 5 +- .../modules/vpc/outputs.tf | 4 +- .../k8s-s390x-infra-setup/modules/vpc/vpc.tf | 80 +++++------ .../k8s-s390x-infra-setup/outputs.tf | 19 --- .../k8s-s390x-infra-setup/providers.tf | 5 - .../k8s-s390x-infra-setup/versions.tf | 17 ++- 21 files changed, 344 insertions(+), 340 deletions(-) create mode 100644 infra/ibmcloud/terraform/k8s-s390x-build-cluster/README.md delete mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/outputs.tf delete mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/resource_group.tf delete mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/versions.tf delete mode 100644 infra/ibmcloud/terraform/k8s-s390x-infra-setup/outputs.tf diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/README.md b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/README.md new file mode 100644 index 00000000000..7c3fcd86a6a --- /dev/null +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/README.md @@ -0,0 +1,127 @@ +# _TF: IBM K8s s390x Build Cluster_ +These terraform resources define a IBM Cloud project containing a s390xVS cluster intended to serve as a "build cluster" for prow.k8s.io. + +--- +## Initial Setup + +### Supporting infrastructure + +#### Deploy k8s-infra-setup resources + +- this covers things like Resource Group, s390x Virtual Server Instances, Virtual Private Cloud, IBM Cloud Secret Manager Secrets, etc. +- Once the deployment successfully completes, the `secrets_manager_id` will be generated and should be used in the subsequent steps. + +--- +#### Deploy k8s-s390x-build-cluster resources + +**1. Navigate to the correct directory** +
You need to be in the `k8s-s390x-build-cluster` directory to run the automation. + +**2. Export COS Secrets** +
Export `access_key` and `secret_key` as environment variables. +``` +export AWS_ACCESS_KEY_ID="" +export AWS_SECRET_ACCESS_KEY="" +``` + +**3. Initialize Terraform** +
Execute the following command to initialize Terraform in your project directory. This command will download the necessary provider plugins and prepare the working environment. +``` +terraform init -reconfigure +``` + +**4. Check the `variables.tf` file** +
Open the `variables.tf` file to review all the available variables. This file lists all customizable inputs for your Terraform configuration. + +`ibmcloud_api_key`, `secrets_manager_id` are the only required variables that you must set in order to proceed. You can set this key either by adding it to your `var.tfvars` file or by exporting it as an environment variable. + +**Option 1:** Set in `var.tfvars` file +Create `var.tfvars` file and set the following variables in `var.tfvars` file: +``` +ibmcloud_api_key = "" +secrets_manager_id = "" +``` +Tip: To get the secrets_manager_id (GUID) for IBM Cloud Secrets Manager instance: +``` +ibmcloud resource service-instances --service-name secrets-manager --output JSON | \ +jq -r '.[] | select(.name | contains("k8s-s390x")) | .guid' +``` +**Option 2:** Export as an environment variable +Alternatively, you can export above as an environment variable before running Terraform: +``` +export TF_VAR_ibmcloud_api_key="" +export TF_VAR_secrets_manager_id=$(ibmcloud resource service-instances --service-name secrets-manager --output JSON | \ +jq -r '.[] | select(.name | contains("k8s-s390x")) | .guid') +``` + +**5. Run Terraform Apply** +
After setting the necessary variables (particularly the API_KEY), execute the following command to apply the Terraform configuration and provision the infrastructure: +``` +terraform apply -var-file var.tfvars +``` +Terraform will display a plan of the actions it will take, and you'll be prompted to confirm the execution. Type `yes` to proceed. + +**6. Get Output Information** +
Once the infrastructure has been provisioned, use the terraform output command to list details about the provisioned resources. +``` +terraform output +``` + +**7. Set up the Kubernetes cluster using ansible** +Clone the repository `https://github.com/kubernetes-sigs/provider-ibmcloud-test-infra` and change the directory to `kubetest2-tf/data/k8s-ansible`: +``` +cd kubetest2-tf/data/k8s-ansible +``` + +**8. Install ansible on the deployer VM** +``` +dnf install ansible -y +``` + +**9. Update the fields under `group_vars/all` to include the Kubernetes version to install** +
The following lines will update the version to the latest stable release of Kubernetes. You can modify it accordingly to set up the CI (alpha) version. +``` +K8S_VERSION=$(curl -Ls https://dl.k8s.io/release/stable.txt) +LOADBALANCER_EP= +sed -i \ +-e "s/^directory: .*/directory: release/" \ +-e "s/build_version: .*/build_version: $K8S_VERSION/" \ +-e "s/release_marker: .*/release_marker: $K8S_VERSION/" \ +-e "s/loadbalancer: .*/loadbalancer: $LOADBALANCER_EP/" group_vars/all +``` + +**10. Update the fields under `examples/k8s-build-cluster/hosts.yml` to contain IP addresses of the VMs to set up Kubernetes** +``` +For example: + +[bastion] +56.77.34.6 + +[masters] +192.168.100.3 +192.168.100.4 + +[workers] +192.168.100.5 +192.168.100.6 +192.168.100.7 + +[workers:vars] +ansible_ssh_common_args='-o ProxyCommand="ssh -W %h:%p -i -q root@56.77.34.6" -i ' + +[masters:vars] +ansible_ssh_common_args='-o ProxyCommand="ssh -W %h:%p -i -q root@56.77.34.6" -i ' +``` + +**11. Update the fields under `group_vars/bastion_configuration` to contain the information of the private network.** +``` +For example: + +bastion_private_gateway: 192.168.100.1 +bastion_private_ip: 192.168.100.2 +``` + +**12. Trigger the installation using ansible** +``` +ansible-playbook -v -i examples/k8s-build-cluster/hosts.yml install-k8s-ha.yaml -e @group_vars/bastion_configuration --extra-vars @group_vars/all +``` diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/bastion.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/bastion.tf index 8d678f29bcb..3d3be5f277c 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/bastion.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/bastion.tf @@ -13,24 +13,37 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + +locals { + bastion_nodes = { + "primary" = { + profile = var.bastion_profile + boot_volume = { + size = var.bastion_boot_volume_size + } + } + } +} + resource "ibm_is_instance" "bastion" { - count = var.bastion.count - name = "bastion-s390x-${count.index + 1}" + for_each = local.bastion_nodes + name = "bastion-s390x-${each.key}" vpc = data.ibm_is_vpc.vpc.id zone = var.zone - profile = var.bastion.profile + profile = each.value.profile image = var.image_id keys = [ibm_is_ssh_key.k8s_ssh_key.id] resource_group = data.ibm_resource_group.resource_group.id + primary_network_interface { - name = "public-nic" + name = "public-nic-${each.key}" subnet = data.ibm_is_subnet.subnet.id - security_groups = [data.ibm_is_security_group.bastion_sg.id] + security_groups = [data.ibm_is_security_group.bastion.id] } boot_volume { - name = "boot-vol-bastion-${count.index}" - size = var.bastion.boot_volume.size + name = "boot-vol-bastion-${each.key}" + size = each.value.boot_volume.size } user_data = <<-EOF @@ -70,60 +83,16 @@ resource "ibm_is_instance" "bastion" { - [netfilter-persistent, save] - [systemctl, restart, systemd-networkd] - [systemctl, restart, sshd] - - [hostnamectl, set-hostname, "bastion-s390x-${count.index + 1}.s390x-vpc.cloud.ibm.com"] - - [echo, "bastion-s390x-${count.index + 1}.s390x-vpc.cloud.ibm.com", ">", /etc/hostname] - - [sed, -i, "s/^127.0.1.1.*/127.0.1.1\tbastion-s390x-${count.index + 1}.s390x-vpc.cloud.ibm.com/", /etc/hosts] + - [hostnamectl, set-hostname, "bastion-s390x-${each.key}.s390x-vpc.cloud.ibm.com"] + - [echo, "bastion-s390x-${each.key}.s390x-vpc.cloud.ibm.com", ">", /etc/hostname] + - [sed, -i, "s/^127.0.1.1.*/127.0.1.1\tbastion-s390x-${each.key}.s390x-vpc.cloud.ibm.com/", /etc/hosts] + - [touch, /var/lib/cloud/instance/bastion-setup-success] EOF } resource "ibm_is_floating_ip" "bastion_fip" { - count = var.bastion.count - name = "bastion-fip-${count.index}" - target = ibm_is_instance.bastion[count.index].primary_network_interface[0].id + for_each = ibm_is_instance.bastion + name = "bastion-fip-${each.key}" + target = each.value.primary_network_interface[0].id resource_group = data.ibm_resource_group.resource_group.id } - -resource "time_sleep" "wait_for_bastion" { - count = var.bastion.count - depends_on = [ibm_is_floating_ip.bastion_fip] - - create_duration = "180s" # Wait 3 minutes for full initialization -} - -resource "null_resource" "bastion_setup" { - count = var.bastion.count - depends_on = [time_sleep.wait_for_bastion] - - connection { - type = "ssh" - user = "root" - host = ibm_is_floating_ip.bastion_fip[count.index].address - private_key = data.ibm_sm_arbitrary_secret.ssh_private_key.payload - timeout = "5m" - } - - provisioner "remote-exec" { - inline = [ - # Network verification - "echo '=== Network Interfaces ==='", - "ip -4 addr show", - "echo '=== Routing Table ==='", - "ip route", - "echo '=== NAT Configuration ==='", - "iptables -t nat -L -n -v", - "echo '=== IP Forwarding ==='", - "sysctl net.ipv4.ip_forward", - - # Hostname verification - "echo '=== Hostname ==='", - "hostname", - "hostnamectl", - "cat /etc/hostname", - - # Final security updates - "command -v apt-get >/dev/null && apt-get update -y && apt-get upgrade -y --security || true", - "command -v yum >/dev/null && yum update -y --security || true", - "command -v dnf >/dev/null && dnf update -y --security || true" - ] - } -} diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/data.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/data.tf index 0bbedc7f9bd..069f3827350 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/data.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/data.tf @@ -25,13 +25,13 @@ data "ibm_is_subnet" "subnet" { name = "k8s-s390x-subnet" } -data "ibm_is_security_group" "bastion_sg" { +data "ibm_is_security_group" "bastion" { name = "k8s-vpc-s390x-bastion-sg" vpc = data.ibm_is_vpc.vpc.id } -data "ibm_is_security_group" "master_sg" { - name = "k8s-vpc-s390x-master-sg" +data "ibm_is_security_group" "control_plane_sg" { + name = "k8s-vpc-s390x-control-plane-sg" vpc = data.ibm_is_vpc.vpc.id } diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/load_balancer.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/load_balancer.tf index fe26c35bede..d520c2472c3 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/load_balancer.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/load_balancer.tf @@ -13,17 +13,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -resource "ibm_is_lb" "k8s_load_balancer" { - name = "k8s-s390x-lb" +resource "ibm_is_lb" "public" { + name = "k8s-s390x-ci" type = "public" subnets = [data.ibm_is_subnet.subnet.id] resource_group = data.ibm_resource_group.resource_group.id - security_groups = [data.ibm_is_security_group.master_sg.id] + security_groups = [data.ibm_is_security_group.control_plane_sg.id] } resource "ibm_is_lb_pool" "k8s_api_pool" { name = "k8s-api-server-pool" - lb = ibm_is_lb.k8s_load_balancer.id + lb = ibm_is_lb.public.id protocol = "tcp" algorithm = "round_robin" health_delay = 5 @@ -31,20 +31,21 @@ resource "ibm_is_lb_pool" "k8s_api_pool" { health_timeout = 2 health_type = "tcp" health_monitor_url = "/" - health_monitor_port = 6443 + health_monitor_port = var.api_server_port } resource "ibm_is_lb_listener" "k8s_api_listener" { - lb = ibm_is_lb.k8s_load_balancer.id + lb = ibm_is_lb.public.id protocol = "tcp" - port = 6443 + port = var.api_server_port default_pool = ibm_is_lb_pool.k8s_api_pool.pool_id } resource "ibm_is_lb_pool_member" "k8s_api_members" { - count = var.control_plane.count - lb = ibm_is_lb.k8s_load_balancer.id + for_each = ibm_is_instance.control_plane + + lb = ibm_is_lb.public.id pool = ibm_is_lb_pool.k8s_api_pool.pool_id - port = 6443 - target_address = ibm_is_instance.control_plane[count.index].primary_network_interface[0].primary_ipv4_address + port = var.api_server_port + target_address = each.value.primary_network_interface[0].primary_ipv4_address } diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/nodes.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/nodes.tf index 2a0323fae64..511d480b3f0 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/nodes.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/nodes.tf @@ -13,6 +13,27 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +locals { + control_plane_nodes = { + for idx in range(1, var.control_plane_node_count + 1) : + "${idx}" => { + profile = var.control_plane_node_profile + boot_volume = { + size = var.control_plane_boot_volume_size + } + } + } + + compute_nodes = { + for idx in range(1, var.compute_node_count + 1) : + "${idx}" => { + profile = var.compute_node_profile + boot_volume = { + size = var.compute_boot_volume_size + } + } + } +} resource "ibm_is_ssh_key" "k8s_ssh_key" { name = "k8s-s390x-ssh-key" public_key = data.ibm_sm_arbitrary_secret.ssh_public_key.payload @@ -20,32 +41,34 @@ resource "ibm_is_ssh_key" "k8s_ssh_key" { } resource "ibm_is_instance" "control_plane" { - count = var.control_plane.count - name = "master-s390x-${count.index + 1}" + for_each = local.control_plane_nodes + + name = "control-plane-s390x-${each.key}" vpc = data.ibm_is_vpc.vpc.id zone = var.zone - profile = var.control_plane.profile + profile = each.value.profile image = var.image_id keys = [ibm_is_ssh_key.k8s_ssh_key.id] resource_group = data.ibm_resource_group.resource_group.id primary_network_interface { subnet = data.ibm_is_subnet.subnet.id - security_groups = [data.ibm_is_security_group.master_sg.id] + security_groups = [data.ibm_is_security_group.control_plane_sg.id] } boot_volume { - name = "boot-vol-master-${count.index + 1}" - size = var.control_plane.boot_volume.size + name = "boot-vol-cp-s390x-${each.key}" + size = each.value.boot_volume.size } } resource "ibm_is_instance" "compute" { - count = var.compute.count - name = "worker-s390x-${count.index + 1}" + for_each = local.compute_nodes + + name = "worker-s390x-${each.key}" vpc = data.ibm_is_vpc.vpc.id zone = var.zone - profile = var.compute.profile + profile = each.value.profile image = var.image_id keys = [ibm_is_ssh_key.k8s_ssh_key.id] resource_group = data.ibm_resource_group.resource_group.id @@ -56,29 +79,7 @@ resource "ibm_is_instance" "compute" { } boot_volume { - name = "boot-vol-worker-${count.index + 1}" - size = var.compute.boot_volume.size - } -} - -resource "null_resource" "node_setup" { - for_each = merge( - { for idx, instance in ibm_is_instance.control_plane : "master-${idx}" => instance }, - { for idx, instance in ibm_is_instance.compute : "worker-${idx}" => instance } - ) - depends_on = [time_sleep.wait_for_bastion] - triggers = { - instance_id = each.value.id - retry_count = 0 - } - connection { - type = "ssh" - user = "root" - host = each.value.primary_network_interface[0].primary_ipv4_address - private_key = data.ibm_sm_arbitrary_secret.ssh_private_key.payload - bastion_host = ibm_is_floating_ip.bastion_fip[0].address - timeout = "5m" - agent = false - + name = "boot-vol-worker-s390x-${each.key}" + size = each.value.boot_volume.size } } diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/outputs.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/outputs.tf index 0b43c09eade..35db025e621 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/outputs.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/outputs.tf @@ -13,29 +13,29 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -output "bastion_public_ips" { - description = "Public Floating IP(s) assigned to the bastion" - value = ibm_is_floating_ip.bastion_fip[*].address +output "bastion_private_ips_map" { + description = "Private IP addresses of the bastion hosts" + value = { for k, instance in ibm_is_instance.bastion : k => instance.primary_network_interface[0].primary_ipv4_address } } -output "bastion_private_ips" { - description = "Private IP address(es) of the bastion host(s)" - value = ibm_is_instance.bastion[*].primary_network_interface[0].primary_ipv4_address +output "bastion_public_ips_map" { + description = "Public IP addresses of the bastion hosts, keyed by instance name" + value = { for k, fip in ibm_is_floating_ip.bastion_fip : k => fip.address } } -output "master_node_ips" { - description = "Private IP addresses of control plane nodes" - value = ibm_is_instance.control_plane[*].primary_network_interface[0].primary_ipv4_address +output "control_plane_node_ips_map" { + description = "Private IP addresses of the control plane nodes, keyed by node name" + value = { for k, instance in ibm_is_instance.control_plane : k => instance.primary_network_interface[0].primary_ipv4_address } } -output "worker_node_ips" { - description = "Private IP addresses of worker nodes" - value = ibm_is_instance.compute[*].primary_network_interface[0].primary_ipv4_address +output "worker_node_ips_map" { + description = "Private IP addresses of the worker nodes, keyed by node name" + value = { for k, instance in ibm_is_instance.compute : k => instance.primary_network_interface[0].primary_ipv4_address } } output "api_load_balancer_hostname" { description = "Hostname of the Kubernetes API load balancer" - value = ibm_is_lb.k8s_load_balancer.hostname + value = ibm_is_lb.public.hostname } output "subnet_cidr" { description = "CIDR block of the public subnet" diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/variables.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/variables.tf index 98d458cae11..faa5590aa30 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/variables.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/variables.tf @@ -31,16 +31,10 @@ variable "zone" { default = "eu-de-1" } -variable "resource_group_name" { - type = string - description = "Resource group name" - default = "rg-build-cluster" -} - variable "image_id" { type = string description = "Image ID for instances" - default = "r010-1476c687-824d-4b10-b604-7538defb3c73" + default = "r010-bdd9c78f-4a2f-441f-a375-2ed288dcad15" } variable "keypair_name" { @@ -60,55 +54,40 @@ variable "secrets_manager_id" { } } -variable "bastion" { - type = object({ - count = number - profile = string - boot_volume = object({ - size = number - }) - }) - default = { - count = 1 - profile = "bz2-8x32" - boot_volume = { - size = 100 - } - } +variable "control_plane_node_count" { + description = "Number of control plane nodes to create" + type = number + default = 5 } -variable "control_plane" { - type = object({ - count = number - profile = string - boot_volume = object({ - size = number - }) - }) - default = { - count = 5 - profile = "bz2-8x32" - boot_volume = { - size = 100 - } - } +variable "control_plane_node_profile" { + description = "The profile to use for all control plane nodes" + type = string + default = "bz2-8x32" } -variable "compute" { - type = object({ - count = number - profile = string - boot_volume = object({ - size = number - }) - }) - default = { - count = 10 - profile = "bz2-8x32" - boot_volume = { - size = 100 - } - } +variable "control_plane_boot_volume_size" { + description = "The boot volume size (in GB) for all control plane nodes" + type = number + default = 100 +} + +variable "compute_node_count" { + description = "Number of compute worker nodes to create" + type = number + default = 10 +} + +variable "compute_node_profile" { + description = "The profile to use for all compute nodes" + type = string + default = "bz2-8x32" +} + +variable "compute_boot_volume_size" { + description = "The boot volume size (in GB) for all compute nodes" + type = number + default = 100 } variable "connection_timeout" { @@ -134,3 +113,9 @@ variable "bastion_profile" { type = string default = "bz2-8x32" } + +variable "api_server_port" { + description = "Port for the Kubernetes API server" + type = number + default = 6443 +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/versions.tf b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/versions.tf index 71b95a24213..f281b75ec98 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-build-cluster/versions.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-build-cluster/versions.tf @@ -16,22 +16,27 @@ limitations under the License. terraform { backend "s3" { - bucket = "k8s-infra-z-eu-cos-bucket" + bucket = "k8s-z-infra-tf-states" key = "k8s-s390x-build-cluster/terraform.tfstate" region = "eu-geo" skip_region_validation = true - skip_requesting_account_id = true skip_credentials_validation = true - skip_s3_checksum = true endpoints = { s3 = "https://s3.eu.cloud-object-storage.appdomain.cloud" } - secret_key = "" - access_key = "" } required_providers { ibm = { - source = "IBM-Cloud/ibm" + source = "IBM-Cloud/ibm" + version = "~> 1.82.0" + } + time = { + source = "hashicorp/time" + version = "~> 0.13.0" + } + null = { + source = "hashicorp/null" + version = "~> 3.2.0" } } } diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/README.md b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/README.md index ff00f5773bd..e69a5a4acbe 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/README.md +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/README.md @@ -1,5 +1,10 @@ # _TF: IBM K8s Account Infrastructure_ -This Terraform configuration sets up an organized structure for deploying various IBM Cloud resources using modules. +This Terraform configuration sets up an organized structure for deploying various IBM Cloud resources using following modules. + +## Modules Used: +- **vpc**: IBM Cloud VPC, subnets, and networking components +- **secrets_manager**: IBM Secrets Manager for credential storage +- **resource_group**: IBM Resource Group management --- # To run the automation, follow these steps in order: @@ -7,13 +12,16 @@ This Terraform configuration sets up an organized structure for deploying variou **1. Navigate to the correct directory**
You need to be in the `k8s-s390x-infra-setup` directory to run the automation. -**2. Check the `versions.tf` file** -
Set `secret_key` and `access_key` in `versions.tf` to configure the remote S3 backend (IBM Cloud COS). - +**2. Export COS Secrets** +
Export `access_key` and `secret_key` as environment variables. +``` +export AWS_ACCESS_KEY_ID="" +export AWS_SECRET_ACCESS_KEY="" +``` **3. Initialize Terraform**
Execute the following command to initialize Terraform in your project directory. This command will download the necessary provider plugins and prepare the working environment. ``` -terraform init -reconfigure +terraform init -upgrade ``` **4. Check the `variables.tf` file** diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/main.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/main.tf index 2508a0b540d..44d076ab914 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/main.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/main.tf @@ -13,14 +13,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - -module "resource_group" { - source = "./modules/resource_group" +resource "ibm_resource_group" "build_resource_group" { + name = "rg-build-cluster" } module "secrets_manager" { source = "./modules/secrets_manager" - resource_group_id = module.resource_group.build_resource_group_id + resource_group_id = ibm_resource_group.build_resource_group.id } module "vpc" { providers = { @@ -28,5 +27,5 @@ module "vpc" { } source = "./modules/vpc" zone = var.zone - resource_group_id = module.resource_group.build_resource_group_id + resource_group_id = ibm_resource_group.build_resource_group.id } diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/outputs.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/outputs.tf deleted file mode 100644 index bbcfc503e87..00000000000 --- a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2025 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -output "build_resource_group_id" { - value = ibm_resource_group.build_resource_group.id -} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/resource_group.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/resource_group.tf deleted file mode 100644 index 78595246a53..00000000000 --- a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/resource_group.tf +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2025 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -resource "ibm_resource_group" "build_resource_group" { - name = "rg-build-cluster" -} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/versions.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/versions.tf deleted file mode 100644 index bd694dea3a8..00000000000 --- a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/resource_group/versions.tf +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2025 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -terraform { - required_providers { - ibm = { - source = "IBM-Cloud/ibm" - } - } -} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/outputs.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/outputs.tf index 21f12242cb4..6d876bae390 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/outputs.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/outputs.tf @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ output "k8s_secrets_manager_id" { - value = data.ibm_resource_instance.existing_sm.id != "" ? data.ibm_resource_instance.existing_sm.guid : ibm_resource_instance.secrets_manager[0].guid + value = ibm_resource_instance.secrets_manager.guid } output "k8s_z_ssh_public_key" { diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/secret_manager.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/secret_manager.tf index 95fca7fbba4..140a4e3a5a5 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/secret_manager.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/secret_manager.tf @@ -13,22 +13,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + locals { secrets_manager_region = "eu-de" secrets_manager_name = "k8s-s390x-secrets-manager" z_service_cred_secret_name = "k8s-s390x-sm-service-credentials-secret" } -# Try to find existing Secrets Manager by name -data "ibm_resource_instance" "existing_sm" { - name = local.secrets_manager_name - location = local.secrets_manager_region - service = "secrets-manager" -} - -# Create Secrets Manager only if not found resource "ibm_resource_instance" "secrets_manager" { - count = data.ibm_resource_instance.existing_sm.id != "" ? 0 : 1 name = local.secrets_manager_name resource_group_id = var.resource_group_id service = "secrets-manager" @@ -43,25 +35,16 @@ resource "ibm_resource_instance" "secrets_manager" { } } -# Get correct Secrets Manager ID (existing or newly created) -locals { - secrets_manager_id = ( - length(data.ibm_resource_instance.existing_sm) > 0 ? - data.ibm_resource_instance.existing_sm.guid : - ibm_resource_instance.secrets_manager[0].guid - ) -} - - # Generate RSA key resource "tls_private_key" "private_key" { algorithm = "RSA" rsa_bits = 4096 } -# Arbitrary secrets + +# Arbitrary secrets - reference the resource directly resource "ibm_sm_arbitrary_secret" "z_ssh_private_key" { name = "zvsi-ssh-private-key" - instance_id = local.secrets_manager_id + instance_id = ibm_resource_instance.secrets_manager.guid # Direct reference region = local.secrets_manager_region labels = ["zvsi-ssh-private-key"] payload = tls_private_key.private_key.private_key_openssh @@ -69,7 +52,7 @@ resource "ibm_sm_arbitrary_secret" "z_ssh_private_key" { resource "ibm_sm_arbitrary_secret" "z_ssh_public_key" { name = "zvsi-ssh-public-key" - instance_id = local.secrets_manager_id + instance_id = ibm_resource_instance.secrets_manager.guid # Direct reference region = local.secrets_manager_region labels = ["zvsi-ssh-public-key"] payload = tls_private_key.private_key.public_key_openssh diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/variables.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/variables.tf index e09671f03cd..ea28034f332 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/variables.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/secrets_manager/variables.tf @@ -13,4 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -variable "resource_group_id" {} +variable "resource_group_id" { + description = "The ID of the IBM Cloud resource group where resources will be created" + type = string +} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/outputs.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/outputs.tf index 9137e64970c..859847fb749 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/outputs.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/outputs.tf @@ -31,8 +31,8 @@ output "bastion_sg_name" { value = ibm_is_security_group.bastion_sg.name } -output "master_sg_name" { - value = ibm_is_security_group.master_sg.name +output "control_plane_sg_name" { + value = ibm_is_security_group.control_plane_sg.name } output "worker_sg_name" { diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/vpc.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/vpc.tf index 3fb48cbebf9..9cf014b556e 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/vpc.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/modules/vpc/vpc.tf @@ -43,8 +43,8 @@ resource "ibm_is_security_group" "bastion_sg" { resource_group = var.resource_group_id } -resource "ibm_is_security_group" "master_sg" { - name = "k8s-vpc-s390x-master-sg" +resource "ibm_is_security_group" "control_plane_sg" { + name = "k8s-vpc-s390x-control-plane-sg" vpc = ibm_is_vpc.vpc.id resource_group = var.resource_group_id } @@ -91,10 +91,10 @@ resource "ibm_is_security_group_rule" "worker_internal" { remote = ibm_is_subnet.subnet.ipv4_cidr_block } -resource "ibm_is_security_group_rule" "worker_master_all" { +resource "ibm_is_security_group_rule" "worker_control_plane_all" { group = ibm_is_security_group.worker_sg.id direction = "inbound" - remote = ibm_is_security_group.master_sg.id + remote = ibm_is_security_group.control_plane_sg.id } resource "ibm_is_security_group_rule" "bastion_private_inbound" { @@ -110,19 +110,19 @@ resource "ibm_is_security_group_rule" "bastion_private_outbound" { } -resource "ibm_is_security_group_rule" "master_to_worker_kubelet_api" { +resource "ibm_is_security_group_rule" "control_plane_to_worker_kubelet_api" { group = ibm_is_security_group.worker_sg.id direction = "inbound" - remote = ibm_is_security_group.master_sg.id + remote = ibm_is_security_group.control_plane_sg.id tcp { port_min = 10250 port_max = 10250 } } -resource "ibm_is_security_group_rule" "allow_master_to_worker_all" { +resource "ibm_is_security_group_rule" "allow_control_plane_to_worker_all" { group = ibm_is_security_group.worker_sg.id direction = "inbound" - remote = ibm_is_security_group.master_sg.id + remote = ibm_is_security_group.control_plane_sg.id tcp { port_min = 1 port_max = 65535 @@ -142,7 +142,7 @@ resource "ibm_is_security_group_rule" "allow_bastion_ssh_to_worker" { resource "ibm_is_security_group_rule" "allow_vpc_cidr_to_worker" { group = ibm_is_security_group.worker_sg.id direction = "inbound" - remote = "10.243.0.0/24" + remote = ibm_is_subnet.subnet.ipv4_cidr_block tcp { port_min = 1 port_max = 65535 @@ -224,14 +224,14 @@ resource "ibm_is_security_group_rule" "worker_pod_outbound" { } } -resource "ibm_is_security_group_rule" "master_inbound_from_workers" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_inbound_from_workers" { + group = ibm_is_security_group.control_plane_sg.id direction = "inbound" remote = ibm_is_security_group.worker_sg.id } -resource "ibm_is_security_group_rule" "master_inbound_from_bastion_ssh" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_inbound_from_bastion_ssh" { + group = ibm_is_security_group.control_plane_sg.id direction = "inbound" remote = ibm_is_security_group.bastion_sg.id tcp { @@ -240,20 +240,20 @@ resource "ibm_is_security_group_rule" "master_inbound_from_bastion_ssh" { } } -resource "ibm_is_security_group_rule" "master_inbound_from_internal_cidr" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_inbound_from_internal_cidr" { + group = ibm_is_security_group.control_plane_sg.id direction = "inbound" - remote = "10.243.0.0/24" # adjust based on your CIDR + remote = ibm_is_subnet.subnet.ipv4_cidr_block } -resource "ibm_is_security_group_rule" "master_inbound_from_self" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_inbound_from_self" { + group = ibm_is_security_group.control_plane_sg.id direction = "inbound" - remote = ibm_is_security_group.master_sg.id + remote = ibm_is_security_group.control_plane_sg.id } -resource "ibm_is_security_group_rule" "master_inbound_api" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_inbound_api" { + group = ibm_is_security_group.control_plane_sg.id direction = "inbound" remote = "0.0.0.0/0" tcp { @@ -261,8 +261,8 @@ resource "ibm_is_security_group_rule" "master_inbound_api" { port_max = 6443 } } -resource "ibm_is_security_group_rule" "master_outbound_http" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_outbound_http" { + group = ibm_is_security_group.control_plane_sg.id direction = "outbound" tcp { port_min = 80 @@ -271,8 +271,8 @@ resource "ibm_is_security_group_rule" "master_outbound_http" { remote = "0.0.0.0/0" } -resource "ibm_is_security_group_rule" "master_outbound_https" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_outbound_https" { + group = ibm_is_security_group.control_plane_sg.id direction = "outbound" tcp { port_min = 443 @@ -281,8 +281,8 @@ resource "ibm_is_security_group_rule" "master_outbound_https" { remote = "0.0.0.0/0" } -resource "ibm_is_security_group_rule" "master_outbound_dns_tcp" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_outbound_dns_tcp" { + group = ibm_is_security_group.control_plane_sg.id direction = "outbound" tcp { port_min = 53 @@ -291,8 +291,8 @@ resource "ibm_is_security_group_rule" "master_outbound_dns_tcp" { remote = "0.0.0.0/0" } -resource "ibm_is_security_group_rule" "master_outbound_dns_udp" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_outbound_dns_udp" { + group = ibm_is_security_group.control_plane_sg.id direction = "outbound" udp { port_min = 53 @@ -301,8 +301,8 @@ resource "ibm_is_security_group_rule" "master_outbound_dns_udp" { remote = "0.0.0.0/0" } -resource "ibm_is_security_group_rule" "master_outbound_api" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_outbound_api" { + group = ibm_is_security_group.control_plane_sg.id direction = "outbound" tcp { port_min = 6443 @@ -311,19 +311,19 @@ resource "ibm_is_security_group_rule" "master_outbound_api" { remote = "0.0.0.0/0" } -resource "ibm_is_security_group_rule" "master_outbound_to_workers" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_outbound_to_workers" { + group = ibm_is_security_group.control_plane_sg.id direction = "outbound" - remote = ibm_is_security_group.master_sg.id + remote = ibm_is_security_group.control_plane_sg.id } -resource "ibm_is_security_group_rule" "master_outbound_to_all" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_outbound_to_all" { + group = ibm_is_security_group.control_plane_sg.id direction = "outbound" remote = "0.0.0.0/0" } -resource "ibm_is_security_group_rule" "master_pod_inbound" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_pod_inbound" { + group = ibm_is_security_group.control_plane_sg.id direction = "inbound" remote = ibm_is_subnet.subnet.ipv4_cidr_block tcp { @@ -332,8 +332,8 @@ resource "ibm_is_security_group_rule" "master_pod_inbound" { } } -resource "ibm_is_security_group_rule" "master_pod_outbound" { - group = ibm_is_security_group.master_sg.id +resource "ibm_is_security_group_rule" "control_plane_pod_outbound" { + group = ibm_is_security_group.control_plane_sg.id direction = "outbound" remote = ibm_is_subnet.subnet.ipv4_cidr_block tcp { diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/outputs.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/outputs.tf deleted file mode 100644 index 7d7d844746d..00000000000 --- a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/outputs.tf +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2025 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -output "secrets_manager_id" { - value = module.secrets_manager.k8s_secrets_manager_id -} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/providers.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/providers.tf index 6a48d3a19db..22d0abdec56 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/providers.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/providers.tf @@ -30,8 +30,3 @@ provider "ibm" { ibmcloud_api_key = local.key region = local.region } -provider "ibm" { - alias = "vpcs" - ibmcloud_api_key = local.key - region = local.region -} diff --git a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/versions.tf b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/versions.tf index 12b72c5229b..7eef45bc572 100644 --- a/infra/ibmcloud/terraform/k8s-s390x-infra-setup/versions.tf +++ b/infra/ibmcloud/terraform/k8s-s390x-infra-setup/versions.tf @@ -16,22 +16,27 @@ limitations under the License. terraform { backend "s3" { - bucket = "k8s-infra-z-eu-cos-bucket" + bucket = "k8s-z-infra-tf-states" key = "k8s-s390x-infra-setup/terraform.tfstate" region = "eu-geo" skip_region_validation = true - skip_requesting_account_id = true skip_credentials_validation = true - skip_s3_checksum = true endpoints = { s3 = "https://s3.eu.cloud-object-storage.appdomain.cloud" } - secret_key = "" - access_key = "" } required_providers { ibm = { - source = "IBM-Cloud/ibm" + source = "IBM-Cloud/ibm" + version = "~> 1.82.0" + } + time = { + source = "hashicorp/time" + version = "~> 0.13.0" + } + null = { + source = "hashicorp/null" + version = "~> 3.2.0" } } }