diff --git a/README.md b/README.md index a741bc30..e4dcf821 100644 --- a/README.md +++ b/README.md @@ -118,11 +118,12 @@ module "ocp_base" { } ``` -### Customizing default cloud service endpoints. +### Customizing default cloud service endpoints The user must export the endpoint as an environment variable in order to use custom cloud service endpoints with this module. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/guides/custom-service-endpoints#getting-started-with-custom-service-endpoints). **Important** The only supported method for customizing cloud service endpoints is to export the environment variables endpoint; be sure to export the value for `IBMCLOUD_IAM_API_ENDPOINT`, `IBMCLOUD_CS_API_ENDPOINT` and `IBMCLOUD_IS_NG_API_ENDPOINT`. For example, + ``` export IBMCLOUD_IAM_API_ENDPOINT="" export IBMCLOUD_CS_API_ENDPOINT="" @@ -140,6 +141,7 @@ There is a provision to toggle outbound traffic by using the modules' `disable_o **Changes Requiring Re-creation of Default Worker Pool** If you need to make changes to the default worker pool that require its re-creation (e.g., changing the worker node `operating_system`), you need to follow 3 steps: + 1. you must set the `allow_default_worker_pool_replacement` variable to `true`, perform the apply. 2. Once the first apply is successful, then make the required change to the default worker pool object, perform the apply. 3. After successful apply of the default worker pool change set `allow_default_worker_pool_replacement` back to `false` in the code before the subsequent apply. @@ -183,6 +185,46 @@ In all cases, note that: - **Important**: If additional load balancers are added after creating the cluster, for example, by exposing a Kubernetes service of type LoadBalancer, update the `number_of_lbs` variable and re-run this module to attach the extra security groups to the newly created load balancer. - The default IBM-managed security group is attached to the LBs in all cases. +### OpenShift Version Upgrade + +Consumers who want to deploy an OpenShift cluster through this module and later manage **master** version upgrades via Terraform must set the variable `enable_openshift_version_upgrade` to `true`. Master upgrade typically require manual checks, and potential updates to the workload, therefore this option is set to `false` by default. This is an advanced capability that we recommend to set to `true` only if you have a robust process to handle master upgrades before updating the version via Terraform. + +Existing users: this capability was introduced in v3.64 of the module. Existing users with a cluster created on previous version of the module can also enable this variable to manage version upgrades through Terraform. However, when `enable_openshift_version_upgrade` is set to `true`, Terraform may plan to destroy and re-create the cluster because the resource type in the module changes. To prevent this, you **must** migrate the existing state to the new resource address before applying any changes - `ibm_container_vpc_cluster.cluster[0]` to `ibm_container_vpc_cluster.cluster_with_upgrade[0]` or, when using auto-scaling, `ibm_container_vpc_cluster.autoscaling_cluster[0]` to `ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0]`. This is a one time migration of the state. + +There are several options to do this: + +#### Use terraform moved blocks (recommended) + +The recommended approach is to use Terraform's native `moved` block feature: + +```hcl +# Example assuming your OCP module is called "ocp" +moved { + from = module.ocp.ibm_container_vpc_cluster.cluster[0] + to = module.ocp.ibm_container_vpc_cluster.cluster_with_upgrade[0] +} +``` + +For auto-scaling clusters: +```hcl +moved { + from = module.ocp.ibm_container_vpc_cluster.autoscaling_cluster[0] + to = module.ocp.ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0] +} +``` + +Add this block to your Terraform configuration, run `terraform plan` to verify the state migration, then `terraform apply` to apply the changes. This approach works for both local Terraform and Schematics deployments. + +After a successful migration and once all team members have applied the changes, you can safely remove the moved blocks from your configuration. + +#### Alternative: manual state migration + +If you prefer not to use moved blocks, you can manually use the terraform state mv command to migrate resources: + +- For local Terraform deployments, see the [terraform state mv documentation](https://developer.hashicorp.com/terraform/cli/commands/state/mv) +- For Schematics deployments, see the [ibmcloud schematics workspace state mv documentation](https://cloud.ibm.com/docs/schematics?topic=schematics-schematics-cli-reference#schematics-wks_statemv) + + ### Troubleshooting #### New kube_version message @@ -262,7 +304,9 @@ Optionally, you need the following permissions to attach Access Management tags | [ibm_container_api_key_reset.reset_api_key](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_api_key_reset) | resource | | [ibm_container_ingress_instance.instance](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_ingress_instance) | resource | | [ibm_container_vpc_cluster.autoscaling_cluster](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_vpc_cluster) | resource | +| [ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_vpc_cluster) | resource | | [ibm_container_vpc_cluster.cluster](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_vpc_cluster) | resource | +| [ibm_container_vpc_cluster.cluster_with_upgrade](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_vpc_cluster) | resource | | [ibm_container_vpc_worker_pool.autoscaling_pool](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_vpc_worker_pool) | resource | | [ibm_container_vpc_worker_pool.pool](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_vpc_worker_pool) | resource | | [ibm_iam_authorization_policy.ocp_secrets_manager_iam_auth_policy](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/iam_authorization_policy) | resource | @@ -302,6 +346,7 @@ Optionally, you need the following permissions to attach Access Management tags | [disable\_outbound\_traffic\_protection](#input\_disable\_outbound\_traffic\_protection) | Whether to allow public outbound access from the cluster workers. This is only applicable for OCP 4.15 and later. | `bool` | `false` | no | | [disable\_public\_endpoint](#input\_disable\_public\_endpoint) | Whether access to the public service endpoint is disabled when the cluster is created. Does not affect existing clusters. You can't disable a public endpoint on an existing cluster, so you can't convert a public cluster to a private cluster. To change a public endpoint to private, create another cluster with this input set to `true`. | `bool` | `false` | no | | [enable\_ocp\_console](#input\_enable\_ocp\_console) | Flag to specify whether to enable or disable the OpenShift console. If set to `null` the module does not modify the current setting on the cluster. Keep in mind that when this input is set to `true` or `false` on a cluster with private only endpoint enabled, the runtime must be able to access the private endpoint. | `bool` | `null` | no | +| [enable\_openshift\_version\_upgrade](#input\_enable\_openshift\_version\_upgrade) | When set to true, allows Terraform to manage major OpenShift version upgrades. This is intended for advanced users who manually control major version upgrades. Defaults to false to avoid unintended drift from IBM-managed patch updates. NOTE: Enabling this on existing clusters requires a one-time terraform state migration. See [README](https://github.com/terraform-ibm-modules/terraform-ibm-base-ocp-vpc/blob/main/README.md#openshift-version-upgrade) for details. | `bool` | `false` | no | | [enable\_registry\_storage](#input\_enable\_registry\_storage) | Set to `true` to enable IBM Cloud Object Storage for the Red Hat OpenShift internal image registry. Set to `false` only for new cluster deployments in an account that is allowlisted for this feature. | `bool` | `true` | no | | [enable\_secrets\_manager\_integration](#input\_enable\_secrets\_manager\_integration) | Integrate with IBM Cloud Secrets Manager so you can centrally manage Ingress subdomain certificates and other secrets. [Learn more](https://cloud.ibm.com/docs/containers?topic=containers-secrets-mgr) | `bool` | `false` | no | | [existing\_cos\_id](#input\_existing\_cos\_id) | The COS id of an already existing COS instance to use for OpenShift internal registry storage. Only required if 'enable\_registry\_storage' and 'use\_existing\_cos' are true. | `string` | `null` | no | diff --git a/common-dev-assets b/common-dev-assets index 8a8a5f03..3c754bf7 160000 --- a/common-dev-assets +++ b/common-dev-assets @@ -1 +1 @@ -Subproject commit 8a8a5f032d7812c65bc0825402e87feca8088af1 +Subproject commit 3c754bf732e99f95e77014077d2d7ed4b29c55b0 diff --git a/examples/advanced/main.tf b/examples/advanced/main.tf index 12214878..53bb9115 100644 --- a/examples/advanced/main.tf +++ b/examples/advanced/main.tf @@ -155,19 +155,20 @@ locals { } module "ocp_base" { - source = "../.." - cluster_name = var.prefix - resource_group_id = module.resource_group.resource_group_id - region = var.region - force_delete_storage = true - vpc_id = ibm_is_vpc.vpc.id - vpc_subnets = local.cluster_vpc_subnets - worker_pools = local.worker_pools - ocp_version = var.ocp_version - tags = var.resource_tags - access_tags = var.access_tags - worker_pools_taints = local.worker_pools_taints - ocp_entitlement = var.ocp_entitlement + source = "../.." + cluster_name = var.prefix + resource_group_id = module.resource_group.resource_group_id + region = var.region + force_delete_storage = true + vpc_id = ibm_is_vpc.vpc.id + vpc_subnets = local.cluster_vpc_subnets + worker_pools = local.worker_pools + ocp_version = var.ocp_version + tags = var.resource_tags + access_tags = var.access_tags + worker_pools_taints = local.worker_pools_taints + ocp_entitlement = var.ocp_entitlement + enable_openshift_version_upgrade = var.enable_openshift_version_upgrade # Enable if using worker autoscaling. Stops Terraform managing worker count. ignore_worker_pool_size_changes = true addons = { diff --git a/examples/advanced/variables.tf b/examples/advanced/variables.tf index 6438c5f8..40129d31 100644 --- a/examples/advanced/variables.tf +++ b/examples/advanced/variables.tf @@ -51,3 +51,9 @@ variable "ocp_entitlement" { description = "Value that is applied to the entitlements for OCP cluster provisioning" default = null } + +variable "enable_openshift_version_upgrade" { + type = bool + description = "When set to true, allows Terraform to manage major OpenShift version upgrades. This is intended for advanced users who manually control major version upgrades. Defaults to false to avoid unintended drift from IBM-managed patch updates. NOTE: Enabling this on existing clusters requires a one-time terraform state migration. See [README](https://github.com/terraform-ibm-modules/terraform-ibm-base-ocp-vpc/blob/main/README.md#openshift-version-upgrade) for details." + default = false +} diff --git a/examples/basic/main.tf b/examples/basic/main.tf index dcfd362b..6a25d269 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -63,7 +63,7 @@ locals { pool_name = "default" # ibm_container_vpc_cluster automatically names default pool "default" (See https://github.com/IBM-Cloud/terraform-provider-ibm/issues/2849) machine_type = "bx2.4x16" workers_per_zone = 2 # minimum of 2 is allowed when using single zone - operating_system = "REDHAT_8_64" + operating_system = "RHCOS" } ] } diff --git a/examples/custom_sg/main.tf b/examples/custom_sg/main.tf index cf8ee125..7d741663 100644 --- a/examples/custom_sg/main.tf +++ b/examples/custom_sg/main.tf @@ -104,6 +104,7 @@ module "ocp_base" { vpc_id = ibm_is_vpc.vpc.id vpc_subnets = local.cluster_vpc_subnets ocp_version = var.ocp_version + enable_openshift_version_upgrade = var.enable_openshift_version_upgrade worker_pools = local.worker_pools access_tags = var.access_tags attach_ibm_managed_security_group = true # true is the default diff --git a/examples/custom_sg/variables.tf b/examples/custom_sg/variables.tf index b2ae3e8a..c89ebf53 100644 --- a/examples/custom_sg/variables.tf +++ b/examples/custom_sg/variables.tf @@ -40,6 +40,12 @@ variable "ocp_version" { default = "4.14" } +variable "enable_openshift_version_upgrade" { + type = bool + description = "When set to true, allows Terraform to manage major OpenShift version upgrades. This is intended for advanced users who manually control major version upgrades. Defaults to false to avoid unintended drift from IBM-managed patch updates. NOTE: Enabling this on existing clusters requires a one-time terraform state migration. See README for details." + default = false +} + variable "access_tags" { type = list(string) description = "A list of access tags to apply to the resources created by the module." diff --git a/main.tf b/main.tf index 801d48cb..097e6a97 100644 --- a/main.tf +++ b/main.tf @@ -24,7 +24,19 @@ locals { create_timeout = "3h" update_timeout = "3h" - cluster_id = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].id : ibm_container_vpc_cluster.cluster[0].id + # tflint-ignore: terraform_unused_declarations + cluster_with_upgrade_id = var.ignore_worker_pool_size_changes ? try(ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].id, null) : try(ibm_container_vpc_cluster.cluster_with_upgrade[0].id, null) + # tflint-ignore: terraform_unused_declarations + cluster_without_upgrade_id = var.ignore_worker_pool_size_changes ? try(ibm_container_vpc_cluster.autoscaling_cluster[0].id, null) : try(ibm_container_vpc_cluster.cluster[0].id, null) + + cluster_id = var.enable_openshift_version_upgrade ? local.cluster_with_upgrade_id : local.cluster_without_upgrade_id + + # tflint-ignore: terraform_unused_declarations + cluster_with_upgrade_crn = var.ignore_worker_pool_size_changes ? try(ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].crn, null) : try(ibm_container_vpc_cluster.cluster_with_upgrade[0].crn, null) + # tflint-ignore: terraform_unused_declarations + cluster_without_upgrade_crn = var.ignore_worker_pool_size_changes ? try(ibm_container_vpc_cluster.autoscaling_cluster[0].crn, null) : try(ibm_container_vpc_cluster.cluster[0].crn, null) + + cluster_crn = var.enable_openshift_version_upgrade ? local.cluster_with_upgrade_crn : local.cluster_without_upgrade_crn # security group attached to worker pool # the terraform provider / iks api take a security group id hardcoded to "cluster", so this pseudo-value is injected into the array based on attach_default_cluster_security_group @@ -127,7 +139,7 @@ resource "ibm_resource_tag" "cos_access_tag" { resource "ibm_container_vpc_cluster" "cluster" { depends_on = [time_sleep.wait_for_reset_api_key] - count = var.ignore_worker_pool_size_changes ? 0 : 1 + count = var.enable_openshift_version_upgrade ? 0 : (var.ignore_worker_pool_size_changes ? 0 : 1) name = var.cluster_name vpc_id = var.vpc_id tags = var.tags @@ -195,10 +207,80 @@ resource "ibm_container_vpc_cluster" "cluster" { } } +# copy of the cluster resource above which allows major openshift version upgrade +resource "ibm_container_vpc_cluster" "cluster_with_upgrade" { + depends_on = [time_sleep.wait_for_reset_api_key] + count = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? 0 : 1) : 0 + name = var.cluster_name + vpc_id = var.vpc_id + tags = var.tags + kube_version = local.ocp_version + flavor = local.default_pool.machine_type + entitlement = var.ocp_entitlement + cos_instance_crn = local.cos_instance_crn + worker_count = local.default_pool.workers_per_zone + resource_group_id = var.resource_group_id + wait_till = var.cluster_ready_when + force_delete_storage = var.force_delete_storage + secondary_storage = local.default_pool.secondary_storage + pod_subnet = var.pod_subnet_cidr + service_subnet = var.service_subnet_cidr + operating_system = local.default_pool.operating_system + disable_public_service_endpoint = var.disable_public_endpoint + worker_labels = local.default_pool.labels + disable_outbound_traffic_protection = local.disable_outbound_traffic_protection + crk = local.default_pool.boot_volume_encryption_kms_config == null ? null : local.default_pool.boot_volume_encryption_kms_config.crk + kms_instance_id = local.default_pool.boot_volume_encryption_kms_config == null ? null : local.default_pool.boot_volume_encryption_kms_config.kms_instance_id + kms_account_id = local.default_pool.boot_volume_encryption_kms_config == null ? null : local.default_pool.boot_volume_encryption_kms_config.kms_account_id + + security_groups = local.cluster_security_groups + + # This resource intentionally omits ignore_changes for kube_version to allow major version upgrades + + # default workers are mapped to the subnets that are "private" + dynamic "zones" { + for_each = local.default_pool.subnet_prefix != null ? var.vpc_subnets[local.default_pool.subnet_prefix] : local.default_pool.vpc_subnets + content { + subnet_id = zones.value.id + name = zones.value.zone + } + } + + # Apply taints to the default worker pools i.e private + + dynamic "taints" { + for_each = var.worker_pools_taints == null ? [] : concat(var.worker_pools_taints["all"], var.worker_pools_taints["default"]) + content { + effect = taints.value.effect + key = taints.value.key + value = taints.value.value + } + } + + dynamic "kms_config" { + for_each = var.kms_config != null ? [1] : [] + content { + crk_id = var.kms_config.crk_id + instance_id = var.kms_config.instance_id + private_endpoint = var.kms_config.private_endpoint == null ? true : var.kms_config.private_endpoint + account_id = var.kms_config.account_id + wait_for_apply = var.kms_config.wait_for_apply + } + } + + timeouts { + # Extend create, update and delete timeout to static values. + delete = local.delete_timeout + create = local.create_timeout + update = local.update_timeout + } +} + + # copy of the cluster resource above which ignores changes to the worker pool for use in autoscaling scenarios resource "ibm_container_vpc_cluster" "autoscaling_cluster" { depends_on = [time_sleep.wait_for_reset_api_key] - count = var.ignore_worker_pool_size_changes ? 1 : 0 + count = var.enable_openshift_version_upgrade ? 0 : (var.ignore_worker_pool_size_changes ? 1 : 0) name = var.cluster_name vpc_id = var.vpc_id tags = var.tags @@ -266,13 +348,86 @@ resource "ibm_container_vpc_cluster" "autoscaling_cluster" { } } +# copy of the cluster resource above which allows major openshift version upgrade +resource "ibm_container_vpc_cluster" "autoscaling_cluster_with_upgrade" { + depends_on = [time_sleep.wait_for_reset_api_key] + count = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? 1 : 0) : 0 + name = var.cluster_name + vpc_id = var.vpc_id + tags = var.tags + kube_version = local.ocp_version + flavor = local.default_pool.machine_type + entitlement = var.ocp_entitlement + cos_instance_crn = local.cos_instance_crn + worker_count = local.default_pool.workers_per_zone + resource_group_id = var.resource_group_id + wait_till = var.cluster_ready_when + force_delete_storage = var.force_delete_storage + operating_system = local.default_pool.operating_system + secondary_storage = local.default_pool.secondary_storage + pod_subnet = var.pod_subnet_cidr + service_subnet = var.service_subnet_cidr + disable_public_service_endpoint = var.disable_public_endpoint + worker_labels = local.default_pool.labels + disable_outbound_traffic_protection = local.disable_outbound_traffic_protection + crk = local.default_pool.boot_volume_encryption_kms_config == null ? null : local.default_pool.boot_volume_encryption_kms_config.crk + kms_instance_id = local.default_pool.boot_volume_encryption_kms_config == null ? null : local.default_pool.boot_volume_encryption_kms_config.kms_instance_id + kms_account_id = local.default_pool.boot_volume_encryption_kms_config == null ? null : local.default_pool.boot_volume_encryption_kms_config.kms_account_id + + security_groups = local.cluster_security_groups + + # This resource intentionally omits ignore_changes for kube_version to allow major version upgrades + + lifecycle { + ignore_changes = [worker_count] + } + + # default workers are mapped to the subnets that are "private" + dynamic "zones" { + for_each = local.default_pool.subnet_prefix != null ? var.vpc_subnets[local.default_pool.subnet_prefix] : local.default_pool.vpc_subnets + content { + subnet_id = zones.value.id + name = zones.value.zone + } + } + + # Apply taints to the default worker pools i.e private + + dynamic "taints" { + for_each = var.worker_pools_taints == null ? [] : concat(var.worker_pools_taints["all"], var.worker_pools_taints["default"]) + content { + effect = taints.value.effect + key = taints.value.key + value = taints.value.value + } + } + + dynamic "kms_config" { + for_each = var.kms_config != null ? [1] : [] + content { + crk_id = var.kms_config.crk_id + instance_id = var.kms_config.instance_id + private_endpoint = var.kms_config.private_endpoint + account_id = var.kms_config.account_id + wait_for_apply = var.kms_config.wait_for_apply + } + } + + timeouts { + # Extend create, update and delete timeout to static values. + delete = local.delete_timeout + create = local.create_timeout + update = local.update_timeout + } +} + ############################################################################## # Cluster Access Tag ############################################################################## resource "ibm_resource_tag" "cluster_access_tag" { count = length(var.access_tags) == 0 ? 0 : 1 - resource_id = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].crn : ibm_container_vpc_cluster.cluster[0].crn + resource_id = local.cluster_crn tags = var.access_tags tag_type = "access" } @@ -448,7 +603,7 @@ resource "null_resource" "confirm_network_healthy" { # Worker pool creation can start before the 'ibm_container_vpc_cluster' completes since there is no explicit # depends_on in 'ibm_container_vpc_worker_pool', just an implicit depends_on on the cluster ID. Cluster ID can exist before # 'ibm_container_vpc_cluster' completes, so hence need to add explicit depends on against 'ibm_container_vpc_cluster' here. - depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_cluster.autoscaling_cluster, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool] + depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_cluster.cluster_with_upgrade, ibm_container_vpc_cluster.autoscaling_cluster, ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool] provisioner "local-exec" { command = "${path.module}/scripts/confirm_network_healthy.sh" @@ -502,7 +657,7 @@ resource "ibm_container_addons" "addons" { # Worker pool creation can start before the 'ibm_container_vpc_cluster' completes since there is no explicit # depends_on in 'ibm_container_vpc_worker_pool', just an implicit depends_on on the cluster ID. Cluster ID can exist before # 'ibm_container_vpc_cluster' completes, so hence need to add explicit depends on against 'ibm_container_vpc_cluster' here. - depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_cluster.autoscaling_cluster, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool, null_resource.confirm_network_healthy] + depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_cluster.cluster_with_upgrade, ibm_container_vpc_cluster.autoscaling_cluster, ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool, null_resource.confirm_network_healthy] cluster = local.cluster_id resource_group_id = var.resource_group_id @@ -575,7 +730,7 @@ resource "kubernetes_config_map_v1_data" "set_autoscaling" { ############################################################################## data "ibm_is_lbs" "all_lbs" { - depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool, null_resource.confirm_network_healthy] + depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_cluster.cluster_with_upgrade, ibm_container_vpc_cluster.autoscaling_cluster, ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool, null_resource.confirm_network_healthy] count = length(var.additional_lb_security_group_ids) > 0 ? 1 : 0 } @@ -611,19 +766,19 @@ locals { data "ibm_is_virtual_endpoint_gateway" "master_vpe" { count = length(var.additional_vpe_security_group_ids["master"]) - depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool, null_resource.confirm_network_healthy] + depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_cluster.cluster_with_upgrade, ibm_container_vpc_cluster.autoscaling_cluster, ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool, null_resource.confirm_network_healthy] name = local.vpes_to_attach_to_sg["master"] } data "ibm_is_virtual_endpoint_gateway" "api_vpe" { count = length(var.additional_vpe_security_group_ids["api"]) - depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool, null_resource.confirm_network_healthy] + depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_cluster.cluster_with_upgrade, ibm_container_vpc_cluster.autoscaling_cluster, ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool, null_resource.confirm_network_healthy] name = local.vpes_to_attach_to_sg["api"] } data "ibm_is_virtual_endpoint_gateway" "registry_vpe" { count = length(var.additional_vpe_security_group_ids["registry"]) - depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool, null_resource.confirm_network_healthy] + depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_cluster.cluster_with_upgrade, ibm_container_vpc_cluster.autoscaling_cluster, ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool, null_resource.confirm_network_healthy] name = local.vpes_to_attach_to_sg["registry"] } @@ -689,7 +844,7 @@ module "cbr_rule" { }, { name = "serviceInstance" - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].id : ibm_container_vpc_cluster.cluster[0].id + value = local.cluster_id operator = "stringEquals" }, { @@ -715,7 +870,7 @@ module "existing_secrets_manager_instance_parser" { resource "ibm_iam_authorization_policy" "ocp_secrets_manager_iam_auth_policy" { count = var.enable_secrets_manager_integration && !var.skip_ocp_secrets_manager_iam_auth_policy ? 1 : 0 - depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_cluster.autoscaling_cluster, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool] + depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_cluster.autoscaling_cluster, ibm_container_vpc_cluster.cluster_with_upgrade, ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool] source_service_name = "containers-kubernetes" source_resource_instance_id = local.cluster_id target_service_name = "secrets-manager" diff --git a/outputs.tf b/outputs.tf index 5e8853ec..56c12af4 100644 --- a/outputs.tf +++ b/outputs.tf @@ -4,19 +4,19 @@ output "cluster_id" { description = "ID of the cluster" - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].id : ibm_container_vpc_cluster.cluster[0].id + value = local.cluster_id depends_on = [null_resource.confirm_network_healthy] } output "cluster_name" { description = "Name of the cluster" - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].name : ibm_container_vpc_cluster.cluster[0].name + value = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].name : ibm_container_vpc_cluster.cluster_with_upgrade[0].name) : (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].name : ibm_container_vpc_cluster.cluster[0].name) depends_on = [null_resource.confirm_network_healthy] } output "cluster_crn" { description = "CRN of the cluster" - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].crn : ibm_container_vpc_cluster.cluster[0].crn + value = local.cluster_crn depends_on = [null_resource.confirm_network_healthy] } @@ -27,17 +27,17 @@ output "workerpools" { output "ocp_version" { description = "Openshift Version of the cluster" - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].kube_version : ibm_container_vpc_cluster.cluster[0].kube_version + value = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].kube_version : ibm_container_vpc_cluster.cluster_with_upgrade[0].kube_version) : (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].kube_version : ibm_container_vpc_cluster.cluster[0].kube_version) } output "cos_crn" { description = "CRN of the COS instance" - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].cos_instance_crn : ibm_container_vpc_cluster.cluster[0].cos_instance_crn + value = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].cos_instance_crn : ibm_container_vpc_cluster.cluster_with_upgrade[0].cos_instance_crn) : (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].cos_instance_crn : ibm_container_vpc_cluster.cluster[0].cos_instance_crn) } output "vpc_id" { description = "ID of the clusters VPC" - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].vpc_id : ibm_container_vpc_cluster.cluster[0].vpc_id + value = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].vpc_id : ibm_container_vpc_cluster.cluster_with_upgrade[0].vpc_id) : (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].vpc_id : ibm_container_vpc_cluster.cluster[0].vpc_id) } output "region" { @@ -47,32 +47,32 @@ output "region" { output "resource_group_id" { description = "Resource group ID the cluster is deployed in" - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].resource_group_id : ibm_container_vpc_cluster.cluster[0].resource_group_id + value = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].resource_group_id : ibm_container_vpc_cluster.cluster_with_upgrade[0].resource_group_id) : (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].resource_group_id : ibm_container_vpc_cluster.cluster[0].resource_group_id) } output "ingress_hostname" { description = "The hostname that was assigned to your Ingress subdomain." - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].ingress_hostname : ibm_container_vpc_cluster.cluster[0].ingress_hostname + value = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].ingress_hostname : ibm_container_vpc_cluster.cluster_with_upgrade[0].ingress_hostname) : (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].ingress_hostname : ibm_container_vpc_cluster.cluster[0].ingress_hostname) } output "private_service_endpoint_url" { description = "Private service endpoint URL" - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].private_service_endpoint_url : ibm_container_vpc_cluster.cluster[0].private_service_endpoint_url + value = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].private_service_endpoint_url : ibm_container_vpc_cluster.cluster_with_upgrade[0].private_service_endpoint_url) : (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].private_service_endpoint_url : ibm_container_vpc_cluster.cluster[0].private_service_endpoint_url) } output "public_service_endpoint_url" { description = "Public service endpoint URL" - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].public_service_endpoint_url : ibm_container_vpc_cluster.cluster[0].public_service_endpoint_url + value = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].public_service_endpoint_url : ibm_container_vpc_cluster.cluster_with_upgrade[0].public_service_endpoint_url) : (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].public_service_endpoint_url : ibm_container_vpc_cluster.cluster[0].public_service_endpoint_url) } output "master_url" { description = "The URL of the Kubernetes master." - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].master_url : ibm_container_vpc_cluster.cluster[0].master_url + value = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].master_url : ibm_container_vpc_cluster.cluster_with_upgrade[0].master_url) : (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].master_url : ibm_container_vpc_cluster.cluster[0].master_url) } output "vpe_url" { description = "The virtual private endpoint URL of the Kubernetes cluster." - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].vpe_service_endpoint_url : ibm_container_vpc_cluster.cluster[0].vpe_service_endpoint_url + value = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].vpe_service_endpoint_url : ibm_container_vpc_cluster.cluster_with_upgrade[0].vpe_service_endpoint_url) : (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].vpe_service_endpoint_url : ibm_container_vpc_cluster.cluster[0].vpe_service_endpoint_url) } output "kms_config" { @@ -82,12 +82,12 @@ output "kms_config" { output "operating_system" { description = "The operating system of the workers in the default worker pool." - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].operating_system : ibm_container_vpc_cluster.cluster[0].operating_system + value = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].operating_system : ibm_container_vpc_cluster.cluster_with_upgrade[0].operating_system) : (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].operating_system : ibm_container_vpc_cluster.cluster[0].operating_system) } output "master_status" { description = "The status of the Kubernetes master." - value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].master_status : ibm_container_vpc_cluster.cluster[0].master_status + value = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade[0].master_status : ibm_container_vpc_cluster.cluster_with_upgrade[0].master_status) : (var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].master_status : ibm_container_vpc_cluster.cluster[0].master_status) } output "master_vpe" { diff --git a/tests/pr_test.go b/tests/pr_test.go index d8fa9928..f9d0f8a1 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -220,9 +220,10 @@ func TestRunCustomsgExample(t *testing.T) { }, ImplicitRequired: false, TerraformVars: map[string]interface{}{ - "ocp_version": ocpVersion2, - "access_tags": permanentResources["accessTags"], - "ocp_entitlement": "cloud_pak", + "ocp_version": ocpVersion2, + "access_tags": permanentResources["accessTags"], + "ocp_entitlement": "cloud_pak", + "enable_openshift_version_upgrade": true, }, }) diff --git a/variables.tf b/variables.tf index 19c66cdf..500c25bc 100644 --- a/variables.tf +++ b/variables.tf @@ -201,6 +201,12 @@ variable "ocp_version" { } } +variable "enable_openshift_version_upgrade" { + type = bool + description = "When set to true, allows Terraform to manage major OpenShift version upgrades. This is intended for advanced users who manually control major version upgrades. Defaults to false to avoid unintended drift from IBM-managed patch updates. NOTE: Enabling this on existing clusters requires a one-time terraform state migration. See [README](https://github.com/terraform-ibm-modules/terraform-ibm-base-ocp-vpc/blob/main/README.md#openshift-version-upgrade) for details." + default = false +} + variable "cluster_ready_when" { type = string description = "The cluster is ready based on one of the following:: MasterNodeReady (not recommended), OneWorkerNodeReady, Normal, IngressReady"