From cb5e2d1f73935df274988a138b2e2a20fca4f244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Conall=20=C3=93=20Cofaigh?= Date: Fri, 8 Aug 2025 08:42:57 +0100 Subject: [PATCH 1/9] Add support for OCP 4.18 and make it the default --- ibm_catalog.json | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ibm_catalog.json b/ibm_catalog.json index bec34c6f1..ee4935575 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -1279,11 +1279,8 @@ "key": "kube_version", "type": "string", "required": true, + "default_value": "4.18_openshift", "options": [ - { - "displayname": "Default", - "value": "default" - }, { "displayname": "4.14_openshift", "value": "4.14_openshift" @@ -1299,6 +1296,10 @@ { "displayname": "4.17_openshift", "value": "4.17_openshift" + }, + { + "displayname": "4.18_openshift", + "value": "4.18_openshift" } ], "custom_config": {} @@ -1594,11 +1595,8 @@ "key": "kube_version", "type": "string", "required": true, + "default_value": "4.18_openshift", "options": [ - { - "displayname": "Default", - "value": "default" - }, { "displayname": "4.14_openshift", "value": "4.14_openshift" @@ -1614,6 +1612,10 @@ { "displayname": "4.17_openshift", "value": "4.17_openshift" + }, + { + "displayname": "4.18_openshift", + "value": "4.18_openshift" } ], "custom_config": {} From 0c254c83ab232bc1542eaa3dea080f8ff5c3217d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Conall=20=C3=93=20Cofaigh?= Date: Fri, 8 Aug 2025 08:44:51 +0100 Subject: [PATCH 2/9] Update variables.tf --- patterns/roks/variables.tf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/patterns/roks/variables.tf b/patterns/roks/variables.tf index 4886fe4db..0fe29fc99 100644 --- a/patterns/roks/variables.tf +++ b/patterns/roks/variables.tf @@ -192,19 +192,20 @@ variable "cluster_zones" { } variable "kube_version" { - description = "The version of the OpenShift cluster that should be provisioned. Current supported values are '4.17_openshift', '4.16_openshift', '4.15_openshift' or '4.14_openshift'. NOTE: This is only used during initial cluster provisioning, but ignored for future updates. Cluster version updates should be done outside of terraform to prevent possible destructive changes." + description = "The version of the OpenShift cluster that should be provisioned. NOTE: This is only used during initial cluster provisioning, but ignored for future updates. Cluster version updates should be done outside of terraform to prevent possible destructive changes." type = string - default = "4.16_openshift" + default = null validation { condition = anytrue([ var.kube_version == null, var.kube_version == "default", + var.kube_version == "4.18_openshift", var.kube_version == "4.17_openshift", var.kube_version == "4.16_openshift", var.kube_version == "4.15_openshift", var.kube_version == "4.14_openshift", ]) - error_message = "The kube_version value can currently only be '4.17_openshift', '4.16_openshift', '4.15_openshift' or '4.14_openshift'" + error_message = "The kube_version value can currently only be '4.18_openshift', '4.17_openshift', '4.16_openshift', '4.15_openshift' or '4.14_openshift'" } } From c9da28f190e4045d8eda1f58262744d9a8c266c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Conall=20=C3=93=20Cofaigh?= Date: Fri, 8 Aug 2025 08:48:06 +0100 Subject: [PATCH 3/9] Update cluster.tf --- cluster.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.tf b/cluster.tf index 9843ca260..8485a2016 100644 --- a/cluster.tf +++ b/cluster.tf @@ -244,7 +244,7 @@ module "cluster" { if cluster.kube_type == "openshift" } source = "terraform-ibm-modules/base-ocp-vpc/ibm" - version = "3.48.3" + version = "3.55.0" resource_group_id = local.resource_groups[each.value.resource_group] region = var.region cluster_name = each.value.cluster_name From 33812f8b4f0ad584d364db7b129bc373b2f3843c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Conall=20=C3=93=20Cofaigh?= Date: Fri, 8 Aug 2025 08:48:55 +0100 Subject: [PATCH 4/9] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6e346b814..d2d724739 100644 --- a/README.md +++ b/README.md @@ -852,7 +852,7 @@ module "cluster_pattern" { | Name | Source | Version | |------|--------|---------| | [bastion\_host](#module\_bastion\_host) | terraform-ibm-modules/landing-zone-vsi/ibm | 5.1.23 | -| [cluster](#module\_cluster) | terraform-ibm-modules/base-ocp-vpc/ibm | 3.48.3 | +| [cluster](#module\_cluster) | terraform-ibm-modules/base-ocp-vpc/ibm | 3.55.0 | | [dynamic\_values](#module\_dynamic\_values) | ./dynamic_values | n/a | | [f5\_vsi](#module\_f5\_vsi) | terraform-ibm-modules/landing-zone-vsi/ibm | 5.1.23 | | [key\_management](#module\_key\_management) | ./kms | n/a | From 1baed4da900be418877537e36251044a7f1d1a98 Mon Sep 17 00:00:00 2001 From: Todd Giguere Date: Fri, 8 Aug 2025 13:47:42 -0400 Subject: [PATCH 5/9] fix: align ocp addon map with new format --- cluster.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.tf b/cluster.tf index 8485a2016..2206640e0 100644 --- a/cluster.tf +++ b/cluster.tf @@ -297,7 +297,7 @@ module "cluster" { disable_public_endpoint = coalesce(each.value.disable_public_endpoint, true) # disable if not set or null verify_worker_network_readiness = each.value.verify_cluster_network_readiness use_private_endpoint = each.value.use_ibm_cloud_private_api_endpoints - addons = each.value.addons + addons = { for addon_name, addon_version in each.value.addons : addon_name => { version = addon_version } } enable_ocp_console = each.value.enable_ocp_console manage_all_addons = each.value.manage_all_addons disable_outbound_traffic_protection = each.value.disable_outbound_traffic_protection From 571a32f080157f07fa99b2e4ed4e63890c7f0398 Mon Sep 17 00:00:00 2001 From: Todd Giguere Date: Fri, 8 Aug 2025 16:36:19 -0400 Subject: [PATCH 6/9] fix: default roks os to rhcos --- dynamic_values/cluster.tf | 2 +- dynamic_values/cluster_worker_pools.tf | 2 +- .../cluster_worker_pools/worker_pools.tf | 2 +- examples/override-example/override.json | 10 +++++----- patterns/mixed/config.tf | 4 ++-- patterns/mixed/override.json | 5 ++--- patterns/roks-quickstart/main.tf | 2 +- patterns/roks/module/variables.tf | 6 +++--- patterns/roks/override.json | 8 ++++---- patterns/roks/variables.tf | 6 +++--- tests/resources/override-example.json | 4 ++-- variables.tf | 4 ++-- 12 files changed, 27 insertions(+), 28 deletions(-) diff --git a/dynamic_values/cluster.tf b/dynamic_values/cluster.tf index 7e2fac70e..73e89883c 100644 --- a/dynamic_values/cluster.tf +++ b/dynamic_values/cluster.tf @@ -37,7 +37,7 @@ module "ut_cluster_map" { workers_per_subnet = 2 flavor = "spicy" secondary_storage = "300gb.5iops-tier" - operating_system = "REDHAT_8_64" + operating_system = "RHCOS" } ] } diff --git a/dynamic_values/cluster_worker_pools.tf b/dynamic_values/cluster_worker_pools.tf index c9483343c..a4d69daf2 100644 --- a/dynamic_values/cluster_worker_pools.tf +++ b/dynamic_values/cluster_worker_pools.tf @@ -36,7 +36,7 @@ module "ut_worker_pools" { workers_per_subnet = 2 flavor = "spicy" secondary_storage = "300gb.5iops-tier" - operating_system = "REDHAT_8_64" + operating_system = "RHCOS" } ] } diff --git a/dynamic_values/config_modules/cluster_worker_pools/worker_pools.tf b/dynamic_values/config_modules/cluster_worker_pools/worker_pools.tf index 493490e64..50d0d19bd 100644 --- a/dynamic_values/config_modules/cluster_worker_pools/worker_pools.tf +++ b/dynamic_values/config_modules/cluster_worker_pools/worker_pools.tf @@ -86,7 +86,7 @@ locals { vpc_id = var.vpc_modules[pool.vpc_name].vpc_id # add vpc_id subnets = module.worker_pool_subnets["${var.prefix}-${cluster.name}-${pool.name}"].subnets kube_type = cluster.kube_type - operating_system = lookup(pool, "operating_system", "REDHAT_8_64") + operating_system = lookup(pool, "operating_system", "RHCOS") }) if pool != null ] if cluster.worker_pools != null ] diff --git a/examples/override-example/override.json b/examples/override-example/override.json index a7ff94f8a..31601c9b1 100644 --- a/examples/override-example/override.json +++ b/examples/override-example/override.json @@ -18,7 +18,7 @@ "name": "workload-cluster", "secondary_storage": "300gb.5iops-tier", "resource_group": "slz-workload-rg", - "operating_system": "REDHAT_8_64", + "operating_system": "RHCOS", "use_ibm_cloud_private_api_endpoints": false, "verify_cluster_network_readiness": false, "kms_config": { @@ -37,7 +37,7 @@ "flavor": "bx2.16x64", "name": "logging-worker-pool", "secondary_storage": "300gb.5iops-tier", - "operating_system" : "REDHAT_8_64", + "operating_system": "RHCOS", "subnet_names": [ "vsi-zone-1", "vsi-zone-2", @@ -115,13 +115,13 @@ "enable": true, "days": 30, "prefix": "logs/" - }, - "archive_rule": { + }, + "archive_rule": { "rule_id": "a-bucket-arch-rule", "enable": true, "days": 0, "type": "Glacier" - } + } } ], "keys": [ diff --git a/patterns/mixed/config.tf b/patterns/mixed/config.tf index 0ae720aea..e3370a121 100644 --- a/patterns/mixed/config.tf +++ b/patterns/mixed/config.tf @@ -110,7 +110,7 @@ locals { entitlement = var.entitlement secondary_storage = var.secondary_storage use_private_endpoint = var.use_private_endpoint - operating_system = "REDHAT_8_64" + operating_system = "RHCOS" verify_worker_network_readiness = var.verify_worker_network_readiness boot_volume_crk_name = "${var.prefix}-roks-key" # By default, create dedicated pool for logging @@ -123,7 +123,7 @@ locals { "vsi-zone-${zone}" ] entitlement = var.entitlement - operating_system = "REDHAT_8_64" + operating_system = "RHCOS" workers_per_subnet = var.workers_per_zone flavor = var.flavor secondary_storage = var.secondary_storage diff --git a/patterns/mixed/override.json b/patterns/mixed/override.json index 3c5a3f21c..3f1b83fba 100644 --- a/patterns/mixed/override.json +++ b/patterns/mixed/override.json @@ -29,7 +29,7 @@ "entitlement": "cloud_pak", "flavor": "bx2.16x64", "name": "logging-worker-pool", - "operating_system": "REDHAT_8_64", + "operating_system": "RHCOS", "subnet_names": [ "vsi-zone-1", "vsi-zone-2", @@ -169,7 +169,6 @@ "resource_group": "slz-service-rg", "use_hs_crypto": false, "service_endpoints": "public-and-private" - }, "resource_groups": [ { @@ -187,7 +186,7 @@ ], "security_groups": [], "service_endpoints": "public-and-private", - "existing_vpc_cbr_zone_id" : null, + "existing_vpc_cbr_zone_id": null, "ssh_keys": [ { "name": "slz-ssh-key", diff --git a/patterns/roks-quickstart/main.tf b/patterns/roks-quickstart/main.tf index 96b99c65e..2082cb52b 100644 --- a/patterns/roks-quickstart/main.tf +++ b/patterns/roks-quickstart/main.tf @@ -34,7 +34,7 @@ locals { "resource_group": "workload-rg", "disable_outbound_traffic_protection": true, "cluster_force_delete_storage": true, - "operating_system": "REDHAT_8_64", + "operating_system": "RHCOS", "kms_wait_for_apply": true, "kms_config": { "crk_name": "roks-key", diff --git a/patterns/roks/module/variables.tf b/patterns/roks/module/variables.tf index edb7a5a34..c891389e4 100644 --- a/patterns/roks/module/variables.tf +++ b/patterns/roks/module/variables.tf @@ -272,10 +272,10 @@ variable "cluster_force_delete_storage" { variable "operating_system" { type = string description = "The operating system of the workers in the default worker pool. If no value is specified, the current default version OS will be used. See https://cloud.ibm.com/docs/openshift?topic=openshift-openshift_versions#openshift_versions_available ." - default = "REDHAT_8_64" + default = "RHCOS" validation { - error_message = "RHEL 8 (REDHAT_8_64) or Red Hat Enterprise Linux CoreOS (RHCOS) are the allowed OS values. RHCOS requires VPC clusters created from 4.15 onwards. Upgraded clusters from 4.14 cannot use RHCOS." - condition = var.operating_system == "REDHAT_8_64" || var.operating_system == "RHCOS" + error_message = "RHEL 8 (REDHAT_8_64), RHEL 9 (REDHAT_9_64) or Red Hat Enterprise Linux CoreOS (RHCOS) are the allowed OS values. RHCOS requires VPC clusters created from 4.15 onwards. Upgraded clusters from 4.14 cannot use RHCOS." + condition = var.operating_system == "REDHAT_8_64" || var.operating_system == "REDHAT_9_64" || var.operating_system == "RHCOS" } } diff --git a/patterns/roks/override.json b/patterns/roks/override.json index 7dc529102..858c146aa 100644 --- a/patterns/roks/override.json +++ b/patterns/roks/override.json @@ -16,7 +16,7 @@ "resource_group": "slz-management-rg", "disable_outbound_traffic_protection": false, "cluster_force_delete_storage": false, - "operating_system": "REDHAT_8_64", + "operating_system": "RHCOS", "kms_wait_for_apply": true, "kms_config": { "crk_name": "slz-roks-key", @@ -39,7 +39,7 @@ "vsi-zone-3" ], "vpc_name": "management", - "operating_system": "REDHAT_8_64", + "operating_system": "RHCOS", "workers_per_subnet": 2 } ], @@ -53,7 +53,7 @@ "machine_type": "bx2.16x64", "name": "workload-cluster", "resource_group": "slz-workload-rg", - "operating_system": "REDHAT_8_64", + "operating_system": "RHCOS", "disable_outbound_traffic_protection": false, "cluster_force_delete_storage": false, "kms_wait_for_apply": true, @@ -78,7 +78,7 @@ "vsi-zone-3" ], "vpc_name": "workload", - "operating_system": "REDHAT_8_64", + "operating_system": "RHCOS", "workers_per_subnet": 2 } ], diff --git a/patterns/roks/variables.tf b/patterns/roks/variables.tf index 0fe29fc99..ee0f7b9c9 100644 --- a/patterns/roks/variables.tf +++ b/patterns/roks/variables.tf @@ -290,10 +290,10 @@ variable "cluster_force_delete_storage" { variable "operating_system" { type = string description = "The operating system of the workers in the default worker pool. If no value is specified, the current default version OS will be used. See https://cloud.ibm.com/docs/openshift?topic=openshift-openshift_versions#openshift_versions_available ." - default = "REDHAT_8_64" + default = "RHCOS" validation { - error_message = "RHEL 8 (REDHAT_8_64) or Red Hat Enterprise Linux CoreOS (RHCOS) are the allowed OS values. RHCOS requires VPC clusters created from 4.15 onwards. Upgraded clusters from 4.14 cannot use RHCOS." - condition = var.operating_system == "REDHAT_8_64" || var.operating_system == "RHCOS" + error_message = "RHEL 8 (REDHAT_8_64), RHEL 9 (REDHAT_9_64) or Red Hat Enterprise Linux CoreOS (RHCOS) are the allowed OS values. RHCOS requires VPC clusters created from 4.15 onwards. Upgraded clusters from 4.14 cannot use RHCOS." + condition = var.operating_system == "REDHAT_8_64" || var.operating_system == "REDHAT_9_64" || var.operating_system == "RHCOS" } } diff --git a/tests/resources/override-example.json b/tests/resources/override-example.json index bdb181a0b..e880f0b46 100644 --- a/tests/resources/override-example.json +++ b/tests/resources/override-example.json @@ -18,7 +18,7 @@ "name": "workload-cluster", "secondary_storage": "300gb.5iops-tier", "resource_group": "slz-work-rg", - "operating_system": "REDHAT_8_64", + "operating_system": "RHCOS", "use_ibm_cloud_private_api_endpoints": false, "verify_cluster_network_readiness": false, "kms_config": { @@ -37,7 +37,7 @@ "flavor": "bx2.16x64", "name": "logging-worker-pool", "secondary_storage": "300gb.5iops-tier", - "operating_system": "REDHAT_8_64", + "operating_system": "RHCOS", "subnet_names": [ "vsi-zone-1", "vsi-zone-2", diff --git a/variables.tf b/variables.tf index a2695549d..947407103 100644 --- a/variables.tf +++ b/variables.tf @@ -1007,8 +1007,8 @@ variable "clusters" { # operating_system validation validation { - error_message = "RHEL 8 (REDHAT_8_64) or Red Hat Enterprise Linux CoreOS (RHCOS) are the allowed OS values. RHCOS requires VPC clusters created from 4.15 onwards. Upgraded clusters from 4.14 cannot use RHCOS." - condition = length([for cluster in var.clusters : true if cluster.operating_system == null || cluster.operating_system == "REDHAT_8_64" || cluster.operating_system == "RHCOS"]) == length(var.clusters) + error_message = "RHEL 8 (REDHAT_8_64), RHEL 9 (REDHAT_9_64) or Red Hat Enterprise Linux CoreOS (RHCOS) are the allowed OS values. RHCOS requires VPC clusters created from 4.15 onwards. Upgraded clusters from 4.14 cannot use RHCOS." + condition = length([for cluster in var.clusters : true if cluster.operating_system == null || cluster.operating_system == "REDHAT_8_64" || cluster.operating_system == "REDHAT_9_64" || cluster.operating_system == "RHCOS"]) == length(var.clusters) } } From e7563964aa59d837f0968fcbc683c386e1700389 Mon Sep 17 00:00:00 2001 From: Todd Giguere Date: Tue, 12 Aug 2025 09:16:25 -0400 Subject: [PATCH 7/9] fix: cluster addons refactor to not include null version --- cluster.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.tf b/cluster.tf index 2206640e0..8343027f9 100644 --- a/cluster.tf +++ b/cluster.tf @@ -297,7 +297,7 @@ module "cluster" { disable_public_endpoint = coalesce(each.value.disable_public_endpoint, true) # disable if not set or null verify_worker_network_readiness = each.value.verify_cluster_network_readiness use_private_endpoint = each.value.use_ibm_cloud_private_api_endpoints - addons = { for addon_name, addon_version in each.value.addons : addon_name => { version = addon_version } } + addons = { for addon_name, addon_version in each.value.addons : addon_name => { version = addon_version } if addon_version != null } enable_ocp_console = each.value.enable_ocp_console manage_all_addons = each.value.manage_all_addons disable_outbound_traffic_protection = each.value.disable_outbound_traffic_protection From e1f26049b8ae2882a160209ab35bc27998b45468 Mon Sep 17 00:00:00 2001 From: Todd Giguere Date: Tue, 12 Aug 2025 09:22:02 -0400 Subject: [PATCH 8/9] fix: os should be RHEL_9_64 --- patterns/roks/module/variables.tf | 4 ++-- patterns/roks/variables.tf | 4 ++-- variables.tf | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/patterns/roks/module/variables.tf b/patterns/roks/module/variables.tf index c891389e4..87661f965 100644 --- a/patterns/roks/module/variables.tf +++ b/patterns/roks/module/variables.tf @@ -274,8 +274,8 @@ variable "operating_system" { description = "The operating system of the workers in the default worker pool. If no value is specified, the current default version OS will be used. See https://cloud.ibm.com/docs/openshift?topic=openshift-openshift_versions#openshift_versions_available ." default = "RHCOS" validation { - error_message = "RHEL 8 (REDHAT_8_64), RHEL 9 (REDHAT_9_64) or Red Hat Enterprise Linux CoreOS (RHCOS) are the allowed OS values. RHCOS requires VPC clusters created from 4.15 onwards. Upgraded clusters from 4.14 cannot use RHCOS." - condition = var.operating_system == "REDHAT_8_64" || var.operating_system == "REDHAT_9_64" || var.operating_system == "RHCOS" + error_message = "RHEL 8 (REDHAT_8_64), RHEL 9 (RHEL_9_64) or Red Hat Enterprise Linux CoreOS (RHCOS) are the allowed OS values. RHCOS requires VPC clusters created from 4.15 onwards. Upgraded clusters from 4.14 cannot use RHCOS." + condition = var.operating_system == "REDHAT_8_64" || var.operating_system == "RHEL_9_64" || var.operating_system == "RHCOS" } } diff --git a/patterns/roks/variables.tf b/patterns/roks/variables.tf index ee0f7b9c9..55f276683 100644 --- a/patterns/roks/variables.tf +++ b/patterns/roks/variables.tf @@ -292,8 +292,8 @@ variable "operating_system" { description = "The operating system of the workers in the default worker pool. If no value is specified, the current default version OS will be used. See https://cloud.ibm.com/docs/openshift?topic=openshift-openshift_versions#openshift_versions_available ." default = "RHCOS" validation { - error_message = "RHEL 8 (REDHAT_8_64), RHEL 9 (REDHAT_9_64) or Red Hat Enterprise Linux CoreOS (RHCOS) are the allowed OS values. RHCOS requires VPC clusters created from 4.15 onwards. Upgraded clusters from 4.14 cannot use RHCOS." - condition = var.operating_system == "REDHAT_8_64" || var.operating_system == "REDHAT_9_64" || var.operating_system == "RHCOS" + error_message = "RHEL 8 (REDHAT_8_64), RHEL 9 (RHEL_9_64) or Red Hat Enterprise Linux CoreOS (RHCOS) are the allowed OS values. RHCOS requires VPC clusters created from 4.15 onwards. Upgraded clusters from 4.14 cannot use RHCOS." + condition = var.operating_system == "REDHAT_8_64" || var.operating_system == "RHEL_9_64" || var.operating_system == "RHCOS" } } diff --git a/variables.tf b/variables.tf index 947407103..3a97208e1 100644 --- a/variables.tf +++ b/variables.tf @@ -1007,8 +1007,8 @@ variable "clusters" { # operating_system validation validation { - error_message = "RHEL 8 (REDHAT_8_64), RHEL 9 (REDHAT_9_64) or Red Hat Enterprise Linux CoreOS (RHCOS) are the allowed OS values. RHCOS requires VPC clusters created from 4.15 onwards. Upgraded clusters from 4.14 cannot use RHCOS." - condition = length([for cluster in var.clusters : true if cluster.operating_system == null || cluster.operating_system == "REDHAT_8_64" || cluster.operating_system == "REDHAT_9_64" || cluster.operating_system == "RHCOS"]) == length(var.clusters) + error_message = "RHEL 8 (REDHAT_8_64), RHEL 9 (RHEL_9_64) or Red Hat Enterprise Linux CoreOS (RHCOS) are the allowed OS values. RHCOS requires VPC clusters created from 4.15 onwards. Upgraded clusters from 4.14 cannot use RHCOS." + condition = length([for cluster in var.clusters : true if cluster.operating_system == null || cluster.operating_system == "REDHAT_8_64" || cluster.operating_system == "RHEL_9_64" || cluster.operating_system == "RHCOS"]) == length(var.clusters) } } From c2c1d5c63b4d3308f5add62b7bff9da77855c46a Mon Sep 17 00:00:00 2001 From: Todd Giguere Date: Tue, 12 Aug 2025 09:22:27 -0400 Subject: [PATCH 9/9] chore: skipping upgrade tests new ocp default + old default os type will never pass for main branch, noted in pr and skipping upgrade. SKIP UPGRADE TEST