run acceptance tests with multiple versions of kubernetes (#2563)

* run acceptance tests with multiple versions of kubernetes

fixes #2562

* restrict tests from running against unsupported versions of kubernetes

* fix replacement check in cron_job_v1beta1 tests

* fix test config dependency reference

---------

Co-authored-by: Mauricio Alvarez Leon <[email protected]>

Author: coryflucas

.github/workflows/acceptance_tests_kind.yaml

@@ -5,7 +5,7 @@ on:
     inputs:
       kindVersion:
         description: The kind version
-        default: 0.20.0 # Kubernetes version: 1.27.X
+        default: 0.20.0
       runTests:
         description: The regex passed to the -run option of `go test`
         default: "^TestAcc"
@@ -34,6 +34,19 @@ jobs:
   acceptance_tests_kind:
     if: ${{ github.repository_owner == 'hashicorp' }}
     runs-on: custom-linux-medium
+    strategy:
+      # Don't cancel all in-progress and queued jobs in the matrix if any job in the matrix fails.
+      # That will be helpful to catch any issues related to a particular Kubernetes version.
+      fail-fast: false
+      matrix:
+        kubernetes_version:
+          # kind images: https://github.com/kubernetes-sigs/kind/releases (note the images are kind release specific)
+          - v1.29.0@sha256:eaa1450915475849a73a9227b8f201df25e55e268e5d619312131292e324d570
+          - v1.28.0@sha256:b7a4cad12c197af3ba43202d3efe03246b3f0793f162afb40a33c923952d5b31
+          - v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
+          - v1.26.6@sha256:6e2d8b28a5b601defe327b98bd1c2d1930b49e5d8c512e1895099e4504007adb
+          - v1.25.11@sha256:227fa11ce74ea76a0474eeefb84cb75d8dad1b08638371ecf0e86259b35be0c8
+          - v1.23.17@sha256:59c989ff8a517a93127d4a536e7014d28e235fb3529d9fba91b3951d461edfdb
     steps:
       - name: Checkout repository
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
@@ -51,6 +64,7 @@ jobs:
         with:
           wait: 2m
           version: v${{ env.KIND_VERSION }}
+          node_image: kindest/node:${{ matrix.kubernetes_version }}
           config: .github/config/acceptance_tests_kind_config.yaml
       - name: Run Acceptance Test Suite
         env:

kubernetes/data_source_kubernetes_service_account_v1_test.go

@@ -165,7 +165,7 @@ resource "kubernetes_secret_v1" "test" {
   }
   type = "kubernetes.io/service-account-token"
   depends_on = [
-    kubernetes_service_account.test
+    kubernetes_service_account_v1.test
   ]
 }
 `, name, name, name)

kubernetes/resource_kubernetes_cron_job_v1_test.go

@@ -177,7 +177,10 @@ func TestAccKubernetesCronJobV1_minimalWithPodFailurePolicy(t *testing.T) {
 	imageName := busyboxImage
 
 	resource.ParallelTest(t, resource.TestCase{
-		PreCheck:          func() { testAccPreCheck(t) },
+		PreCheck: func() {
+			testAccPreCheck(t)
+			skipIfClusterVersionLessThan(t, "1.25.0")
+		},
 		IDRefreshName:     resourceName,
 		IDRefreshIgnore:   []string{"metadata.0.resource_version"},
 		ProviderFactories: testAccProviderFactories,

kubernetes/resource_kubernetes_cron_job_v1beta1_test.go

@@ -355,7 +355,7 @@ func testAccKubernetesCronJobV1Beta1Config_extraModified(name, imageName string)
 func testAccCheckKubernetesCronJobV1Beta1ForceNew(old, new *batchv1beta1.CronJob, wantNew bool) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		if wantNew {
-			if old.ObjectMeta.UID != new.ObjectMeta.UID {
+			if old.ObjectMeta.UID == new.ObjectMeta.UID {
 				return fmt.Errorf("Expecting forced replacement")
 			}
 		} else {

kubernetes/resource_kubernetes_job_v1_test.go

@@ -55,7 +55,10 @@ func TestAccKubernetesJobV1_basic(t *testing.T) {
 	resourceName := "kubernetes_job_v1.test"
 
 	resource.ParallelTest(t, resource.TestCase{
-		PreCheck:          func() { testAccPreCheck(t) },
+		PreCheck: func() {
+			testAccPreCheck(t)
+			skipIfClusterVersionLessThan(t, "1.26.0")
+		},
 		IDRefreshName:     resourceName,
 		IDRefreshIgnore:   []string{"metadata.0.resource_version"},
 		ProviderFactories: testAccProviderFactories,
@@ -122,7 +125,10 @@ func TestAccKubernetesJobV1_update(t *testing.T) {
 	resourceName := "kubernetes_job_v1.test"
 
 	resource.ParallelTest(t, resource.TestCase{
-		PreCheck:          func() { testAccPreCheck(t) },
+		PreCheck: func() {
+			testAccPreCheck(t)
+			skipIfClusterVersionLessThan(t, "1.26.0")
+		},
 		IDRefreshName:     resourceName,
 		IDRefreshIgnore:   []string{"metadata.0.resource_version"},
 		ProviderFactories: testAccProviderFactories,

kubernetes/resource_kubernetes_persistent_volume_v1_test.go

@@ -741,7 +741,10 @@ func TestAccKubernetesPersistentVolumeV1_accessMode_ReadWriteOncePod(t *testing.
 	resourceName := "kubernetes_persistent_volume_v1.test"
 
 	resource.ParallelTest(t, resource.TestCase{
-		PreCheck:          func() { testAccPreCheck(t) },
+		PreCheck: func() {
+			testAccPreCheck(t)
+			skipIfClusterVersionLessThan(t, "1.27.0")
+		},
 		IDRefreshName:     resourceName,
 		IDRefreshIgnore:   []string{"metadata.0.resource_version"},
 		ProviderFactories: testAccProviderFactories,

kubernetes/resource_kubernetes_pod_v1_test.go

@@ -1440,6 +1440,7 @@ func TestAccKubernetesPodV1_topologySpreadConstraint(t *testing.T) {
 	resource.ParallelTest(t, resource.TestCase{
 		PreCheck: func() {
 			testAccPreCheck(t)
+			skipIfClusterVersionLessThan(t, "1.27.0")
 		},
 		ProviderFactories: testAccProviderFactories,
 		CheckDestroy:      testAccCheckKubernetesPodV1Destroy,
@@ -1477,6 +1478,7 @@ func TestAccKubernetesPodV1_topologySpreadConstraintMinDomains(t *testing.T) {
 	resource.ParallelTest(t, resource.TestCase{
 		PreCheck: func() {
 			testAccPreCheck(t)
+			skipIfClusterVersionLessThan(t, "1.27.0")
 		},
 		ProviderFactories: testAccProviderFactories,
 		CheckDestroy:      testAccCheckKubernetesPodV1Destroy,
@@ -1609,7 +1611,10 @@ func TestAccKubernetesPodV1_os(t *testing.T) {
 	imageName := busyboxImage
 
 	resource.ParallelTest(t, resource.TestCase{
-		PreCheck:          func() { testAccPreCheck(t) },
+		PreCheck: func() {
+			testAccPreCheck(t)
+			skipIfClusterVersionLessThan(t, "1.24.0")
+		},
 		ProviderFactories: testAccProviderFactories,
 		CheckDestroy:      testAccCheckKubernetesPodV1Destroy,
 		Steps: []resource.TestStep{

kubernetes/resource_kubernetes_stateful_set_v1_test.go

@@ -49,7 +49,10 @@ func TestAccKubernetesStatefulSetV1_basic(t *testing.T) {
 	imageName := agnhostImage
 
 	resource.ParallelTest(t, resource.TestCase{
-		PreCheck:      func() { testAccPreCheck(t) },
+		PreCheck: func() {
+			testAccPreCheck(t)
+			skipIfClusterVersionLessThan(t, "1.27.0")
+		},
 		IDRefreshName: resourceName,
 		IDRefreshIgnore: []string{
 			"metadata.0.resource_version",
@@ -128,7 +131,10 @@ func TestAccKubernetesStatefulSetV1_basic_idempotency(t *testing.T) {
 	imageName := agnhostImage
 
 	resource.ParallelTest(t, resource.TestCase{
-		PreCheck:      func() { testAccPreCheck(t) },
+		PreCheck: func() {
+			testAccPreCheck(t)
+			skipIfClusterVersionLessThan(t, "1.27.0")
+		},
 		IDRefreshName: resourceName,
 		IDRefreshIgnore: []string{
 			"metadata.0.resource_version",
@@ -163,7 +169,10 @@ func TestAccKubernetesStatefulSetV1_Update(t *testing.T) {
 	imageName := agnhostImage
 
 	resource.ParallelTest(t, resource.TestCase{
-		PreCheck:      func() { testAccPreCheck(t) },
+		PreCheck: func() {
+			testAccPreCheck(t)
+			skipIfClusterVersionLessThan(t, "1.27.0")
+		},
 		IDRefreshName: resourceName,
 		IDRefreshIgnore: []string{
 			"metadata.0.resource_version",