Only Remove internal keys from objects, not templates / job_template (#1983)

* Fix logic to only cut internal keys from objects, not templates

* Add changelog-entry

* removed unnecessary prefix check

* Add logic for annotations to be ignored in object and not in templates/jobTemplates

* Add annotation check in config_basic test

* fix test fortmat

Author: BBBmau

.changelog/1983.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+`resource/kubernetes_cron_job_v1`: Fix annotation logic to prevent internalkeys from being removed in templates
+```

kubernetes/resource_kubernetes_cron_job_v1.go

@@ -55,11 +55,15 @@ func resourceKubernetesCronJobV1Create(ctx context.Context, d *schema.ResourceDa
 		return diag.FromErr(err)
 	}
 
+	configAnnotations := d.Get("metadata.0.annotations").(map[string]interface{})
+	ignoreAnnotations := meta.(kubeClientsets).IgnoreAnnotations
+	annotations := removeInternalKeys(metadata.Annotations, make(map[string]interface{}))
+	metadata.Annotations = removeKeys(annotations, configAnnotations, ignoreAnnotations)
+
 	job := batch.CronJob{
 		ObjectMeta: metadata,
 		Spec:       spec,
 	}
-
 	log.Printf("[INFO] Creating new cron job: %#v", job)
 
 	out, err := conn.BatchV1().CronJobs(metadata.Namespace).Create(ctx, &job, metav1.CreateOptions{})
@@ -89,6 +93,10 @@ func resourceKubernetesCronJobV1Update(ctx context.Context, d *schema.ResourceDa
 	if err != nil {
 		return diag.FromErr(err)
 	}
+	configAnnotations := d.Get("metadata.0.annotations").(map[string]interface{})
+	ignoreAnnotations := meta.(kubeClientsets).IgnoreAnnotations
+	annotations := removeInternalKeys(metadata.Annotations, make(map[string]interface{}))
+	metadata.Annotations = removeKeys(annotations, configAnnotations, ignoreAnnotations)
 	spec.JobTemplate.ObjectMeta.Annotations = metadata.Annotations
 
 	cronjob := &batch.CronJob{
@@ -156,7 +164,6 @@ func resourceKubernetesCronJobV1Read(ctx context.Context, d *schema.ResourceData
 			delete(labels, "controller-uid")
 		}
 	}
-
 	err = d.Set("metadata", flattenMetadata(job.ObjectMeta, d, meta))
 	if err != nil {
 		return diag.FromErr(err)

kubernetes/resource_kubernetes_cron_job_v1_test.go

@@ -33,6 +33,7 @@ func TestAccKubernetesCronJobV1_basic(t *testing.T) {
 				Check: resource.ComposeAggregateTestCheckFunc(
 					testAccCheckKubernetesCronJobV1Exists("kubernetes_cron_job_v1.test", &conf1),
 					resource.TestCheckResourceAttr("kubernetes_cron_job_v1.test", "metadata.0.name", name),
+					resource.TestCheckResourceAttr("kubernetes_cron_job_v1.test", "metadata.0.annotations.hashicorp", "terraform"),
 					resource.TestCheckResourceAttrSet("kubernetes_cron_job_v1.test", "metadata.0.generation"),
 					resource.TestCheckResourceAttrSet("kubernetes_cron_job_v1.test", "metadata.0.resource_version"),
 					resource.TestCheckResourceAttrSet("kubernetes_cron_job_v1.test", "metadata.0.uid"),
@@ -46,8 +47,10 @@ func TestAccKubernetesCronJobV1_basic(t *testing.T) {
 					resource.TestCheckResourceAttr("kubernetes_cron_job_v1.test", "spec.0.starting_deadline_seconds", "10"),
 					resource.TestCheckResourceAttr("kubernetes_cron_job_v1.test", "spec.0.successful_jobs_history_limit", "10"),
 					resource.TestCheckResourceAttr("kubernetes_cron_job_v1.test", "spec.0.suspend", "true"),
+					resource.TestCheckResourceAttr("kubernetes_cron_job_v1.test", "spec.0.job_template.0.metadata.0.annotations.cluster-autoscaler.kubernetes.io/safe-to-evict", "false"),
 					resource.TestCheckResourceAttr("kubernetes_cron_job_v1.test", "spec.0.job_template.0.spec.0.parallelism", "1"),
 					resource.TestCheckResourceAttr("kubernetes_cron_job_v1.test", "spec.0.job_template.0.spec.0.backoff_limit", "2"),
+					resource.TestCheckResourceAttr("kubernetes_cron_job_v1.test", "spec.0.job_template.0.spec.0.template.0.metadata.0.annotations.controller.kubernetes.io/pod-deletion-cost", "10000"),
 					resource.TestCheckResourceAttr("kubernetes_cron_job_v1.test", "spec.0.job_template.0.spec.0.template.0.spec.0.container.0.name", "hello"),
 					resource.TestCheckResourceAttr("kubernetes_cron_job_v1.test", "spec.0.job_template.0.spec.0.template.0.spec.0.container.0.image", imageName),
 				),
@@ -186,6 +189,9 @@ func testAccKubernetesCronJobV1Config_basic(name, imageName string) string {
 	return fmt.Sprintf(`resource "kubernetes_cron_job_v1" "test" {
   metadata {
     name = "%s"
+    annotations = {
+      "hashicorp" = "terraform"
+    }
   }
   spec {
     concurrency_policy            = "Replace"
@@ -196,11 +202,19 @@ func testAccKubernetesCronJobV1Config_basic(name, imageName string) string {
     successful_jobs_history_limit = 10
     suspend                       = true
     job_template {
-      metadata {}
+      metadata {
+        annotations = {
+          "cluster-autoscaler.kubernetes.io/safe-to-evict" = "false"
+        }
+      }
       spec {
         backoff_limit = 2
         template {
-          metadata {}
+          metadata {
+            annotations = {
+              "controller.kubernetes.io/pod-deletion-cost" = 10000
+            }
+          }
           spec {
             container {
               name    = "hello"

kubernetes/structure_cron_job_v1.go

@@ -41,7 +41,7 @@ func flattenCronJobSpecV1(in batch.CronJobSpec, d *schema.ResourceData, meta int
 func flattenJobTemplateV1(in batch.JobTemplateSpec, d *schema.ResourceData, meta interface{}) ([]interface{}, error) {
 	att := make(map[string]interface{})
 
-	att["metadata"] = flattenMetadata(in.ObjectMeta, d, meta)
+	att["metadata"] = flattenMetadata(in.ObjectMeta, d, meta, "spec.0.job_template.0.")
 
 	jobSpec, err := flattenJobSpec(in.Spec, d, meta, "spec.0.job_template.0.spec.0.template.0.")
 	if err != nil {

kubernetes/structures.go

@@ -126,10 +126,15 @@ func flattenMetadata(meta metav1.ObjectMeta, d *schema.ResourceData, providerMet
 		prefix = metaPrefix[0]
 	}
 
-	configAnnotations := d.Get(prefix + "metadata.0.annotations").(map[string]interface{})
-	ignoreAnnotations := providerMetadata.(kubeClientsets).IgnoreAnnotations
-	annotations := removeInternalKeys(meta.Annotations, configAnnotations)
-	m["annotations"] = removeKeys(annotations, configAnnotations, ignoreAnnotations)
+	if prefix == "" {
+		configAnnotations := d.Get(prefix + "metadata.0.annotations").(map[string]interface{})
+		ignoreAnnotations := providerMetadata.(kubeClientsets).IgnoreAnnotations
+		annotations := removeInternalKeys(meta.Annotations, configAnnotations)
+		m["annotations"] = removeKeys(annotations, configAnnotations, ignoreAnnotations)
+	} else {
+		m["annotations"] = d.Get(prefix + "metadata.0.annotations").(map[string]interface{})
+	}
+
 	if meta.GenerateName != "" {
 		m["generate_name"] = meta.GenerateName
 	}