[SPARK-52702] Verify generated CRD yaml with Helm chart crds content

### What changes were proposed in this pull request?

This PR adds a gradle task to compare latest CRD version vs the one used by Helm charts

### Why are the changes needed?

To avoid potential human error when updating spark-operator-api

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

CIs

### Was this patch authored or co-authored using generative AI tooling?

No

Closes #283 from jiangzho/crd_validator.

Authored-by: Zhou JIANG <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>

Author: jiangzho

spark-operator-api/build.gradle

@@ -57,3 +57,43 @@ tasks.register('relocateGeneratedCRD', Copy) {
   into "../build-tools/helm/spark-kubernetes-operator/crds"
   rename '(.+).yml', '$1.yaml'
 }
+
+tasks.register("assertGeneratedCRDMatchesHelmChart") {
+  dependsOn 'finalizeGeneratedCRD'
+  description = 'Asserts that the generated CRD yaml matches the staged version in Helm Chart'
+  doLast {
+    def currentPath = projectDir.absolutePath
+    def generatedCRDFileBase = "$currentPath/build/classes/java/main/META-INF/fabric8/"
+    def stagedCRDFileBase = "$currentPath/../build-tools/helm/spark-kubernetes-operator/crds/"
+    def generatedAppCRD = [
+        "yq",
+        "e",
+        ".spec.versions[0]",
+        "${generatedCRDFileBase}sparkapplications.spark.apache.org-v1.yml"
+    ].execute().text.trim()
+    def generatedClusterCRD = [
+        "yq",
+        "e",
+        ".spec.versions[0]",
+        "${generatedCRDFileBase}sparkclusters.spark.apache.org-v1.yml"
+    ].execute().text.trim()
+    def stagedAppCRD = [
+        "yq",
+        "e",
+        ".spec.versions[-1]",
+        "${stagedCRDFileBase}sparkapplications.spark.apache.org-v1.yaml"
+    ].execute().text.trim()
+    def stagedClusterCRD = [
+        "yq",
+        "e",
+        ".spec.versions[-1]",
+        "${stagedCRDFileBase}sparkclusters.spark.apache.org-v1.yaml"
+    ].execute().text.trim()
+    if (generatedAppCRD != stagedAppCRD || generatedClusterCRD != stagedClusterCRD) {
+      throw new GradleException("Generated CRD yaml does not match the staged version in " +
+          "Helm Chart, please keep the chart updated.")
+    }
+  }
+}
+
+test.finalizedBy('assertGeneratedCRDMatchesHelmChart')