tgc-revival: add google_dataproc_batch (#15378)

Author: zli82016

mmv1/products/dataproc/Batch.yaml

@@ -44,6 +44,7 @@ async:
   result:
     resource_inside_response: false
 collection_url_key: 'batches'
+include_in_tgc_next_DO_NOT_USE: true
 custom_code:
   constants: 'templates/terraform/constants/cloud_dataproc_batch.go.tmpl'
   decoder: 'templates/terraform/decoders/cloud_dataproc_batch.go.tmpl'
@@ -357,6 +358,7 @@ properties:
               the conditions are treated as OR conditions: the workload will be terminated when it has been idle for idleTtl or
               when ttl has been exceeded, whichever occurs first.
             default_from_api: true
+            is_missing_in_cai: true
           - name: 'stagingBucket'
             type: String
             description: |
@@ -456,6 +458,7 @@ properties:
         description: |
           The arguments to pass to the driver. Do not include arguments that can be set as batch
           properties, such as --conf, since a collision can occur that causes an incorrect batch submission.
+        is_missing_in_cai: true
         item_type:
           type: String
       - name: 'pythonFileUris'
@@ -496,6 +499,7 @@ properties:
     properties:
       - name: 'args'
         type: Array
+        is_missing_in_cai: true
         description: |
           The arguments to pass to the driver. Do not include arguments that can be set as batch
           properties, such as --conf, since a collision can occur that causes an incorrect batch submission.
@@ -552,6 +556,7 @@ properties:
         description: |
           The arguments to pass to the driver. Do not include arguments that can be set as batch
           properties, such as --conf, since a collision can occur that causes an incorrect batch submission.
+        is_missing_in_cai: true
         item_type:
           type: String
       - name: 'fileUris'
@@ -591,3 +596,4 @@ properties:
         type: KeyValuePairs
         description: |
           Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).
+        is_missing_in_cai: true

mmv1/templates/terraform/decoders/cloud_dataproc_batch.go.tmpl

@@ -23,11 +23,14 @@ if obj1, ok := res["runtimeConfig"]; ok {
 
                // Update properties back to original client set properties
                originalPropertiesCopy := make(map[string]interface{})
-               originalProperties := d.Get("runtime_config.0.properties").(interface{}).(map[string]interface{})
-               for k, v := range originalProperties {
-                   originalPropertiesCopy[k] = v
+               properties := d.Get("runtime_config.0.properties")
+               if properties != nil {
+                    originalProperties := properties.(interface{}).(map[string]interface{})
+                    for k, v := range originalProperties {
+                        originalPropertiesCopy[k] = v
+                    }
+                    rconfig["properties"] = originalPropertiesCopy
                }
-               rconfig["properties"] = originalPropertiesCopy
                return res, nil
            }
         }

mmv1/third_party/tgc_next/test/assert_test_files.go

@@ -227,8 +227,18 @@ func testSingleResource(t *testing.T, testName string, testData ResourceTestData
 	}
 
 	if diff := cmp.Diff(string(roundtripConfigData), string(exportConfigData)); diff != "" {
-		log.Printf("Roundtrip config is different from the export config.\nroundtrip config:\n%s\nexport config:\n%s", string(roundtripConfigData), string(exportConfigData))
-		return fmt.Errorf("test %s got diff (-want +got): %s", testName, diff)
+		tfFileName := fmt.Sprintf("%s_roundtrip", testName)
+		jsonFileName := fmt.Sprintf("%s_reexport", testName)
+
+		reexportAssets, err := tfplan2caiConvert(t, tfFileName, jsonFileName, tfDir, ancestryCache, defaultProject, logger)
+		if err != nil {
+			return fmt.Errorf("error when converting the third round-trip config: %#v", err)
+		}
+
+		if assestsDiff := cmp.Diff(reexportAssets, roundtripAssets); assestsDiff != "" {
+			log.Printf("Roundtrip config is different from the export config.\nroundtrip config:\n%s\nexport config:\n%s", string(roundtripConfigData), string(exportConfigData))
+			return fmt.Errorf("test %s got diff (-want +got): %s", testName, diff)
+		}
 	}
 	log.Printf("Step 2 passes for resource %s. Roundtrip config and export config are identical", testData.ResourceAddress)
 
@@ -377,19 +387,46 @@ func isIgnored(key string, ignoredFields map[string]any) bool {
 // Converts a tfplan to CAI asset, and then converts the CAI asset into HCL
 func getRoundtripConfig(t *testing.T, testName string, tfDir string, ancestryCache map[string]string, defaultProject string, logger *zap.Logger, ignoredAssetFields []string) ([]caiasset.Asset, []byte, error) {
 	fileName := fmt.Sprintf("%s_export", testName)
+	roundtripAssetFile := fmt.Sprintf("%s_roundtrip.json", testName)
+
+	roundtripAssets, err := tfplan2caiConvert(t, fileName, roundtripAssetFile, tfDir, ancestryCache, defaultProject, logger)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var roundtripAssetsCopy []caiasset.Asset
+	// Perform the deep copy in case the assets are transformed in cai2hcl
+	err = DeepCopyMap(roundtripAssets, &roundtripAssetsCopy)
+	if err != nil {
+		fmt.Println("Error during deep copy:", err)
+		return nil, nil, err
+	}
+
+	deleteFieldsFromAssets(roundtripAssetsCopy, ignoredAssetFields)
+	roundtripConfig, err := cai2hcl.Convert(roundtripAssetsCopy, &cai2hcl.Options{
+		ErrorLogger: logger,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return roundtripAssets, roundtripConfig, nil
+}
 
+// Converts tf file to CAI assets
+func tfplan2caiConvert(t *testing.T, tfFileName, jsonFileName string, tfDir string, ancestryCache map[string]string, defaultProject string, logger *zap.Logger) ([]caiasset.Asset, error) {
 	// Run terraform init and terraform apply to generate tfplan.json files
-	terraformWorkflow(t, tfDir, fileName)
+	terraformWorkflow(t, tfDir, tfFileName)
 
-	planFile := fmt.Sprintf("%s.tfplan.json", fileName)
+	planFile := fmt.Sprintf("%s.tfplan.json", tfFileName)
 	planfilePath := filepath.Join(tfDir, planFile)
 	jsonPlan, err := os.ReadFile(planfilePath)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	ctx := context.Background()
-	roundtripAssets, err := tfplan2cai.Convert(ctx, jsonPlan, &tfplan2cai.Options{
+	assets, err := tfplan2cai.Convert(ctx, jsonPlan, &tfplan2cai.Options{
 		ErrorLogger:    logger,
 		Offline:        true,
 		DefaultProject: defaultProject,
@@ -400,24 +437,14 @@ func getRoundtripConfig(t *testing.T, testName string, tfDir string, ancestryCac
 	})
 
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	deleteFieldsFromAssets(roundtripAssets, ignoredAssetFields)
-
 	if os.Getenv("WRITE_FILES") != "" {
-		roundtripAssetFile := fmt.Sprintf("%s_roundtrip.json", testName)
-		writeJSONFile(roundtripAssetFile, roundtripAssets)
-	}
-
-	roundtripConfig, err := cai2hcl.Convert(roundtripAssets, &cai2hcl.Options{
-		ErrorLogger: logger,
-	})
-	if err != nil {
-		return nil, nil, err
+		writeJSONFile(jsonFileName, assets)
 	}
 
-	return roundtripAssets, roundtripConfig, nil
+	return assets, nil
 }
 
 // Example:

mmv1/third_party/tgc_next/test/utils.go

@@ -102,3 +102,18 @@ func run(t *testing.T, cmd *exec.Cmd, wantError bool) ([]byte, []byte) {
 	}
 	return stdout.Bytes(), stderr.Bytes()
 }
+
+// Creates a deep copy of a source map using JSON marshalling and unmarshalling.
+func DeepCopyMap(source interface{}, destination interface{}) error {
+	marshalled, err := json.Marshal(source)
+	if err != nil {
+		return fmt.Errorf("failed to marshal source map: %w", err)
+	}
+
+	err = json.Unmarshal(marshalled, destination)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal JSON into destination map: %w", err)
+	}
+
+	return nil
+}