add option yaml files and update manifest file (#3310)

* add option yaml files and update manifest file

* spotless

* address comments

Author: derrickaw

plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesReleaseMojo.java

@@ -45,7 +45,9 @@
 import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.apache.commons.lang3.StringUtils;
@@ -68,8 +70,7 @@ public class TemplatesReleaseMojo extends TemplatesBaseMojo {
 
   private static final Logger LOG = LoggerFactory.getLogger(TemplatesReleaseMojo.class);
 
-  private record Blueprint(String name, String path) {}
-  ;
+  private record ManifestEntry(String name, String path) {}
 
   private static final Gson GSON = new Gson();
 
@@ -193,6 +194,16 @@ private record Blueprint(String name, String path) {}
       required = false)
   protected String yamlManifestName;
 
+  // Options directory for additional parameter files for yaml blueprints.
+  // These files provide more context for jinja variables used in the yaml
+  // blueprints.
+  @Parameter(
+      defaultValue = "yaml/src/main/python/options",
+      property = "yamlOptionsPath",
+      readonly = true,
+      required = false)
+  protected String yamlOptionsPath;
+
   public void execute() throws MojoExecutionException {
 
     if (librariesBucketName == null || librariesBucketName.isEmpty()) {
@@ -279,50 +290,38 @@ public void execute() throws MojoExecutionException {
             "Trying to upload Job Builder blueprints to bucket '{}'...",
             bucketNameOnly(bucketName));
         Path yamlPath = Paths.get(project.getBasedir().getAbsolutePath(), yamlBlueprintsPath);
-        if (!Files.exists(yamlPath) || !Files.isDirectory(yamlPath)) {
-          LOG.warn("YAML blueprints directory not found, skipping upload for path: ", yamlPath);
+        Path yamlOptionsPath =
+            Paths.get(project.getBasedir().getAbsolutePath(), this.yamlOptionsPath);
+
+        if ((!Files.exists(yamlPath) || !Files.isDirectory(yamlPath))
+            && (!Files.exists(yamlOptionsPath) || !Files.isDirectory(yamlOptionsPath))) {
+          LOG.warn(
+              "YAML blueprints {} and/or options directory {} not found, skipping upload for paths.",
+              yamlPath,
+              yamlOptionsPath);
         } else {
 
-          try (Storage storage = StorageOptions.getDefaultInstance().getService();
-              Stream<Path> paths = Files.list(yamlPath)) {
-            List<Blueprint> blueprints = new ArrayList<>();
-            paths
-                .filter(
-                    path ->
-                        Files.isRegularFile(path)
-                            && path.getFileName().toString().endsWith(".yaml"))
-                .forEach(
-                    path -> {
-                      String fileName = path.getFileName().toString();
-                      String objectName =
-                          String.join("/", stagePrefix, yamlBlueprintsGCSPath, fileName);
-                      BlobId blobId = BlobId.of(bucketNameOnly(bucketName), objectName);
-
-                      BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
-
-                      // Upload every blueprint with retries
-                      Failsafe.with(gcsRetryPolicy())
-                          .run(
-                              () -> {
-                                try (InputStream inputStream = Files.newInputStream(path)) {
-                                  storage.create(blobInfo, inputStream);
-                                }
-                              });
-
-                      LOG.info(
-                          "Uploaded blueprint {} to gs://{}/{}",
-                          fileName,
-                          bucketNameOnly(bucketName),
-                          objectName);
-                      blueprints.add(new Blueprint(fileName, objectName));
-                    });
+          try (Storage storage = StorageOptions.getDefaultInstance().getService()) {
+            List<ManifestEntry> blueprints = new ArrayList<>();
+            List<ManifestEntry> options = new ArrayList<>();
+
+            // Upload the main Yaml blueprints
+            uploadArtifacts(storage, yamlPath, "", blueprints);
+
+            // Upload the jinja parameter option files
+            uploadArtifacts(storage, yamlOptionsPath, "options", options);
+
+            // Build the manifest file
             String manifestObjectName =
                 String.join("/", stagePrefix, yamlBlueprintsGCSPath, yamlManifestName);
             BlobId manifestBlobId = BlobId.of(bucketNameOnly(bucketName), manifestObjectName);
             BlobInfo manifestBlobInfo = BlobInfo.newBuilder(manifestBlobId).build();
 
             // Upload the manifest file with retries
-            byte[] manifestBytes = GSON.toJson(blueprints).getBytes(StandardCharsets.UTF_8);
+            Map<String, List<ManifestEntry>> manifestMap = new HashMap<>();
+            manifestMap.put("blueprints", blueprints);
+            manifestMap.put("options", options);
+            byte[] manifestBytes = GSON.toJson(manifestMap).getBytes(StandardCharsets.UTF_8);
             Failsafe.with(gcsRetryPolicy())
                 .run(
                     () ->
@@ -353,6 +352,50 @@ public void execute() throws MojoExecutionException {
     }
   }
 
+  private void uploadArtifacts(
+      Storage storage, Path directory, String subFolder, List<ManifestEntry> entries)
+      throws IOException {
+    if (Files.exists(directory) && Files.isDirectory(directory)) {
+      try (Stream<Path> paths = Files.list(directory)) {
+        paths
+            .filter(
+                path ->
+                    Files.isRegularFile(path) && path.getFileName().toString().endsWith(".yaml"))
+            .forEach(
+                path -> {
+                  String fileName = path.getFileName().toString();
+                  String objectName =
+                      subFolder.isEmpty()
+                          ? String.join("/", stagePrefix, yamlBlueprintsGCSPath, fileName)
+                          : String.join(
+                              "/", stagePrefix, yamlBlueprintsGCSPath, subFolder, fileName);
+                  uploadToGcs(storage, path, objectName);
+                  entries.add(new ManifestEntry(fileName, objectName));
+                });
+      }
+    }
+  }
+
+  private void uploadToGcs(Storage storage, Path path, String objectName) {
+    BlobId blobId = BlobId.of(bucketNameOnly(bucketName), objectName);
+    BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
+
+    // Upload every blueprint with retries
+    Failsafe.with(gcsRetryPolicy())
+        .run(
+            () -> {
+              try (InputStream inputStream = Files.newInputStream(path)) {
+                storage.create(blobInfo, inputStream);
+              }
+            });
+
+    LOG.info(
+        "Uploaded file {} to gs://{}/{}",
+        path.getFileName().toString(),
+        bucketNameOnly(bucketName),
+        objectName);
+  }
+
   private static <T> RetryPolicy<T> gcsRetryPolicy() {
     return RetryPolicy.<T>builder()
         .handle(IOException.class)