Merge pull request #4529 from xdelox/4308-static-schema-subfolders

Segregate the schema definitions under relevant sub folders

Author: hansva

core/src/main/java/org/apache/hop/metadata/api/HopMetadata.java

@@ -51,4 +51,12 @@
    * @return the type of metadata this property represents.
    */
   HopMetadataPropertyType hopMetadataPropertyType() default HopMetadataPropertyType.NONE;
+
+  /**
+   * A boolean flag, represented as a string, which tells whether the given HopMetadata
+   * implementation has to be shown into a tree of folders
+   *
+   * @return a boolean as string flag that will enable the UI to allow folders creation
+   */
+  String subfoldersEnabled() default "false";
 }

core/src/main/java/org/apache/hop/metadata/api/HopMetadataBase.java

@@ -22,6 +22,8 @@
 
 public class HopMetadataBase implements IHopMetadata {
 
+  @HopMetadataProperty protected String path;
+
   /** All metadata objects have a name to uniquely identify it. */
   @HopMetadataProperty protected String name;
 
@@ -36,6 +38,13 @@ public HopMetadataBase() {}
   public HopMetadataBase(String name) {
     this();
     this.name = name;
+    this.path = "";
+  }
+
+  public HopMetadataBase(String name, String path) {
+    this();
+    this.name = name;
+    this.path = path;
   }
 
   @Override
@@ -96,4 +105,27 @@ public String getMetadataProviderName() {
   public void setMetadataProviderName(String metadataProviderName) {
     this.metadataProviderName = metadataProviderName;
   }
+
+  @Override
+  public String getPath() {
+    return path;
+  }
+
+  @Override
+  public void setPath(String path) {
+    this.path = path;
+  }
+
+  @Override
+  public String getFullName() {
+    if (path == null || path.isBlank()) {
+      return name;
+    } else {
+      if (path.endsWith("/")) {
+        return path + name;
+      } else {
+        return path + "/" + name;
+      }
+    }
+  }
 }

core/src/main/java/org/apache/hop/metadata/api/IHopMetadata.java

@@ -52,4 +52,10 @@ public interface IHopMetadata {
    * @param metadataProviderName The source of metadata or null if it's not specified
    */
   void setMetadataProviderName(String metadataProviderName);
+
+  String getPath();
+
+  void setPath(String path);
+
+  String getFullName();
 }

core/src/main/java/org/apache/hop/metadata/api/IHopMetadataSerializer.java

@@ -19,6 +19,7 @@
 
 import java.util.List;
 import org.apache.hop.core.exception.HopException;
+import org.apache.hop.metadata.serializer.FileSystemNode;
 
 /**
  * This metadata interface describes how an object T can be serialized and analyzed.
@@ -67,6 +68,11 @@ public interface IHopMetadataSerializer<T extends IHopMetadata> {
    */
   List<String> listObjectNames() throws HopException;
 
+  default FileSystemNode getFileSystemTree() throws HopException {
+    throw new UnsupportedOperationException(
+        "listObjectNames(String folderName, Boolean recursive) is not supported by this metadata serializer");
+  }
+
   /**
    * See if an object with the given name exists.
    *

core/src/main/java/org/apache/hop/metadata/serializer/FileSystemNode.java

@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.metadata.serializer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+// Represents a node in the file system tree
+public class FileSystemNode {
+  private String name; // Name of the file or folder
+  private String path; // Absolute path to the file or folder
+  private Type type; // Type (file or folder)
+  private FileSystemNode parent; // Parent node
+  private List<FileSystemNode> children; // Children nodes (only for folder)
+
+  // Enum for node type
+  public enum Type {
+    FILE,
+    FOLDER
+  }
+
+  // Constructor
+
+  public FileSystemNode(String name, String path, Type type, FileSystemNode parent) {
+    this.name = name;
+    this.path = path;
+    this.type = type;
+    this.parent = parent;
+    this.children = new ArrayList<>();
+
+    if (this.parent != null) {
+      this.parent.addChild(this);
+    }
+  }
+
+  // Add a child node
+
+  public void addChild(FileSystemNode child) {
+    if (this.type == Type.FOLDER) {
+      this.children.add(child);
+    }
+  }
+
+  // Getters
+
+  public String getName() {
+    return name;
+  }
+
+  public String getPath() {
+    return path;
+  }
+
+  public Type getType() {
+    return type;
+  }
+
+  public FileSystemNode getParent() {
+    return parent;
+  }
+
+  public List<FileSystemNode> getChildren() {
+    return children;
+  }
+
+  public boolean isFolder() {
+    return type == Type.FOLDER;
+  }
+
+  public boolean isFile() {
+    return type == Type.FILE;
+  }
+}

core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializer.java

@@ -27,6 +27,7 @@
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import org.apache.commons.lang.StringUtils;
@@ -39,6 +40,7 @@
 import org.apache.hop.metadata.api.IHopMetadata;
 import org.apache.hop.metadata.api.IHopMetadataProvider;
 import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.serializer.FileSystemNode;
 import org.json.simple.JSONObject;
 
 /**
@@ -140,11 +142,11 @@ private void inheritVariables(T t) {
 
   @Override
   public void save(T t) throws HopException {
-    if (StringUtils.isEmpty(t.getName())) {
+    if (StringUtils.isEmpty(t.getFullName())) {
       throw new HopException("Error: To save a metadata object it needs to have a name");
     }
 
-    String filename = calculateFilename(t.getName());
+    String filename = calculateFilename(t);
     try {
 
       JSONObject jObject = parser.getJsonObject(t);
@@ -170,29 +172,63 @@ public void save(T t) throws HopException {
     }
   }
 
+  public String calculateFilename(T t) {
+    String prefix = t.getPath() == null ? baseFolder : t.getPath();
+    if (!prefix.endsWith("/")) {
+      prefix += "/";
+    }
+    return prefix + t.getName() + ".json";
+  }
+
   public String calculateFilename(String name) {
+    if (name.startsWith(baseFolder)) {
+      return name;
+    }
     return baseFolder + "/" + name + ".json";
   }
 
+  public String calculateFolderName(String name) {
+    if (name.startsWith(baseFolder)) {
+      return name;
+    }
+    return baseFolder + "/" + name;
+  }
+
   @Override
   public T delete(String name) throws HopException {
     if (name == null) {
       throw new HopException(
           "Error: you need to specify the name of the metadata object to delete");
     }
-    if (!exists(name)) {
+    boolean exists = exists(name);
+    if (!exists) {
       throw new HopException("Error: Object '" + name + "' doesn't exist");
     }
-    T t = load(name);
-    String filename = calculateFilename(name);
+
+    T t = null;
+    String objectName = null;
+    if (existsFolder(name)) {
+      objectName = calculateFolderName(name);
+    }
+    if (existsFile(name)) {
+      t = load(name);
+      objectName = calculateFilename(name);
+    }
+
     try {
-      boolean deleted = HopVfs.getFileObject(filename).delete();
+      var object = HopVfs.getFileObject(objectName);
+      boolean deleted = false;
+      if (object.isFolder()) {
+        deleted = object.deleteAll() > 0;
+      } else {
+        deleted = object.delete();
+      }
       if (!deleted) {
         throw new HopException(
-            "Error: Object '" + name + "' could not be deleted, filename : " + filename);
+            "Error: Object '" + name + "' could not be deleted, name : " + objectName);
       }
     } catch (FileSystemException e) {
-      throw new HopException("Error deleting Object '" + name + "' with filename : " + filename);
+      throw new HopException("Error deleting Object '" + name + "' with name : " + objectName);
     }
     return t;
   }
@@ -214,9 +250,77 @@ public List<String> listObjectNames() throws HopException {
     }
   }
 
+  @Override
+  public FileSystemNode getFileSystemTree() throws HopException {
+    FileObject currentItem = HopVfs.getFileObject(baseFolder);
+    FileSystemNode root =
+        new FileSystemNode(
+            currentItem.getName().getBaseName(),
+            currentItem.getName().getPath(),
+            FileSystemNode.Type.FOLDER,
+            null);
+    return getFileSystemTree(root);
+  }
+
+  private FileSystemNode getFileSystemTree(FileSystemNode parent) throws HopException {
+    FileObject currentItem = HopVfs.getFileObject(parent.getPath());
+    try {
+      Arrays.asList(currentItem.getChildren()).stream()
+          .forEach(
+              f -> {
+                try {
+                  if (f.isFolder()) {
+                    FileSystemNode currentFolder =
+                        new FileSystemNode(
+                            f.getName().getBaseName(),
+                            f.getName().getPath(),
+                            FileSystemNode.Type.FOLDER,
+                            parent);
+                    getFileSystemTree(currentFolder); // Recursive call for the folder
+                  } else {
+                    if (f.getName().getExtension().equals("json")) {
+                      FileSystemNode currentFile =
+                          new FileSystemNode(
+                              f.getName().getBaseName().replaceAll("\\.json$", ""),
+                              f.getName().getPath(),
+                              FileSystemNode.Type.FILE,
+                              parent);
+                    }
+                  }
+                } catch (FileSystemException | HopException e) {
+                  throw new RuntimeException(e);
+                }
+              });
+    } catch (Exception e) {
+      throw new HopException("Error searching for JSON files", e);
+    }
+    return parent;
+  }
+
+  //    try {
+  //      List<FileObject> jsonFiles = HopVfs.findFiles(folder, "json", false);
+  //      List<String> names = new ArrayList<>();
+  //      for (FileObject jsonFile : jsonFiles) {
+  //        String baseName = jsonFile.getName().getBaseName();
+  //        names.add(baseName.replaceAll("\\.json$", ""));
+  //      }
+  //      return names;
+  //    } catch (Exception e) {
+  //      throw new HopException("Error searching for JSON files", e);
+  //    }
   @Override
   public boolean exists(String name) throws HopException {
-    return HopVfs.fileExists(calculateFilename(name));
+    return existsFolder(name) || existsFile(name);
+  }
+
+  private boolean existsFile(String name) throws HopException {
+
+    return !existsFolder(name)
+        && (HopVfs.fileExists(name) || HopVfs.fileExists(calculateFilename(name)));
+  }
+
+  private boolean existsFolder(String name) throws HopException {
+    return HopVfs.fileExists(name) || HopVfs.fileExists(calculateFolderName(name));
   }
 
   /**

core/src/main/java/org/apache/hop/metadata/serializer/multi/MultiMetadataSerializer.java

@@ -29,6 +29,7 @@
 import org.apache.hop.metadata.api.IHopMetadata;
 import org.apache.hop.metadata.api.IHopMetadataProvider;
 import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.serializer.FileSystemNode;
 
 public class MultiMetadataSerializer<T extends IHopMetadata> implements IHopMetadataSerializer<T> {
 
@@ -141,6 +142,15 @@ public List<String> listObjectNames() throws HopException {
     return new ArrayList<>(set);
   }
 
+  @Override
+  public FileSystemNode getFileSystemTree() throws HopException {
+    FileSystemNode root = new FileSystemNode("root", "/", FileSystemNode.Type.FOLDER, null);
+    for (IHopMetadataProvider provider : multiProvider.getProviders()) {
+      root.addChild(provider.getSerializer(managedClass).getFileSystemTree());
+    }
+    return root;
+  }
+
   @Override
   public boolean exists(String name) throws HopException {
     for (IHopMetadataProvider provider : multiProvider.getProviders()) {