Add !mermaid command (#32)

Author: ryannedolan

hoptimator-catalog/src/main/java/com/linkedin/hoptimator/catalog/Resource.java

@@ -1,10 +1,15 @@
 package com.linkedin.hoptimator.catalog;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.UUID;
 import java.util.regex.Pattern;
 import java.util.regex.Matcher;
 import java.util.Scanner;
@@ -23,14 +28,29 @@
  *
  * Resources are injected into a Pipeline by the planner as needed. Generally,
  * each Resource Template corresponds to a Kubernetes controller.
+ *
+ * Resources may optionally link to input Resources, which is used strictly
+ * for informational/debugging purposes.
  */
 public abstract class Resource {
-  private final String template;
-  private final Map<String, Supplier<String>> properties = new HashMap<>();
+  private final String kind;
+  private final SortedMap<String, Supplier<String>> properties = new TreeMap<>();
+  private final List<Resource> inputs = new ArrayList<>();
+
+  /** A Resource of some kind. */
+  public Resource(String kind) {
+    this.kind = kind;
+  }
+
+  /** Copy constructor */
+  public Resource(Resource other) {
+    this.kind = other.kind;
+    this.properties.putAll(other.properties);
+    this.inputs.addAll(other.inputs);
+  }
 
-  /** A Resource that will be rendered with the specified template */
-  public Resource(String template) {
-    this.template = template;
+  public String kind() {
+    return kind;
   }
 
   /** Export a computed value to the template */
@@ -54,15 +74,16 @@ protected void export(String key, List<String> values) {
     export(key, values.stream().collect(Collectors.joining("\n")));
   }
 
-  /** The name of the template used to render this Resource */
-  public String template() {
-    return template;
+  /** Reference an input resource */
+  protected void input(Resource resource) {
+    inputs.add(resource);
   }
 
   public String property(String key) {
     return getOrDefault(key, null);
   }
 
+  /** Keys for all defined properties, in natural order. */
   public Set<String> keys() {
     return properties.keySet();
   }
@@ -76,15 +97,27 @@ public String render(Template template) {
     return template.render(this);
   }
 
+  public Collection<Resource> inputs() {
+    return inputs;
+  }
+
+  @Override
+  public int hashCode() {
+    return toString().hashCode();
+  } 
+
   @Override
   public String toString() {
     StringBuilder sb = new StringBuilder();
-    sb.append("[ ");
+    sb.append("[ kind: " + kind() + " ");
     for (Map.Entry<String, Supplier<String>> entry : properties.entrySet()) {
-      sb.append(entry.getKey());
-      sb.append(":");
-      sb.append(entry.getValue().get());
-      sb.append(" ");
+      String value = entry.getValue().get();
+      if (value != null && !value.isEmpty()) {
+        sb.append(entry.getKey());
+        sb.append(":");
+        sb.append(entry.getValue().get());
+        sb.append(" ");
+      }
     }
     sb.append("]");
     return sb.toString();
@@ -247,10 +280,10 @@ public SimpleTemplateFactory(Environment env) {
 
     @Override
     public Template get(Resource resource) {
-      String template = resource.template();
-      InputStream in = getClass().getClassLoader().getResourceAsStream(template + ".yaml.template");
+      String kind = resource.kind();
+      InputStream in = getClass().getClassLoader().getResourceAsStream(kind + ".yaml.template");
       if (in == null) {
-        throw new IllegalArgumentException("No template '" + template + "' found in jar resources");
+        throw new IllegalArgumentException("No template '" + kind + "' found in jar resources");
       }
       StringBuilder sb = new StringBuilder();
       Scanner scanner = new Scanner(in);

hoptimator-catalog/src/main/java/com/linkedin/hoptimator/catalog/ResourceProvider.java

@@ -1,10 +1,98 @@
 package com.linkedin.hoptimator.catalog;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
-/** In Hoptimator, Tables can have baggage in the form of Resources. */
+/**
+ * Enables an adapter to emit arbitrary Resources for a given table.
+ *
+ * Optionally, establishes source->sink relationships between such Resources. These are used
+ * strictly for debugging purposes.
+ */
 public interface ResourceProvider {
 
   /** Resources for the given table */
   Collection<Resource> resources(String tableName);
+
+  /**
+   * Establishes a source->sink relationship between this ResourceProvider and a sink Resource.
+   *
+   * All leaf-node Resources provided by this ResourceProvider will become inputs to the sink.
+   *
+   * e.g.
+   * <pre>
+   *   ResourceProvider.empty().with(x -> a).with(x -> b).to(x -> c).to(x -> d)
+   * </pre>
+   *
+   * encodes the following DAG:
+   * <pre>
+   *   a --> c
+   *   b --> c
+   *   c --> d
+   * </pre>
+   */
+  default ResourceProvider toAll(ResourceProvider sink) {
+    return x -> {
+      List<Resource> combined = new ArrayList<>();
+      List<Resource> sources = new ArrayList<>();
+      List<Resource> sinks = new ArrayList<>();
+      sources.addAll(resources(x));
+      combined.addAll(sources);
+
+      // remove all non-leaf-node upstream Resources
+      sources.removeAll(sources.stream().flatMap(y -> y.inputs().stream())
+        .collect(Collectors.toList()));
+
+      // link all sources to all sinks       
+      sink.resources(x).forEach(y -> {
+        combined.add(new Resource(y) {{
+          sources.forEach(z -> input(z));
+        }});
+      });
+
+      return combined;
+    };
+  }
+
+  default ResourceProvider to(Resource resource) {
+    return toAll(x -> Collections.singleton(resource));
+  }
+
+  default ResourceProvider to(Function<String, Resource> resourceFunc) {
+    return toAll(x -> Collections.singleton(resourceFunc.apply(x)));
+  }
+
+  /** Combines this ResourceProvider with another ResourceProvider */
+  default ResourceProvider withAll(ResourceProvider resourceProvider) {
+    return x -> {
+      List<Resource> combined = new ArrayList<>();
+      combined.addAll(resources(x));
+      combined.addAll(resourceProvider.resources(x));
+      return combined;
+    };
+  }
+
+  default ResourceProvider with(Resource resource) {
+    return withAll(x -> Collections.singleton(resource));
+  }
+
+  default ResourceProvider with(Function<String, Resource> resourceFunc) {
+    return withAll(x -> Collections.singleton(resourceFunc.apply(x)));
+  }
+
+  static ResourceProvider empty() {
+    return x -> Collections.emptyList();
+  }
+
+  static ResourceProvider from(Collection<Resource> resources) {
+    return x -> resources;
+  }
+
+  static ResourceProvider from(Resource resource) {
+    return x -> Collections.singleton(resource);
+  }
 }

hoptimator-cli/src/main/java/com/linkedin/hoptimator/HoptimatorCliApp.java

@@ -50,7 +50,8 @@ protected int run(String[] args) throws IOException {
     commandHandlers.add(new PipelineCommandHandler());
     commandHandlers.add(new IntroCommandHandler());
     commandHandlers.add(new InsertCommandHandler());
-    commandHandlers.add(new TestCommandHandler());
+    commandHandlers.add(new CheckCommandHandler());
+    commandHandlers.add(new MermaidCommandHandler());
     sqlline.updateCommandHandlers(commandHandlers);
     return sqlline.begin(args, null, true).ordinal();
   }
@@ -268,7 +269,7 @@ public boolean echoToFile() {
     }
   }
 
-  private class TestCommandHandler implements CommandHandler {
+  private class CheckCommandHandler implements CommandHandler {
 
     @Override
     public String getName() {
@@ -529,4 +530,79 @@ public boolean echoToFile() {
       return false;
     }
   }
+
+  private class MermaidCommandHandler implements CommandHandler {
+
+    @Override
+    public String getName() {
+      return "mermaid";
+    }
+
+    @Override
+    public List<String> getNames() {
+      return Collections.singletonList(getName());
+    }
+
+    @Override
+    public String getHelpText() {
+      return "Render a pipeline in mermaid format (similar to graphviz)";
+    }
+
+    @Override
+    public String matches(String line) {
+      String sql = line;
+      if (sql.startsWith(SqlLine.COMMAND_PREFIX)) {
+        sql = sql.substring(1);
+      }
+
+      if (sql.startsWith("mermaid")) {
+        sql = sql.substring("mermaid".length() + 1);
+        return sql;
+      }
+
+      return null;
+    }
+
+    @Override
+    public void execute(String line, DispatchCallback dispatchCallback) {
+      String sql = line;
+      if (sql.startsWith(SqlLine.COMMAND_PREFIX)) {
+        sql = sql.substring(1);
+      }
+
+      if (sql.startsWith("mermaid")) {
+        sql = sql.substring("mermaid".length() + 1);
+      }
+
+      //remove semicolon from query if present
+      if (sql.length() > 0 && sql.charAt(sql.length() - 1) == ';') {
+        sql = sql.substring(0, sql.length() - 1);
+      }
+
+      String connectionUrl = sqlline.getConnectionMetadata().getUrl();
+      try {
+        HoptimatorPlanner planner = HoptimatorPlanner.fromModelFile(connectionUrl, new Properties());
+        PipelineRel plan = planner.pipeline(sql);
+        PipelineRel.Implementor impl = new PipelineRel.Implementor(plan);
+        HopTable outputTable = new HopTable("PIPELINE", "SINK", plan.getRowType(),
+          Collections.singletonMap("connector", "dummy"));
+        Pipeline pipeline = impl.pipeline(outputTable);
+        sqlline.output(pipeline.mermaid());
+        dispatchCallback.setToSuccess();
+      } catch (Exception e) {
+        sqlline.error(e.toString());
+        dispatchCallback.setToFailure();
+      }
+    }
+
+    @Override
+    public List<Completer> getParameterCompleters() {
+      return Collections.emptyList();
+    }
+
+    @Override
+    public boolean echoToFile() {
+      return false;
+    }
+  }
 }

hoptimator-planner/src/main/java/com/linkedin/hoptimator/planner/Pipeline.java

@@ -5,6 +5,9 @@
 import com.linkedin.hoptimator.catalog.Resource;
 
 import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 /** A set of Resources that deliver data.
  *
@@ -35,4 +38,46 @@ public String render(Resource.TemplateFactory templateFactory) {
     }
     return sb.toString();
   }
+
+  /** Render a graph of resources in mermaid format */
+  public String mermaid() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("flowchart\n");
+    Map<String, List<Resource>> grouped = resources.stream()
+      .collect(Collectors.groupingBy(x -> x.kind()));
+    grouped.forEach((k, v) -> {
+      sb.append("  subgraph " + k + "\n");
+      v.forEach(x -> {
+        String description = x.keys().stream()
+          .filter(k2 -> x.property(k2) != null)
+          .filter(k2 -> !x.property(k2).isEmpty())
+          .map(k2 -> k2 + ": " + sanitize(x.property(k2)))
+          .collect(Collectors.joining("\n"));
+        sb.append("  " + id(x) + "[\"" + description + "\"]\n");
+      });
+      sb.append("  end\n");
+    });
+    grouped.forEach((k, v) -> {
+      sb.append("  subgraph " + k + "\n");
+      v.forEach(x -> {
+        x.inputs().forEach(y -> {
+          sb.append("    " + id(y) + " --> " + id(x) + "\n");
+        });
+      });
+      sb.append("  end\n");
+    });
+    return sb.toString();
+  }
+
+  private static String id(Resource resource) {
+    return "R" + Integer.toString(resource.hashCode());
+  }
+
+  private static String sanitize(String s) {
+    String safe = s.replaceAll("\"", "&quot;").replaceAll("\n", " ").trim();
+    if (safe.length() > 20) {
+      return safe.substring(0, 17) + "..."; 
+    }
+    return safe;
+  }
 }