feat: DAG support for define expressions

Author: noahboerger

pom.xml

@@ -163,7 +163,9 @@
                 <configuration>
                     <!-- Sets the VM argument line used when unit tests are run. -->
                     <!-- Suppress UnresolvedMavenProperty. -->
-                    <argLine>-Xms1g -Xmx2g @{surefireArgLine}</argLine>
+                    <argLine>-Xms1g -Xmx2g --add-opens fr.insee.vtl.engine/fr.insee.vtl.engine.utils.dag=ALL-UNNAMED
+                        @{surefireArgLine}
+                    </argLine>
                     <useSystemClassLoader>false</useSystemClassLoader>
                     <forkedProcessExitTimeoutInSeconds>180</forkedProcessExitTimeoutInSeconds>
                     <forkCount>1</forkCount>

vtl-engine/src/main/java/fr/insee/vtl/engine/VtlSyntaxPreprocessor.java

@@ -72,15 +72,19 @@ public void checkForMultipleAssignments() throws VtlScriptException {
     List<DAGStatement> bindingPseudoStatements =
         bindingVarIds.stream()
             .map(
-                bindingVar ->
-                    new DAGStatement(DAGStatement.PSEUDO_BINDING_POSITION, bindingVar, Set.of()))
+                bindingVarId ->
+                    new DAGStatement(
+                        DAGStatement.PSEUDO_BINDING_POSITION,
+                        new DAGStatement.Identifier(
+                            DAGStatement.Identifier.Type.VARIABLE, bindingVarId),
+                        Set.of()))
             .toList();
-    Map<String, List<DAGStatement>> groupedByProducedVar =
+    Map<DAGStatement.Identifier, List<DAGStatement>> groupedByProducedIdentifier =
         Stream.concat(bindingPseudoStatements.stream(), unsortedStatements.stream())
             .collect(Collectors.groupingBy(DAGStatement::produces));
 
     List<VtlScriptException> multiProducedExceptions =
-        groupedByProducedVar.entrySet().stream()
+        groupedByProducedIdentifier.entrySet().stream()
             .filter(produced -> produced.getValue().size() > 1)
             .map(
                 multiProduced ->
@@ -95,21 +99,23 @@ public void checkForMultipleAssignments() throws VtlScriptException {
   }
 
   private VtlScriptException buildScriptExceptionFromMultipleAssignment(
-      String varId, List<DAGStatement> statements) {
+      DAGStatement.Identifier identifier, List<DAGStatement> statements) {
     final List<DAGStatement> statementsWithoutBinding =
         statements.stream()
             .filter(statement -> statement.unsortedIndex() != DAGStatement.PSEUDO_BINDING_POSITION)
             .toList();
 
     if (statementsWithoutBinding.size() == 1) {
       return new VtlScriptException(
-          "Dataset " + varId + " is part of the bindings and therefore cannot be assigned",
+          "Dataset "
+              + identifier.name()
+              + " is part of the bindings and therefore cannot be assigned",
           statementsWithoutBinding.get(0).getPosition(startContext));
     }
 
     return DAGStatement.buildMultiStatementExceptionUsingTheLastDAGStatementAsMainPosition(
         "Dataset "
-            + varId
+            + identifier.name()
             + " has already been assigned"
             + (statements.size() == statementsWithoutBinding.size()
                 ? ""

vtl-engine/src/main/java/fr/insee/vtl/engine/utils/dag/DAGBuilder.java

@@ -50,7 +50,7 @@ private static Graph<DAGStatement, DefaultEdge> buildDAG(List<DAGStatement> dagS
 
   private static boolean dependsOn(DAGStatement stmt2, DAGStatement stmt1) {
     // Check if stmt2 consumes data from stmt1
-    String produced = stmt1.produces();
+    DAGStatement.Identifier produced = stmt1.produces();
     return stmt2.consumes().contains(produced);
   }
 
@@ -101,55 +101,54 @@ private VtlScriptException buildVTLScriptExceptionForCycles(List<Set<DAGStatemen
 
   private String buildAssignmentChain(Set<DAGStatement> cycle) {
     // Collect all produced variable names in this cycle
-    Set<String> producedVars =
+    Set<DAGStatement.Identifier> producedIdentifiers =
         cycle.stream().map(DAGStatement::produces).collect(Collectors.toSet());
 
     // Pick a stable start
-    String startVar =
-        producedVars.stream()
-            .sorted()
-            .findFirst()
+    DAGStatement.Identifier startIdentifier =
+        producedIdentifiers.stream()
+            .min(Comparator.comparing(DAGStatement.Identifier::name))
             .orElseThrow(() -> new AssertionError("Cycle contains out of at least two statements"));
 
     StringBuilder sb = new StringBuilder();
-    String current = startVar;
+    DAGStatement.Identifier currentIdentifier = startIdentifier;
 
     do {
-      sb.append(current).append(" <- ");
+      sb.append(currentIdentifier.name()).append(" <- ");
 
       // Find the unique statement that produces 'current'
-      String finalCurrent = current;
+      DAGStatement.Identifier finalCurrentIdentifier = currentIdentifier;
       DAGStatement producer =
           cycle.stream()
-              .filter(stmt -> stmt.produces().equals(finalCurrent))
+              .filter(stmt -> stmt.produces().equals(finalCurrentIdentifier))
               .reduce(
                   (a, b) -> {
                     throw new AssertionError(
                         "Multiple producers of "
-                            + finalCurrent
+                            + finalCurrentIdentifier.name()
                             + " cannot occur here, this is already validated before");
                   })
               .orElseThrow(
                   () ->
                       new AssertionError(
                           "A cycle is always closed, there must be a consumer for  "
-                              + finalCurrent));
+                              + finalCurrentIdentifier.name()));
 
       // Choose the next consumed variable that stays inside the cycle
-      current =
+      currentIdentifier =
           producer.consumes().stream()
-              .filter(producedVars::contains)
+              .filter(producedIdentifiers::contains)
               .findFirst()
               .orElseThrow(
                   () ->
                       new AssertionError(
                           "Broken cycle at "
-                              + finalCurrent
+                              + finalCurrentIdentifier.name()
                               + ": no consumed var stays inside cycle"));
-    } while (!current.equals(startVar));
+    } while (!currentIdentifier.equals(startIdentifier));
 
     // close the loop
-    sb.append(startVar);
+    sb.append(startIdentifier.name());
     return "[" + sb + "]";
   }
 

vtl-engine/src/main/java/fr/insee/vtl/engine/utils/dag/DAGStatement.java

@@ -9,6 +9,10 @@
 import java.util.HashSet;
 import java.util.Set;
 import java.util.stream.Collectors;
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.tree.ParseTree;
+import org.antlr.v4.runtime.tree.RuleNode;
+import org.antlr.v4.runtime.tree.TerminalNode;
 
 /**
  * Representation of a VTL Statement
@@ -17,7 +21,7 @@
  * @param produces Produced data
  * @param consumes Consumed data
  */
-public record DAGStatement(int unsortedIndex, String produces, Set<String> consumes) {
+public record DAGStatement(int unsortedIndex, Identifier produces, Set<Identifier> consumes) {
 
   public static int PSEUDO_BINDING_POSITION = Integer.MIN_VALUE;
 
@@ -45,6 +49,28 @@ public record DAGStatement(int unsortedIndex, String produces, Set<String> consu
         message, lastDagStatement.getPosition(startContext), restPositions);
   }
 
+  public static DAGStatement of(
+      Identifier.Type outIdentifierType,
+      TerminalNode outIdentifierNode,
+      Set<Identifier> inIdentifiers,
+      ParserRuleContext node) {
+    Identifier rulesetOutIdentifier =
+        new Identifier(outIdentifierType, outIdentifierNode.getSymbol().getText());
+    final int statementIndex = getParentStatementIndex(node);
+    return new DAGStatement(statementIndex, rulesetOutIdentifier, inIdentifiers);
+  }
+
+  private static int getParentStatementIndex(final RuleNode node) {
+    final ParseTree parent = node.getParent();
+    for (int i = 0; i < parent.getChildCount(); ++i) {
+      final ParseTree child = parent.getChild(i);
+      if (child == node) {
+        return i;
+      }
+    }
+    throw new AssertionError("Statement must always be part of the its parent node");
+  }
+
   @Override
   public String toString() {
     return "Statement{"
@@ -61,4 +87,13 @@ public String toString() {
   public Positioned getPosition(final VtlParser.StartContext startContext) {
     return VtlScriptEngine.fromContext(startContext.getChild(unsortedIndex));
   }
+
+  public record Identifier(Type identifierType, String name) {
+    public enum Type {
+      VARIABLE,
+      OPERATOR,
+      RULESET_HIERARCHICAL,
+      RULESET_DATAPOINT
+    }
+  }
 }