VortexExpr VTables (#3713)

Adds the same vtable machinery as arrays and layouts already use. It
uses the "Encoding" naming scheme from arrays and layouts. I don't
particularly like it, but it's consistent. Open to renames later.

Further, adds an expression registry to the Vortex session that will be
used for deserialization.

Expressions only decide their "options" serialization. So in theory, can
support many container formats, not just proto, provided each expression
can deserialize their own options format.

---------

Signed-off-by: Nicholas Gates <[email protected]>

Author: gatesn

Cargo.lock

@@ -88,6 +88,7 @@ datafusion-common = { version = "48" }
 datafusion-physical-plan = { version = "48" }
 dirs = "6.0.0"
 divan = { package = "codspeed-divan-compat", version = "3.0" }
+dyn-eq = "0.1.3"
 dyn-hash = "0.2.0"
 enum-iterator = "2.0.0"
 erased-serde = "0.4"

Cargo.toml

@@ -5,13 +5,31 @@
 
 import dev.vortex.api.expressions.*;
 
+import java.util.List;
+import java.util.Optional;
+
 /**
  * Vortex expression language.
  */
 public interface Expression {
-    String type();
-
-    <T> T accept(Visitor<T> visitor);
+    /**
+     * The globally unique identifier for this type of expression.
+     */
+    String id();
+
+    /**
+     * Returns the children of this expression.
+     */
+    List<Expression> children();
+
+    /**
+     * Returns the serialized metadata for this expression, or empty if serialization is not supported.
+     */
+    Optional<byte[]> metadata();
+
+    default <T> T accept(Visitor<T> visitor) {
+        return visitor.visitOther(this);
+    }
 
     interface Visitor<T> {
         T visitLiteral(Literal<?> literal);
@@ -23,5 +41,10 @@ interface Visitor<T> {
         T visitNot(Not not);
 
         T visitGetItem(GetItem getItem);
+
+        /**
+         * For expressions that do not have a specific visitor method.
+         */
+        T visitOther(Expression expression);
     }
 }

java/testfiles/Cargo.lock

@@ -3,8 +3,13 @@
 
 package dev.vortex.api.expressions;
 
+import com.google.protobuf.InvalidProtocolBufferException;
 import dev.vortex.api.Expression;
+import dev.vortex.proto.ExprProtos;
+
+import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.stream.Stream;
 
 public final class Binary implements Expression {
@@ -18,6 +23,20 @@ private Binary(BinaryOp operator, Expression left, Expression right) {
         this.right = right;
     }
 
+    public static Binary parse(byte[] metadata, List<Expression> children) {
+        if (children.size() != 2) {
+            throw new IllegalArgumentException(
+                    "Binary expression must have exactly two children, found: " + children.size());
+        }
+        try {
+            ExprProtos.BinaryOpts opts = ExprProtos.BinaryOpts.parseFrom(metadata);
+            BinaryOp operator = BinaryOp.fromProto(opts.getOp());
+            return new Binary(operator, children.get(0), children.get(1));
+        } catch (InvalidProtocolBufferException e) {
+            throw new IllegalArgumentException("Failed to parse Binary metadata", e);
+        }
+    }
+
     public static Binary of(BinaryOp operator, Expression left, Expression right) {
         return new Binary(operator, left, right);
     }
@@ -57,10 +76,23 @@ public static Binary ltEq(Expression left, Expression right) {
     }
 
     @Override
-    public String type() {
+    public String id() {
         return "binary";
     }
 
+    @Override
+    public List<Expression> children() {
+        return List.of(left, right);
+    }
+
+    @Override
+    public Optional<byte[]> metadata() {
+        return Optional.of(ExprProtos.BinaryOpts.newBuilder()
+                .setOp(operator.toProto())
+                .build()
+                .toByteArray());
+    }
+
     @Override
     public String toString() {
         return "(" + left + " " + operator + " " + right + ")";
@@ -131,5 +163,51 @@ public String toString() {
                     throw new IllegalStateException("Unknown Operator: " + this);
             }
         }
+
+        static BinaryOp fromProto(ExprProtos.BinaryOpts.BinaryOp proto) {
+            switch (proto) {
+                case Eq:
+                    return EQ;
+                case NotEq:
+                    return NOT_EQ;
+                case Gt:
+                    return GT;
+                case Gte:
+                    return GT_EQ;
+                case Lt:
+                    return LT;
+                case Lte:
+                    return LT_EQ;
+                case And:
+                    return AND;
+                case Or:
+                    return OR;
+                default:
+                    throw new IllegalArgumentException("Unsupported binary operator proto: " + proto);
+            }
+        }
+
+        ExprProtos.BinaryOpts.BinaryOp toProto() {
+            switch (this) {
+                case EQ:
+                    return ExprProtos.BinaryOpts.BinaryOp.Eq;
+                case NOT_EQ:
+                    return ExprProtos.BinaryOpts.BinaryOp.NotEq;
+                case GT:
+                    return ExprProtos.BinaryOpts.BinaryOp.Gt;
+                case GT_EQ:
+                    return ExprProtos.BinaryOpts.BinaryOp.Gte;
+                case LT:
+                    return ExprProtos.BinaryOpts.BinaryOp.Lt;
+                case LT_EQ:
+                    return ExprProtos.BinaryOpts.BinaryOp.Lte;
+                case AND:
+                    return ExprProtos.BinaryOpts.BinaryOp.And;
+                case OR:
+                    return ExprProtos.BinaryOpts.BinaryOp.Or;
+                default:
+                    throw new IllegalArgumentException("Unsupported binary operator: " + this);
+            }
+        }
     }
 }

java/vortex-jni/src/main/java/dev/vortex/api/Expression.java

@@ -3,8 +3,13 @@
 
 package dev.vortex.api.expressions;
 
+import com.google.protobuf.InvalidProtocolBufferException;
 import dev.vortex.api.Expression;
+import dev.vortex.proto.ExprProtos;
+
+import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 
 public final class GetItem implements Expression {
     private final String path;
@@ -19,6 +24,19 @@ public static GetItem of(Expression child, String path) {
         return new GetItem(child, path);
     }
 
+    public static GetItem parse(byte[] metadata, List<Expression> children) {
+        if (children.size() != 1) {
+            throw new IllegalArgumentException(
+                    "GetItem expression must have exactly one child, found: " + children.size());
+        }
+        try {
+            ExprProtos.GetItemOpts opts = ExprProtos.GetItemOpts.parseFrom(metadata);
+            return new GetItem(children.get(0), opts.getPath());
+        } catch (InvalidProtocolBufferException e) {
+            throw new IllegalArgumentException("Failed to parse GetItem metadata", e);
+        }
+    }
+
     public Expression getChild() {
         return child;
     }
@@ -28,10 +46,21 @@ public String getPath() {
     }
 
     @Override
-    public String type() {
+    public String id() {
         return "get_item";
     }
 
+    @Override
+    public List<Expression> children() {
+        return List.of(child);
+    }
+
+    @Override
+    public Optional<byte[]> metadata() {
+        return Optional.of(
+                ExprProtos.GetItemOpts.newBuilder().setPath(path).build().toByteArray());
+    }
+
     @Override
     public <T> T accept(Visitor<T> visitor) {
         return visitor.visitGetItem(this);

java/vortex-jni/src/main/java/dev/vortex/api/expressions/Binary.java

@@ -4,15 +4,51 @@
 package dev.vortex.api.expressions;
 
 import dev.vortex.api.Expression;
+import dev.vortex.proto.ExprProtos;
+
+import java.util.List;
+import java.util.Optional;
 
 public final class Identity implements Expression {
     public static final Identity INSTANCE = new Identity();
 
     private Identity() {}
 
+    public static Identity parse(byte[] metadata, List<Expression> children) {
+        if (!children.isEmpty()) {
+            throw new IllegalArgumentException("Identity expression must have no children, found: " + children.size());
+        }
+        try {
+            ExprProtos.VarOpts varOpts = ExprProtos.VarOpts.parseFrom(metadata);
+            if (!varOpts.getVar().isEmpty()) {
+                throw new IllegalArgumentException(
+                        "Identity expression must have empty var name, found: " + varOpts.getVar());
+            }
+            return INSTANCE;
+        } catch (Exception e) {
+            throw new IllegalArgumentException("Failed to parse metadata for Identity expression", e);
+        }
+    }
+
     @Override
-    public String type() {
-        return "identity";
+    public String id() {
+        return "var";
+    }
+
+    @Override
+    public List<Expression> children() {
+        return List.of();
+    }
+
+    @Override
+    public Optional<byte[]> metadata() {
+        // TODO(ngates): currently Vortex has a Var expression, but this will be reverted back
+        //  to an Identity expression in the future.
+        return Optional.of(ExprProtos.VarOpts.newBuilder()
+                // Identity has empty var name
+                .setVar("")
+                .build()
+                .toByteArray());
     }
 
     @Override