Merge pull request #531 from mspruc/main

add support for cross joins in the sql-api

Author: juripetersen

wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/WayangCrossJoinVisitor.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.wayang.api.sql.calcite.converter;
+
+import java.io.Serializable;
+
+import org.apache.wayang.api.sql.calcite.converter.functions.FlattenJoinResult;
+import org.apache.wayang.api.sql.calcite.rel.WayangJoin;
+import org.apache.wayang.basic.data.Record;
+import org.apache.wayang.basic.data.Tuple2;
+import org.apache.wayang.basic.operators.CartesianOperator;
+import org.apache.wayang.basic.operators.MapOperator;
+import org.apache.wayang.core.function.FunctionDescriptor.SerializableFunction;
+import org.apache.wayang.core.plan.wayangplan.Operator;
+import org.apache.wayang.core.util.ReflectionUtils;
+
+public class WayangCrossJoinVisitor extends WayangRelNodeVisitor<WayangJoin> implements Serializable {
+
+    WayangCrossJoinVisitor(final WayangRelConverter wayangRelConverter) {
+        super(wayangRelConverter);
+    }
+
+    @Override
+    Operator visit(final WayangJoin wayangRelNode) {
+        final Operator childOpLeft = wayangRelConverter.convert(wayangRelNode.getInput(0));
+        final Operator childOpRight = wayangRelConverter.convert(wayangRelNode.getInput(1));
+
+        final CartesianOperator<Record, Record> join = new CartesianOperator<Record, Record>(
+                Record.class,
+                Record.class);
+
+        childOpLeft.connectTo(0, join, 0);
+        childOpRight.connectTo(0, join, 1);
+
+        final SerializableFunction<Tuple2<Record, Record>, Record> mp = new FlattenJoinResult();
+
+        final MapOperator<Tuple2<Record, Record>, Record> mapOperator = new MapOperator<Tuple2<Record, Record>, Record>(
+                mp,
+                ReflectionUtils.specify(Tuple2.class),
+                Record.class);
+
+        join.connectTo(0, mapOperator, 0);
+
+        return mapOperator;
+    }
+
+}

wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/WayangRelConverter.java

@@ -54,6 +54,8 @@ public Operator convert(final RelNode node) {
             return new WayangProjectVisitor(this).visit((WayangProject) node);
         } else if (node instanceof WayangFilter) {
             return new WayangFilterVisitor(this).visit((WayangFilter) node);
+        } else if (node instanceof WayangJoin && WayangJoin.class.cast(node).getCondition().isAlwaysTrue()) {
+            return new WayangCrossJoinVisitor(this).visit((WayangJoin) node);
         } else if (node instanceof WayangJoin) {
             return new WayangJoinVisitor(this).visit((WayangJoin) node);
         } else if (node instanceof WayangAggregate) {

wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/calcite/converter/functions/FlattenJoinResult.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.wayang.api.sql.calcite.converter.functions;
+
+import org.apache.wayang.basic.data.Record;
+import org.apache.wayang.basic.data.Tuple2;
+import org.apache.wayang.core.function.FunctionDescriptor;
+
+/**
+ * Flattens the result of a join i.e. a {@link Tuple2} of a left and a right
+ * {@link Record} to a single {@link Record}.
+ */
+public class FlattenJoinResult implements FunctionDescriptor.SerializableFunction<Tuple2<Record, Record>, Record> {
+
+    @Override
+    public Record apply(final Tuple2<Record, Record> tuple2) {
+        final int length0 = tuple2.getField0().size();
+        final int length1 = tuple2.getField1().size();
+
+        final int totalLength = length0 + length1;
+
+        final Object[] fields = new Object[totalLength];
+
+        for (int i = 0; i < length0; i++) {
+            fields[i] = tuple2.getField0().getField(i);
+        }
+
+        for (int i = length0; i < totalLength; i++) {
+            fields[i] = tuple2.getField1().getField(i - length0);
+        }
+
+        return new Record(fields);
+    }
+}

wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java

@@ -58,7 +58,10 @@
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
+import java.util.Map;
 import java.util.Properties;
+import java.util.stream.Collectors;
 
 public class SqlToWayangRelTest {
 
@@ -212,6 +215,34 @@ public void filterIsNotNull() throws Exception {
         assert (!result.stream().anyMatch(record -> record.getField(0).equals(null)));
     }
 
+    @Test
+    public void javaCrossJoin() throws Exception {
+        final SqlContext sqlContext = createSqlContext("/data/largeLeftTableIndex.csv");
+
+        final Tuple2<Collection<Record>, WayangPlan> t = this.buildCollectorAndWayangPlan(
+                sqlContext,
+                "select * from fs.exampleSmallA cross join fs.exampleSmallB");
+
+        final Collection<Record> result = t.field0;
+        final WayangPlan wayangPlan = t.field1;
+
+        sqlContext.execute(wayangPlan);
+
+        final List<Record> shouldBe = List.of(
+                new Record("item1", "item2", "item1", "item2", "item3"),
+                new Record("item1", "item2", "item1", "item2", "item3"),
+                new Record("item1", "item2", "item1", "item2", "item3"),
+                new Record("item1", "item2", "item1", "item2", "item3"),
+                new Record("item1", "item2", "x"    , "x"    , "x"),
+                new Record("item1", "item2", "x"    , "x"    , "x")
+        );
+
+        final Map<Record, Integer> resultTally = result.stream().collect(Collectors.toMap(rec -> rec, rec -> 1, Integer::sum));
+        final Map<Record, Integer> shouldBeTally = shouldBe.stream().collect(Collectors.toMap(rec -> rec, rec -> 1, Integer::sum));
+
+        assert (resultTally.equals(shouldBeTally));
+    }
+
     @Test
     public void filterWithNotLike() throws Exception {
         final SqlContext sqlContext = createSqlContext("/data/largeLeftTableIndex.csv");

wayang-api/wayang-api-sql/src/test/resources/data/exampleSmallA.csv

@@ -0,0 +1,3 @@
+COLA:string,COLB:string
+item1;item2
+item1;item2

wayang-api/wayang-api-sql/src/test/resources/data/exampleSmallB.csv

@@ -0,0 +1,4 @@
+COLA:string,COLB:string,COLC:string
+item1;item2;item3
+item1;item2;item3
+x;x;x