mongodb add predicate

todd5167 · todd5167 · commit 18feeaf0b91c · 2019-12-17T15:15:23.000+08:00
diff --git a/cassandra/cassandra-side/cassandra-all-side/pom.xml b/cassandra/cassandra-side/cassandra-all-side/pom.xml
@@ -36,6 +36,7 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <artifactSet>
                                 <excludes>
                                     <exclude>org.slf4j</exclude>
diff --git a/cassandra/cassandra-side/cassandra-async-side/pom.xml b/cassandra/cassandra-side/cassandra-async-side/pom.xml
@@ -52,6 +52,7 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <artifactSet>
                                 <excludes>
                                     <exclude>org.slf4j</exclude>
diff --git a/console/console-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/console/console-sink/src/test/java/com/dtstack/flinkx/AppTest.java
diff --git a/hbase/hbase-side/hbase-all-side/pom.xml b/hbase/hbase-side/hbase-all-side/pom.xml
@@ -50,6 +50,7 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <artifactSet>
                                 <excludes>
                                     <exclude>org.apache.hadoop:hadoop-common</exclude>
diff --git a/hbase/hbase-side/hbase-async-side/pom.xml b/hbase/hbase-side/hbase-async-side/pom.xml
@@ -47,6 +47,7 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <artifactSet>
                                 <excludes>
                                     <exclude>org.apache.hadoop:hadoop-common</exclude>
diff --git a/hbase/hbase-sink/pom.xml b/hbase/hbase-sink/pom.xml
@@ -25,6 +25,7 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <artifactSet>
                                 <excludes>
                                     <exclude>org.slf4j:*</exclude>
diff --git a/kafka/kafka-sink/pom.xml b/kafka/kafka-sink/pom.xml
@@ -37,6 +37,7 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <artifactSet>
                                 <excludes>
                                     <exclude>org.slf4j</exclude>
diff --git a/kafka/kafka-source/pom.xml b/kafka/kafka-source/pom.xml
@@ -30,6 +30,7 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <artifactSet>
                                 <excludes>
                                     <exclude>org.slf4j</exclude>
diff --git a/kafka10/kafka10-sink/pom.xml b/kafka10/kafka10-sink/pom.xml
@@ -37,6 +37,7 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <artifactSet>
                                 <excludes>
                                     <exclude>org.slf4j</exclude>
diff --git a/kafka10/kafka10-source/pom.xml b/kafka10/kafka10-source/pom.xml
@@ -27,6 +27,7 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <artifactSet>
                                 <excludes>
                                     <exclude>org.slf4j</exclude>
diff --git a/kafka11/kafka11-sink/pom.xml b/kafka11/kafka11-sink/pom.xml
@@ -37,6 +37,7 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <artifactSet>
                                 <excludes>
                                     <exclude>org.slf4j</exclude>
diff --git a/kafka11/kafka11-source/pom.xml b/kafka11/kafka11-source/pom.xml
@@ -29,6 +29,7 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <artifactSet>
                                 <excludes>
                                     <exclude>org.slf4j</exclude>
diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java
@@ -23,6 +23,7 @@
 import com.dtstack.flink.sql.side.JoinInfo;
 import com.dtstack.flink.sql.side.SideTableInfo;
 import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo;
+import com.dtstack.flink.sql.side.mongo.utils.MongoUtil;
 import com.mongodb.BasicDBObject;
 import com.mongodb.MongoClient;
 import com.mongodb.MongoClientOptions;
@@ -239,7 +240,22 @@ private void loadData(Map<String, List<Map<String, Object>>> tmpCache) throws SQ
             for (String selectField : sideFieldNames) {
                 basicDBObject.append(selectField, 1);
             }
-            FindIterable<Document> findIterable = dbCollection.find().projection(basicDBObject).limit(FETCH_SIZE);
+            BasicDBObject filterObject = new BasicDBObject();
+            try {
+                // 填充谓词
+                sideInfo.getSideTableInfo().getPredicateInfoes().stream().map(info -> {
+                    BasicDBObject filterCondition = MongoUtil.buildFilterObject(info);
+                    if (null != filterCondition) {
+                        filterObject.append(info.getFieldName(), filterCondition);
+                    }
+                    return info;
+                }).count();
+            } catch (Exception e) {
+                LOG.info("add predicate infoes error ", e);
+            }
+
+
+            FindIterable<Document> findIterable = dbCollection.find(filterObject).projection(basicDBObject).limit(FETCH_SIZE);
             MongoCursor<Document> mongoCursor = findIterable.iterator();
             while (mongoCursor.hasNext()) {
                 Document doc = mongoCursor.next();
diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java
@@ -22,7 +22,6 @@
 import com.dtstack.flink.sql.side.JoinInfo;
 import com.dtstack.flink.sql.side.SideInfo;
 import com.dtstack.flink.sql.side.SideTableInfo;
-import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo;
 import com.dtstack.flink.sql.util.ParseUtils;
 import org.apache.calcite.sql.SqlNode;
 import org.apache.commons.collections.CollectionUtils;
@@ -47,11 +46,6 @@ public MongoAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List<FieldIn
 
     @Override
     public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) {
-        MongoSideTableInfo MongoSideTableInfo = (MongoSideTableInfo) sideTableInfo;
-
-        sqlCondition = "select ${selectField} from ${tableName} ";
-        sqlCondition = sqlCondition.replace("${tableName}", MongoSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields);
-        System.out.println("---------side_exe_sql-----\n" + sqlCondition);
     }
 
     @Override
diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java
@@ -26,6 +26,7 @@
 import com.dtstack.flink.sql.side.SideTableInfo;
 import com.dtstack.flink.sql.side.cache.CacheObj;
 import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo;
+import com.dtstack.flink.sql.side.mongo.utils.MongoUtil;
 import com.mongodb.BasicDBObject;
 import com.mongodb.Block;
 import com.mongodb.MongoCredential;
@@ -134,6 +135,19 @@ public void asyncInvoke(Row input, ResultFuture<Row> resultFuture) throws Except
             }
             basicDBObject.put(sideInfo.getEqualFieldList().get(i), equalObj);
         }
+        try {
+            // 填充谓词
+            sideInfo.getSideTableInfo().getPredicateInfoes().stream().map(info -> {
+                BasicDBObject filterCondition = MongoUtil.buildFilterObject(info);
+                if (null != filterCondition) {
+                    basicDBObject.append(info.getFieldName(), filterCondition);
+                }
+                return info;
+            }).count();
+        } catch (Exception e) {
+            LOG.info("add predicate infoes error ", e);
+        }
+
         String key = buildCacheKey(basicDBObject.values());
         if (openCache()) {
             CacheObj val = getFromCache(key);
diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java
@@ -49,31 +49,6 @@ public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List<Field
 
     @Override
     public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) {
-        MongoSideTableInfo MongoSideTableInfo = (MongoSideTableInfo) sideTableInfo;
-
-        String sideTableName = joinInfo.getSideTableName();
-
-        SqlNode conditionNode = joinInfo.getCondition();
-
-        List<SqlNode> sqlNodeList = Lists.newArrayList();
-        ParseUtils.parseAnd(conditionNode, sqlNodeList);
-
-        for(SqlNode sqlNode : sqlNodeList){
-            dealOneEqualCon(sqlNode, sideTableName);
-        }
-
-        sqlCondition = "select ${selectField} from ${tableName} where ";
-        for(int i=0; i<equalFieldList.size(); i++){
-            String equalField = equalFieldList.get(i);
-
-            sqlCondition += equalField + "=? ";
-            if(i != equalFieldList.size() - 1){
-                sqlCondition += " and ";
-            }
-        }
-
-        sqlCondition = sqlCondition.replace("${tableName}", MongoSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields);
-        System.out.println("---------side_exe_sql-----\n" + sqlCondition);
     }
 
 
diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.dtstack.flink.sql.side.mongo.utils;
+
+import com.dtstack.flink.sql.side.PredicateInfo;
+import com.mongodb.BasicDBObject;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Date: 2019/12/17
+ * Company: www.dtstack.com
+ * @author maqi
+ */
+public class MongoUtil {
+    public static BasicDBObject buildFilterObject(PredicateInfo info) {
+        switch (info.getOperatorName()) {
+            case "=":
+                return new BasicDBObject("$eq", info.getCondition());
+            case ">":
+                return new BasicDBObject("$gt", info.getCondition());
+            case ">=":
+                return new BasicDBObject("$gte", info.getCondition());
+            case "<":
+                return new BasicDBObject("$lt", info.getCondition());
+            case "<=":
+                return new BasicDBObject("$lte", info.getCondition());
+            case "<>":
+                return new BasicDBObject("$ne", info.getCondition());
+            case "IN":
+                Object[] values = Arrays.stream(info.getCondition().split(",")).map(String::trim)
+                        .collect(Collectors.toList()).toArray();
+                return new BasicDBObject("$in", values);
+            case "NOT IN":
+                return new BasicDBObject("$nin", info.getCondition().split(","));
+            case "IS NOT NULL":
+                return new BasicDBObject("$exists", true);
+            case "IS NULL":
+                return new BasicDBObject("$exists", false);
+        }
+        return null;
+    }
+}
diff --git a/mongo/mongo-sink/pom.xml b/mongo/mongo-sink/pom.xml
@@ -26,6 +26,7 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <artifactSet>
                                 <excludes>
 
