Dev 1.7.4 (#37)

* add basic table ignore config
* fix bug of data leak

Author: wangyue168git

ChangeLog.md

@@ -1,5 +1,9 @@
 # 更新历史
 
+### V1.7.4 (2021-07-08)
+- 增加基础表和表字段过滤能力
+- 修复区块多笔交易场景下，事件解析只解析首条的bug
+
 ### V1.7.3 (2021-05-21)
 - 服务方式启动整合编译插件，配置合约即可完成导出
 - sdk修复部分bug

WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/bo/data/BlockRawDataBO.java

@@ -31,5 +31,4 @@ public class BlockRawDataBO {
     private String gasUsed;
     private String signatureList;
     private String transactionList;
-    protected Date depotUpdatetime;
 }

WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/entity/ExportConfig.java

@@ -33,6 +33,11 @@ public class ExportConfig {
      */
     private Map<String, Map<String,List<String>>> ignoreParam = new HashMap<>();
 
+    /**
+     * ex: Map<contractName, Map<methodName or eventName, List<solidityParamName>>>
+     */
+    private Map<String, List<String>> ignoreBasicDataTableParam = new HashMap<>();
+
     /**
      * ex: Map<contractName, Map<methodName or eventName, Map<solidityParamName,paramType>>>
      */

WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/enums/DataType.java

@@ -36,6 +36,15 @@ public static List<DataType> getDefault(){
         return ListUtil.toList(NULL);
     }
 
+    public static DataType getDataType(String tableName){
+        for(DataType dataType : DataType.values()){
+            if (dataType.tableName.equals(tableName)){
+                return dataType;
+            }
+        }
+        return NULL;
+    }
+
     public static List<String> getTables(List<DataType> dataTypes){
         if (CollectionUtil.isEmpty(dataTypes)) {
             return Collections.emptyList();

WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-common/src/main/java/com/webank/blockchain/data/export/common/enums/IgnoreBasicDataParam.java

@@ -0,0 +1,75 @@
+package com.webank.blockchain.data.export.common.enums;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+/**
+ * @author wesleywang
+ * @Description:
+ * @date 2021/6/23
+ */
+@AllArgsConstructor
+@Getter
+public class IgnoreBasicDataParam {
+
+    @Getter
+    public enum IgnoreBasicDataTable {
+        BLOCK_RAW_DATA_TABLE,
+        TX_RAW_DATA_TABLE,
+        TX_RECEIPT_RAW_DATA_TABLE;
+
+        IgnoreBasicDataTable() { }
+    }
+
+    @Getter
+    public enum BlockRawDataParams {
+        DB_HASH,
+        EXTRA_DATA,
+        GAS_LIMIT,
+        GAS_USED,
+        LOGS_BLOOM,
+        PARENT_HASH,
+        RECEIPTS_ROOT,
+        SEALER,
+        SEALER_LIST,
+        SIGNATURE_LIST,
+        STATE_ROOT,
+        TRANSACTION_LIST,
+        TRANSACTIONS_ROOT;
+
+        BlockRawDataParams() { }
+    }
+
+    @Getter
+    public enum TxReceiptRawDataParams {
+        FROM,
+        GAS_USED,
+        LOGS,
+        INPUT,
+        MESSAGE,
+        OUTPUT,
+        LOGS_BLOOM,
+        ROOT,
+        TO,
+        TX_INDEX,
+        TX_PROOF,
+        RECEIPT_PROOF;
+
+        TxReceiptRawDataParams() { }
+    }
+
+    @Getter
+    public enum TxRawDataParams {
+        FROM,
+        GAS,
+        GAS_PRICE,
+        INPUT,
+        NONCE,
+        VALUE,
+        TO;
+
+
+        TxRawDataParams() { }
+    }
+
+}

WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/BlockCrawlerHandler.java

@@ -16,10 +16,16 @@
 import com.webank.blockchain.data.export.common.bo.data.BlockDetailInfoBO;
 import com.webank.blockchain.data.export.common.bo.data.BlockDetailInfoBO.Status;
 import com.webank.blockchain.data.export.common.bo.data.BlockRawDataBO;
+import com.webank.blockchain.data.export.common.entity.ExportConfig;
+import com.webank.blockchain.data.export.common.entity.ExportConstant;
+import com.webank.blockchain.data.export.common.enums.IgnoreBasicDataParam;
 import com.webank.blockchain.data.export.common.tools.DateUtils;
 import com.webank.blockchain.data.export.common.tools.JacksonUtils;
 import org.fisco.bcos.sdk.client.protocol.response.BcosBlock.Block;
 
+import java.util.List;
+import java.util.Map;
+
 /**
  * BlockCrawlerHandler is responsible for crawling block info.
  *
@@ -36,7 +42,6 @@ public class BlockCrawlerHandler {
      * transaction's count in current block, block hash and block timestamp.
      * 
      * @param block
-     * @param blockHeight
      * @return boolean
      */
     public static BlockDetailInfoBO handleBlockDetail(Block block) {
@@ -50,24 +55,68 @@ public static BlockDetailInfoBO handleBlockDetail(Block block) {
     }
 
     public static BlockRawDataBO handleBlockRawData(Block block) {
+        ExportConfig config = ExportConstant.getCurrentContext().getConfig();
+        Map<String, List<String>> ignoreBasicDataTableParam = config.getIgnoreBasicDataTableParam();
         BlockRawDataBO blockRawDataBO = new BlockRawDataBO();
         blockRawDataBO.setBlockHeight(block.getNumber().longValue());
         blockRawDataBO.setBlockHash(block.getHash());
         blockRawDataBO.setBlockTimeStamp(DateUtils.hexStrToDate(block.getTimestamp()));
-        blockRawDataBO.setDbHash(block.getDbHash());
-        blockRawDataBO.setExtraData(JacksonUtils.toJson(block.getExtraData()));
-        blockRawDataBO.setGasLimit(block.getGasLimit());
-        blockRawDataBO.setGasUsed(block.getGasUsed());
-        blockRawDataBO.setLogsBloom(block.getLogsBloom());
-        blockRawDataBO.setParentHash(block.getParentHash());
-        blockRawDataBO.setReceiptsRoot(block.getReceiptsRoot());
-        blockRawDataBO.setSealer(block.getSealer());
-        blockRawDataBO.setSealerList(JacksonUtils.toJson(block.getSealerList()));
-        blockRawDataBO.setSignatureList(JacksonUtils.toJson(block.getSignatureList()));
-        blockRawDataBO.setStateRoot(block.getStateRoot());
-        blockRawDataBO.setTransactionsRoot(block.getTransactionsRoot());
-        blockRawDataBO.setTransactionList(JacksonUtils.toJson(block.getTransactions()));
-        blockRawDataBO.setBlockObject(JacksonUtils.toJson(block));
+        if (!ignoreBasicDataTableParam.containsKey(IgnoreBasicDataParam.IgnoreBasicDataTable.BLOCK_RAW_DATA_TABLE.name())) {
+            blockRawDataBO.setDbHash(block.getDbHash());
+            blockRawDataBO.setExtraData(JacksonUtils.toJson(block.getExtraData()));
+            blockRawDataBO.setGasLimit(block.getGasLimit());
+            blockRawDataBO.setGasUsed(block.getGasUsed());
+            blockRawDataBO.setLogsBloom(block.getLogsBloom());
+            blockRawDataBO.setParentHash(block.getParentHash());
+            blockRawDataBO.setReceiptsRoot(block.getReceiptsRoot());
+            blockRawDataBO.setSealer(block.getSealer());
+            blockRawDataBO.setSealerList(JacksonUtils.toJson(block.getSealerList()));
+            blockRawDataBO.setSignatureList(JacksonUtils.toJson(block.getSignatureList()));
+            blockRawDataBO.setStateRoot(block.getStateRoot());
+            blockRawDataBO.setTransactionsRoot(block.getTransactionsRoot());
+            blockRawDataBO.setTransactionList(JacksonUtils.toJson(block.getTransactions()));
+        }else {
+            List<String> params = ignoreBasicDataTableParam.get(IgnoreBasicDataParam.IgnoreBasicDataTable.BLOCK_RAW_DATA_TABLE.name());
+            if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.DB_HASH.name())) {
+                blockRawDataBO.setDbHash(block.getDbHash());
+            }
+            if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.EXTRA_DATA.name())) {
+                blockRawDataBO.setExtraData(JacksonUtils.toJson(block.getExtraData()));
+            }
+            if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.GAS_LIMIT.name())) {
+                blockRawDataBO.setGasLimit(block.getGasLimit());
+            }
+            if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.GAS_USED.name())) {
+                blockRawDataBO.setGasUsed(block.getGasUsed());
+            }
+            if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.LOGS_BLOOM.name())) {
+                blockRawDataBO.setLogsBloom(block.getLogsBloom());
+            }
+            if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.PARENT_HASH.name())) {
+                blockRawDataBO.setParentHash(block.getParentHash());
+            }
+            if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.RECEIPTS_ROOT.name())) {
+                blockRawDataBO.setReceiptsRoot(block.getReceiptsRoot());
+            }
+            if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.SEALER.name())) {
+                blockRawDataBO.setSealer(block.getSealer());
+            }
+            if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.SEALER_LIST.name())) {
+                blockRawDataBO.setSealerList(JacksonUtils.toJson(block.getSealerList()));
+            }
+            if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.SEALER_LIST.name())) {
+                blockRawDataBO.setSignatureList(JacksonUtils.toJson(block.getSignatureList()));
+            }
+            if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.STATE_ROOT.name())) {
+                blockRawDataBO.setStateRoot(block.getStateRoot());
+            }
+            if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.TRANSACTIONS_ROOT.name())) {
+                blockRawDataBO.setTransactionsRoot(block.getTransactionsRoot());
+            }
+            if (!params.contains(IgnoreBasicDataParam.BlockRawDataParams.TRANSACTION_LIST.name())) {
+                blockRawDataBO.setTransactionList(JacksonUtils.toJson(block.getTransactions()));
+            }
+        }
         return blockRawDataBO;
     }
 }

WeBankBlockchain-Data-Export-sdk/WeBankBlockchain-Data-Export-parser/src/main/java/com/webank/blockchain/data/export/parser/handler/EventCrawlerHandler.java

@@ -56,10 +56,10 @@
 @Slf4j
 public class EventCrawlerHandler {
 
-
     @SuppressWarnings({ "rawtypes", "unused" })
     public static List<EventBO> crawl(Block block, Map<String, String> txHashContractNameMapping) throws IOException {
         List<TransactionResult> transactionResults = block.getTransactions();
+        List<EventBO> boList = new ArrayList<>();
         for (TransactionResult result : transactionResults) {
             TransactionObject to = (TransactionObject) result;
             JsonTransactionResponse transaction = to.get();
@@ -81,10 +81,10 @@ public static List<EventBO> crawl(Block block, Map<String, String> txHashContrac
                 if (abi == null) {
                     continue;
                 }
-                return parserEvent(contractAbiMap, contractName.get(), abi, tr,block);
+                boList.addAll(parserEvent(contractAbiMap, contractName.get(), abi, tr,block));
             }
         }
-        return ListUtil.empty();
+        return boList;
     }
 
     private static List<EventBO> parserEvent(Map<String, ContractInfo> contractAbiMap, String contractName, String abi,