Merge pull request #5556 from tronprotocol/master

merge master into develop

Author: forfreeday

actuator/build.gradle

@@ -52,7 +52,7 @@ jacocoTestReport {
         xml.enabled = true
         html.enabled = true
     }
-    executionData.from = '../framework/build/jacoco/jacocoTest.exec'
+    getExecutionData().setFrom(fileTree('../framework/build/jacoco').include("**.exec"))
     afterEvaluate {
         classDirectories.from = classDirectories.files.collect {
             fileTree(dir: it,)

actuator/src/main/java/org/tron/core/actuator/DelegateResourceActuator.java

@@ -189,7 +189,7 @@ public boolean validate() throws ContractValidateException {
 
     byte[] receiverAddress = delegateResourceContract.getReceiverAddress().toByteArray();
 
-    if (ArrayUtils.isEmpty(receiverAddress) || !DecodeUtil.addressValid(receiverAddress)) {
+    if (!DecodeUtil.addressValid(receiverAddress)) {
       throw new ContractValidateException("Invalid receiverAddress");
     }
 

actuator/src/main/java/org/tron/core/vm/nativecontract/DelegateResourceProcessor.java

@@ -93,7 +93,7 @@ public void validate(DelegateResourceParam param, Repository repo) throws Contra
 
     byte[] receiverAddress = param.getReceiverAddress();
 
-    if (ArrayUtils.isEmpty(receiverAddress) || !DecodeUtil.addressValid(receiverAddress)) {
+    if (!DecodeUtil.addressValid(receiverAddress)) {
       throw new ContractValidateException("Invalid receiverAddress");
     }
     if (Arrays.equals(receiverAddress, ownerAddress)) {

chainbase/build.gradle

@@ -80,7 +80,7 @@ jacocoTestReport {
         xml.enabled = true
         html.enabled = true
     }
-    executionData.from = '../framework/build/jacoco/jacocoTest.exec'
+    getExecutionData().setFrom(fileTree('../framework/build/jacoco').include("**.exec"))
     afterEvaluate {
         classDirectories.from = classDirectories.files.collect {
             fileTree(dir: it,)

chainbase/src/main/java/org/tron/core/db/TransactionCache.java

@@ -3,16 +3,20 @@
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
 import org.tron.core.capsule.BytesCapsule;
 import org.tron.core.db2.common.TxCacheDB;
+import org.tron.core.store.DynamicPropertiesStore;
 
 @Slf4j
+@Component
 public class TransactionCache extends TronStoreWithRevoking<BytesCapsule> {
 
   @Autowired
   public TransactionCache(@Value("trans-cache") String dbName,
-                          RecentTransactionStore recentTransactionStore) {
-    super(new TxCacheDB(dbName, recentTransactionStore));
+                          @Autowired RecentTransactionStore recentTransactionStore,
+                          @Autowired DynamicPropertiesStore dynamicPropertiesStore) {
+    super(new TxCacheDB(dbName, recentTransactionStore, dynamicPropertiesStore));
   }
 
   public void initCache() {

chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java

@@ -2,9 +2,13 @@
 
 import com.google.common.hash.BloomFilter;
 import com.google.common.hash.Funnels;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hashing;
+import com.google.common.io.ByteSource;
 import com.google.common.primitives.Longs;
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
@@ -18,6 +22,7 @@
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Objects;
 import java.util.Properties;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -40,6 +45,7 @@
 import org.tron.core.db.RecentTransactionItem;
 import org.tron.core.db.RecentTransactionStore;
 import org.tron.core.db.common.iterator.DBIterator;
+import org.tron.core.store.DynamicPropertiesStore;
 
 @Slf4j(topic = "DB")
 public class TxCacheDB implements DB<byte[], byte[]>, Flusher {
@@ -59,7 +65,6 @@ public class TxCacheDB implements DB<byte[], byte[]>, Flusher {
   private BloomFilter<byte[]>[] bloomFilters = new BloomFilter[2];
   // filterStartBlock record the start block of the active filter
   private volatile long filterStartBlock = INVALID_BLOCK;
-  private volatile long currentBlockNum = INVALID_BLOCK;
   // currentFilterIndex records the index of the active filter
   private volatile int currentFilterIndex = 0;
 
@@ -75,21 +80,28 @@ public class TxCacheDB implements DB<byte[], byte[]>, Flusher {
   // replace persistentStore and optimizes startup performance
   private RecentTransactionStore recentTransactionStore;
 
+  private DynamicPropertiesStore dynamicPropertiesStore;
+
   private final Path cacheFile0;
   private final Path cacheFile1;
+  private String crc32c0;
+  private String crc32c1;
   private final Path cacheProperties;
   private final Path cacheDir;
   private AtomicBoolean isValid = new AtomicBoolean(false);
+  private boolean txCacheInitOptimization;
 
   @Getter
   @Setter
   private volatile boolean alive;
 
-  public TxCacheDB(String name, RecentTransactionStore recentTransactionStore) {
+  public TxCacheDB(String name, RecentTransactionStore recentTransactionStore,
+                   DynamicPropertiesStore dynamicPropertiesStore) {
     this.name = name;
     this.TRANSACTION_COUNT =
         CommonParameter.getInstance().getStorage().getEstimatedBlockTransactions();
     this.recentTransactionStore = recentTransactionStore;
+    this.dynamicPropertiesStore = dynamicPropertiesStore;
     String dbEngine = CommonParameter.getInstance().getStorage().getDbEngine();
     if ("LEVELDB".equals(dbEngine.toUpperCase())) {
       this.persistentStore = new LevelDB(
@@ -117,6 +129,8 @@ public TxCacheDB(String name, RecentTransactionStore recentTransactionStore) {
     this.cacheFile0 = Paths.get(cacheDir.toString(), "bloomFilters_0");
     this.cacheFile1 = Paths.get(cacheDir.toString(), "bloomFilters_1");
     this.cacheProperties = Paths.get(cacheDir.toString(), "txCache.properties");
+    this.txCacheInitOptimization = CommonParameter.getInstance()
+        .getStorage().isTxCacheInitOptimization();
 
   }
 
@@ -211,7 +225,6 @@ public void put(byte[] key, byte[] value) {
               MAX_BLOCK_SIZE * TRANSACTION_COUNT);
     }
     bloomFilters[currentFilterIndex].put(key);
-    currentBlockNum = blockNum;
     if (lastMetricBlock != blockNum) {
       lastMetricBlock = blockNum;
       Metrics.gaugeSet(MetricKeys.Gauge.TX_CACHE,
@@ -270,6 +283,12 @@ public void reset() {
   }
 
   private boolean recovery() {
+    if (!txCacheInitOptimization) {
+      logger.info("txCache init optimization is disabled, skip fast recovery mode.");
+      logger.info("If you want fast recovery mode,"
+          + " please set `storage.txCache.initOptimization = true` in config.conf.");
+      return false;
+    }
     FileUtil.createDirIfNotExists(this.cacheDir.toString());
     logger.info("recovery bloomFilters start.");
     CompletableFuture<Boolean> loadProperties = CompletableFuture.supplyAsync(this::loadProperties);
@@ -278,13 +297,18 @@ private boolean recovery() {
     CompletableFuture<Boolean> tk1 = loadProperties.thenApplyAsync(
         v -> recovery(1, this.cacheFile1));
 
-    return CompletableFuture.allOf(tk0, tk1).thenApply(v -> {
-      logger.info("recovery bloomFilters success.");
-      return true;
-    }).exceptionally(this::handleException).join();
+    try {
+      return CompletableFuture.allOf(tk0, tk1).thenApply(v -> {
+        logger.info("recovery bloomFilters success.");
+        return true;
+      }).exceptionally(this::handleException).join();
+    } finally {
+      clearCrc32c();
+    }
   }
 
   private boolean recovery(int index, Path file) {
+    checkCrc32c(index, file);
     try (InputStream in = new BufferedInputStream(Files.newInputStream(file,
         StandardOpenOption.READ, StandardOpenOption.DELETE_ON_CLOSE))) {
       logger.info("recovery bloomFilter[{}] from file.", index);
@@ -326,24 +350,38 @@ private void dump() {
         () -> dump(0, this.cacheFile0));
     CompletableFuture<Void> task1 = CompletableFuture.runAsync(
         () -> dump(1, this.cacheFile1));
-    CompletableFuture.allOf(task0, task1).thenRun(() -> {
-      writeProperties();
-      logger.info("dump bloomFilters done.");
-
-    }).exceptionally(e -> {
-      logger.info("dump bloomFilters to file failed. {}", e.getMessage());
-      return null;
-    }).join();
+    try {
+      CompletableFuture.allOf(task0, task1).thenRun(() -> {
+        writeProperties();
+        logger.info("dump bloomFilters done.");
+      }).exceptionally(e -> {
+        logger.info("dump bloomFilters to file failed. {}", e.getMessage());
+        return null;
+      }).join();
+    } finally {
+      clearCrc32c();
+    }
   }
 
   private void dump(int index, Path file) {
+    logger.info("dump bloomFilters[{}] to file.", index);
+    long start = System.currentTimeMillis();
     try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(file))) {
-      logger.info("dump bloomFilters[{}] to file.", index);
-      long start = System.currentTimeMillis();
       bloomFilters[index].writeTo(out);
-      logger.info("dump bloomFilters[{}] to file done,filter: {}, filter-fpp: {}, cost {} ms.",
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+    try {
+      String crc32c = getCrc32c(file);
+      if (index == 0) {
+        this.crc32c0 = crc32c;
+      } else {
+        this.crc32c1 = crc32c;
+      }
+      logger.info("dump bloomFilters[{}] to file done,filter: {}, filter-fpp: {}, "
+              + "crc32c: {}, cost {} ms.",
           index, bloomFilters[index].approximateElementCount(), bloomFilters[index].expectedFpp(),
-          System.currentTimeMillis() - start);
+          crc32c, System.currentTimeMillis() - start);
     } catch (Exception e) {
       throw new RuntimeException(e);
     }
@@ -356,8 +394,16 @@ private boolean loadProperties() {
       Properties properties = new Properties();
       properties.load(r);
       filterStartBlock = Long.parseLong(properties.getProperty("filterStartBlock"));
-      currentBlockNum = Long.parseLong(properties.getProperty("currentBlockNum"));
+      long currentBlockNum = Long.parseLong(properties.getProperty("currentBlockNum"));
+      long currentBlockNumFromDB = dynamicPropertiesStore.getLatestBlockHeaderNumberFromDB();
       currentFilterIndex = Integer.parseInt(properties.getProperty("currentFilterIndex"));
+      if (currentBlockNum != currentBlockNumFromDB) {
+        throw new IllegalStateException(
+            String.format("currentBlockNum not match. filter: %d, db: %d",
+            currentBlockNum, currentBlockNumFromDB));
+      }
+      this.crc32c0 = properties.getProperty("crc32c0");
+      this.crc32c1 = properties.getProperty("crc32c1");
       logger.info("filterStartBlock: {}, currentBlockNum: {}, currentFilterIndex: {}, load done.",
           filterStartBlock, currentBlockNum, currentFilterIndex);
       return true;
@@ -369,9 +415,12 @@ private boolean loadProperties() {
   private void writeProperties() {
     try (Writer w = Files.newBufferedWriter(this.cacheProperties, StandardCharsets.UTF_8)) {
       Properties properties = new Properties();
+      long currentBlockNum = dynamicPropertiesStore.getLatestBlockHeaderNumberFromDB();
       properties.setProperty("filterStartBlock", String.valueOf(filterStartBlock));
       properties.setProperty("currentBlockNum", String.valueOf(currentBlockNum));
       properties.setProperty("currentFilterIndex", String.valueOf(currentFilterIndex));
+      properties.setProperty("crc32c0", this.crc32c0);
+      properties.setProperty("crc32c1", this.crc32c1);
       properties.store(w, "Generated by the application.  PLEASE DO NOT EDIT! ");
       logger.info("filterStartBlock: {}, currentBlockNum: {}, currentFilterIndex: {}, write done.",
           filterStartBlock, currentBlockNum, currentFilterIndex);
@@ -380,9 +429,33 @@ private void writeProperties() {
     }
   }
 
+  private String getCrc32c(Path file) throws IOException {
+    ByteSource byteSource = com.google.common.io.Files.asByteSource(file.toFile());
+    HashCode hc = byteSource.hash(Hashing.crc32c());
+    return hc.toString();
+  }
+
+  private void checkCrc32c(int index, Path file) {
+    try {
+      String actual = getCrc32c(file);
+      String expect = index == 0 ? this.crc32c0 : this.crc32c1;
+      if (!Objects.equals(actual, expect)) {
+        throw new IllegalStateException("crc32c not match. index: " + index + ", expect: " + expect
+            + ", actual: " + actual);
+      }
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void clearCrc32c() {
+    this.crc32c0 = null;
+    this.crc32c1 = null;
+  }
+
   @Override
   public TxCacheDB newInstance() {
-    return new TxCacheDB(name, recentTransactionStore);
+    return new TxCacheDB(name, recentTransactionStore, dynamicPropertiesStore);
   }
 
   @Override

chainbase/src/main/java/org/tron/core/store/CheckPointV2Store.java

@@ -1,6 +1,7 @@
 package org.tron.core.store;
 
 import com.google.protobuf.InvalidProtocolBufferException;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.tron.core.db.TronDatabase;
 import org.tron.core.exception.BadItemException;
@@ -9,6 +10,7 @@
 import java.util.Spliterator;
 import java.util.function.Consumer;
 
+@Slf4j(topic = "DB")
 public class CheckPointV2Store extends TronDatabase<byte[]> {
 
   @Autowired
@@ -50,4 +52,19 @@ public Spliterator spliterator() {
   protected void init() {
   }
 
+  /**
+   * close the database.
+   */
+  @Override
+  public void close() {
+    logger.debug("******** Begin to close {}. ********", getName());
+    try {
+      dbSource.closeDB();
+    } catch (Exception e) {
+      logger.warn("Failed to close {}.", getName(), e);
+    } finally {
+      logger.debug("******** End to close {}. ********", getName());
+    }
+  }
+
 }

common/build.gradle

@@ -53,7 +53,7 @@ dependencies {
     compile 'org.aspectj:aspectjrt:1.8.13'
     compile 'org.aspectj:aspectjweaver:1.8.13'
     compile 'org.aspectj:aspectjtools:1.8.13'
-    compile group: 'com.github.tronprotocol', name: 'libp2p', version: 'test-v2.0.2',{
+    compile group: 'io.github.tronprotocol', name: 'libp2p', version: '2.1.0',{
         exclude group: 'io.grpc', module: 'grpc-context'
         exclude group: 'io.grpc', module: 'grpc-core'
         exclude group: 'io.grpc', module: 'grpc-netty'
@@ -68,7 +68,7 @@ jacocoTestReport {
         xml.enabled = true
         html.enabled = true
     }
-    executionData.from = '../framework/build/jacoco/jacocoTest.exec'
+    getExecutionData().setFrom(fileTree('../framework/build/jacoco').include("**.exec"))
     afterEvaluate {
         classDirectories.from = classDirectories.files.collect {
             fileTree(dir: it,)

common/src/main/java/org/tron/common/parameter/CommonParameter.java

@@ -415,6 +415,8 @@ public class CommonParameter {
   @Setter
   public int rateLimiterGlobalIpQps;
   @Getter
+  public int rateLimiterGlobalApiQps;
+  @Getter
   public DbBackupConfig dbBackupConfig;
   @Getter
   public RocksDbSettings rocksDBCustomSettings;
@@ -516,6 +518,9 @@ public class CommonParameter {
   public int pBFTHttpPort;
   @Getter
   @Setter
+  public long pBFTExpireNum;
+  @Getter
+  @Setter
   public long oldSolidityBlockNum = -1;
 
   @Getter/**/

common/src/main/java/org/tron/core/Constant.java

@@ -256,6 +256,8 @@ public class Constant {
 
   public static final String RATE_LIMITER_GLOBAL_IP_QPS = "rate.limiter.global.ip.qps";
 
+  public static final String RATE_LIMITER_GLOBAL_API_QPS = "rate.limiter.global.api.qps";
+
   public static final String COMMITTEE_CHANGED_DELEGATION = "committee.changedDelegation";
 
   public static final String CRYPTO_ENGINE = "crypto.engine";
@@ -301,6 +303,7 @@ public class Constant {
   public static final String SEED_NODE_IP_LIST = "seed.node.ip.list";
   public static final String NODE_METRICS_ENABLE = "node.metricsEnable";
   public static final String COMMITTEE_ALLOW_PBFT = "committee.allowPBFT";
+  public static final String COMMITTEE_PBFT_EXPIRE_NUM = "committee.pBFTExpireNum";
   public static final String NODE_AGREE_NODE_COUNT = "node.agreeNodeCount";
 
   public static final String COMMITTEE_ALLOW_TRANSACTION_FEE_POOL = "committee.allowTransactionFeePool";