Merge master into develop branch (#5668)

lurais · web-flow · commit 0f5a8defc65f · 2024-01-12T19:48:27.000+08:00
* feat(api):fix a concurrency issue for toString in BlockCapsule (#5657) * feat(db):optimize cache settings (#5659) * feat(lite):optimize DbLite tool (#5658)
diff --git a/chainbase/src/main/java/org/tron/core/store/WitnessStore.java b/chainbase/src/main/java/org/tron/core/store/WitnessStore.java
@@ -11,25 +11,17 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
-import org.tron.common.cache.CacheManager;
-import org.tron.common.cache.CacheStrategies;
-import org.tron.common.cache.CacheType;
-import org.tron.common.cache.TronCache;
 import org.tron.core.capsule.WitnessCapsule;
 import org.tron.core.config.Parameter;
 import org.tron.core.db.TronStoreWithRevoking;
 
 @Slf4j(topic = "DB")
 @Component
 public class WitnessStore extends TronStoreWithRevoking<WitnessCapsule> {
- // cache for 127 SR
-  private final TronCache<Integer, List<WitnessCapsule>> witnessStandbyCache;
 
   @Autowired
   protected WitnessStore(@Value("witness") String dbName) {
     super(dbName);
-    String strategy =  String.format(CacheStrategies.PATTERNS, 1, 1, "30s", 1);
-    witnessStandbyCache = CacheManager.allocate(CacheType.witnessStandby, strategy);
   }
 
   /**
@@ -48,19 +40,8 @@ public WitnessCapsule get(byte[] key) {
   }
 
   public List<WitnessCapsule> getWitnessStandby() {
-    List<WitnessCapsule> list =
-        witnessStandbyCache.getIfPresent(Parameter.ChainConstant.WITNESS_STANDBY_LENGTH);
-    if (list != null) {
-      return list;
-    }
-    return updateWitnessStandby(null);
-  }
-
-  public List<WitnessCapsule> updateWitnessStandby(List<WitnessCapsule> all) {
     List<WitnessCapsule> ret;
-    if (all == null) {
-      all = getAllWitnesses();
-    }
+    List<WitnessCapsule> all = getAllWitnesses();
     all.sort(Comparator.comparingLong(WitnessCapsule::getVoteCount)
         .reversed().thenComparing(Comparator.comparingInt(
             (WitnessCapsule w) -> w.getAddress().hashCode()).reversed()));
@@ -71,7 +52,6 @@ public List<WitnessCapsule> updateWitnessStandby(List<WitnessCapsule> all) {
     }
     // trim voteCount = 0
     ret.removeIf(w -> w.getVoteCount() < 1);
-    witnessStandbyCache.put(Parameter.ChainConstant.WITNESS_STANDBY_LENGTH, ret);
     return ret;
   }
 
diff --git a/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java b/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java
@@ -108,10 +108,6 @@ public List<WitnessCapsule> getAllWitnesses() {
     return witnessStore.getAllWitnesses();
   }
 
-  public List<WitnessCapsule> updateWitnessStandby(List<WitnessCapsule> all) {
-    return witnessStore.updateWitnessStandby(all);
-  }
-
   public void saveStateFlag(int flag) {
     dynamicPropertiesStore.saveStateFlag(flag);
   }
diff --git a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java
@@ -151,13 +151,11 @@ public void doMaintenance() {
     if (dynamicPropertiesStore.allowChangeDelegation()) {
       long nextCycle = dynamicPropertiesStore.getCurrentCycleNumber() + 1;
       dynamicPropertiesStore.saveCurrentCycleNumber(nextCycle);
-      List<WitnessCapsule> all = consensusDelegate.getAllWitnesses();
-      all.forEach(witness -> {
+      consensusDelegate.getAllWitnesses().forEach(witness -> {
         delegationStore.setBrokerage(nextCycle, witness.createDbKey(),
             delegationStore.getBrokerage(witness.createDbKey()));
         delegationStore.setWitnessVote(nextCycle, witness.createDbKey(), witness.getVoteCount());
       });
-      consensusDelegate.updateWitnessStandby(all);
     }
   }
 
diff --git a/framework/src/main/java/org/tron/program/Version.java b/framework/src/main/java/org/tron/program/Version.java
@@ -2,9 +2,9 @@
 
 public class Version {
 
-  public static final String VERSION_NAME = "GreatVoyage-v4.7.2-140-g9d13f9cb69";
-  public static final String VERSION_CODE = "18173";
-  private static final String VERSION = "4.7.3";
+  public static final String VERSION_NAME = "GreatVoyage-v4.7.3-5-g788136ebe";
+  public static final String VERSION_CODE = "18180";
+  private static final String VERSION = "4.7.3.1";
 
   public static String getVersion() {
     return VERSION;
diff --git a/framework/src/test/java/org/tron/core/db/ManagerTest.java b/framework/src/test/java/org/tron/core/db/ManagerTest.java
@@ -581,14 +581,24 @@ public void pushSwitchFork()
       AccountResourceInsufficientException, EventBloomException {
 
     String key = "f31db24bfbd1a2ef19beddca0a0fa37632eded9ac666a05d3bd925f01dde1f62";
+    String key2 = "c85ef7d79691fe79573b1a7064c19c1a9819ebdbd1faaab1a8ec92344438aaf4";
     byte[] privateKey = ByteArray.fromHexString(key);
     final ECKey ecKey = ECKey.fromPrivate(privateKey);
     byte[] address = ecKey.getAddress();
-
+    WitnessCapsule sr1 = new WitnessCapsule(
+        ByteString.copyFrom(address), "www.tron.net/first");
+    sr1.setVoteCount(1000000000L);
+    byte[] privateKey2 = ByteArray.fromHexString(key2);
+    final ECKey ecKey2 = ECKey.fromPrivate(privateKey2);
+    byte[] address2 = ecKey2.getAddress();
+    WitnessCapsule sr2 = new WitnessCapsule(
+        ByteString.copyFrom(address2), "www.tron.net/second");
+    sr2.setVoteCount(100000L);
+    chainManager.getWitnessStore().put(address, sr1);
     WitnessCapsule witnessCapsule = new WitnessCapsule(ByteString.copyFrom(address));
     chainManager.getWitnessScheduleStore().saveActiveWitnesses(new ArrayList<>());
     chainManager.addWitness(ByteString.copyFrom(address));
-
+    List<WitnessCapsule> witnessStandby1 = chainManager.getWitnessStore().getWitnessStandby();
     Block block = getSignedBlock(witnessCapsule.getAddress(), 1533529947843L, privateKey);
     dbManager.pushBlock(new BlockCapsule(block));
 
@@ -625,6 +635,9 @@ public void pushSwitchFork()
     } catch (Exception e) {
       Assert.assertTrue(e instanceof Exception);
     }
+    chainManager.getWitnessStore().put(address, sr2);
+    List<WitnessCapsule> witnessStandby2 = chainManager.getWitnessStore().getWitnessStandby();
+    Assert.assertNotEquals(witnessStandby1, witnessStandby2);
   }
 
 
diff --git a/plugins/src/main/java/org/tron/plugins/DbLite.java b/plugins/src/main/java/org/tron/plugins/DbLite.java
@@ -3,6 +3,8 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.google.common.primitives.Bytes;
+import com.google.common.primitives.Ints;
 import com.google.common.primitives.Longs;
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -153,10 +155,10 @@ public void generateSnapshot(String sourceDir, String snapshotDir) {
     long start = System.currentTimeMillis();
     snapshotDir = Paths.get(snapshotDir, SNAPSHOT_DIR_NAME).toString();
     try {
-      mergeCheckpoint(sourceDir);
       hasEnoughBlock(sourceDir);
       List<String> snapshotDbs = getSnapshotDbs(sourceDir);
       split(sourceDir, snapshotDir, snapshotDbs);
+      mergeCheckpoint2Snapshot(sourceDir, snapshotDir);
       // write genesisBlock , latest recent blocks and trans
       fillSnapshotBlockAndTransDb(sourceDir, snapshotDir);
       // save min block to info
@@ -190,9 +192,9 @@ public void generateHistory(String sourceDir, String historyDir) {
         throw new IllegalStateException(
             String.format("Unavailable sourceDir: %s is not fullNode data.", sourceDir));
       }
-      mergeCheckpoint(sourceDir);
       hasEnoughBlock(sourceDir);
       split(sourceDir, historyDir, archiveDbs);
+      mergeCheckpoint2History(sourceDir, historyDir);
       // save max block to info
       generateInfoProperties(Paths.get(historyDir, INFO_FILE_NAME).toString(),
           getLatestBlockHeaderNum(sourceDir));
@@ -261,6 +263,15 @@ private List<String> getSnapshotDbs(String sourceDir) {
     return snapshotDbs;
   }
 
+  private void mergeCheckpoint2Snapshot(String sourceDir, String historyDir) {
+    List<String> snapshotDbs = getSnapshotDbs(sourceDir);
+    mergeCheckpoint(sourceDir, historyDir, snapshotDbs);
+  }
+
+  private void mergeCheckpoint2History(String sourceDir, String destDir) {
+    mergeCheckpoint(sourceDir, destDir, archiveDbs);
+  }
+
   private void split(String sourceDir, String destDir, List<String> dbs) throws IOException {
     logger.info("Begin to split the dbs.");
     spec.commandLine().getOut().println("Begin to split the dbs.");
@@ -278,7 +289,7 @@ private void split(String sourceDir, String destDir, List<String> dbs) throws IO
     FileUtils.copyDatabases(Paths.get(sourceDir), Paths.get(destDir), dbs);
   }
 
-  private void mergeCheckpoint(String sourceDir) {
+  private void mergeCheckpoint(String sourceDir, String destDir, List<String> destDbs) {
     logger.info("Begin to merge checkpoint to dataset.");
     spec.commandLine().getOut().println("Begin to merge checkpoint to dataset.");
     try {
@@ -287,18 +298,18 @@ private void mergeCheckpoint(String sourceDir) {
         for (String cp : cpList) {
           DBInterface checkpointDb = DbTool.getDB(
               sourceDir + "/" + DBUtils.CHECKPOINT_DB_V2, cp);
-          recover(checkpointDb, sourceDir);
+          recover(checkpointDb, destDir, destDbs);
         }
       } else if (Paths.get(sourceDir, CHECKPOINT_DB).toFile().exists()) {
         DBInterface tmpDb = DbTool.getDB(sourceDir, CHECKPOINT_DB);
-        recover(tmpDb, sourceDir);
+        recover(tmpDb, destDir, destDbs);
       }
     } catch (IOException | RocksDBException e) {
       throw new RuntimeException(e);
     }
   }
 
-  private void recover(DBInterface db, String destDir)
+  private void recover(DBInterface db, String destDir, List<String> destDbs)
       throws IOException, RocksDBException {
     try (DBIterator iterator = db.iterator()) {
       for (iterator.seekToFirst(); iterator.hasNext(); iterator.next()) {
@@ -312,15 +323,17 @@ private void recover(DBInterface db, String destDir)
         byte[] realKey = Arrays.copyOfRange(key, dbName.getBytes().length + 4, key.length);
         byte[] realValue =
             value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length);
-        DBInterface destDb = DbTool.getDB(destDir, dbName);
-        if (realValue != null) {
-          destDb.put(realKey, realValue);
-        } else {
-          byte op = value[0];
-          if (DBUtils.Operator.DELETE.getValue() == op) {
-            destDb.delete(realKey);
+        if (destDbs != null && destDbs.contains(dbName)) {
+          DBInterface destDb = DbTool.getDB(destDir, dbName);
+          if (realValue != null) {
+            destDb.put(realKey, realValue);
           } else {
-            destDb.put(realKey, new byte[0]);
+            byte op = value[0];
+            if (DBUtils.Operator.DELETE.getValue() == op) {
+              destDb.delete(realKey);
+            } else {
+              destDb.put(realKey, new byte[0]);
+            }
           }
         }
       }
@@ -340,14 +353,30 @@ private void generateInfoProperties(String propertyfile, long num)
   }
 
   private long getLatestBlockHeaderNum(String databaseDir) throws IOException, RocksDBException {
+    // query latest_block_header_number from checkpoint first
     final String latestBlockHeaderNumber = "latest_block_header_number";
+    DBInterface checkpointDb = getCheckpointDb(databaseDir);
+    Long blockNumber = getLatestBlockHeaderNumFromCP(checkpointDb,
+        latestBlockHeaderNumber.getBytes());
+    if (blockNumber != null) {
+      return blockNumber;
+    }
+    // query from propertiesDb if checkpoint not contains latest_block_header_number
     DBInterface propertiesDb = DbTool.getDB(databaseDir, PROPERTIES_DB_NAME);
     return Optional.ofNullable(propertiesDb.get(ByteArray.fromString(latestBlockHeaderNumber)))
             .map(ByteArray::toLong)
             .orElseThrow(
                 () -> new IllegalArgumentException("not found latest block header number"));
   }
 
+  private Long getLatestBlockHeaderNumFromCP(DBInterface db, byte[] key) {
+    byte[] value = db.get(Bytes.concat(simpleEncode(PROPERTIES_DB_NAME), key));
+    if (value != null && value.length > 1) {
+      return ByteArray.toLong(Arrays.copyOfRange(value, 1, value.length));
+    }
+    return null;
+  }
+
   /**
    * recent blocks, trans and genesis block.
    */
@@ -414,6 +443,15 @@ private byte[] getGenesisBlockHash(String parentDir) throws IOException, RocksDB
     return result;
   }
 
+  private static byte[] simpleEncode(String s) {
+    byte[] bytes = s.getBytes();
+    byte[] length = Ints.toByteArray(bytes.length);
+    byte[] r = new byte[4 + bytes.length];
+    System.arraycopy(length, 0, r, 0, 4);
+    System.arraycopy(bytes, 0, r, 4, bytes.length);
+    return r;
+  }
+
   private BlockNumInfo checkAndGetBlockNumInfo(String historyDir, String liteDir)
           throws IOException, RocksDBException {
     logger.info("Check the compatibility of this history.");
@@ -485,6 +523,7 @@ private void trimExtraHistory(String liteDir, BlockNumInfo blockNumInfo)
     DBInterface transDb = DbTool.getDB(liteDir, TRANS_DB_NAME);
     DBInterface tranRetDb = DbTool.getDB(liteDir, TRANSACTION_RET_DB_NAME);
 
+
     ProgressBar.wrap(LongStream.rangeClosed(start, end)
         .boxed()
         .sorted((a, b) -> Long.compare(b, a)), "trimHistory").forEach(n -> {
@@ -519,6 +558,7 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws
       return;
     }
 
+
     Path bakDir = Paths.get(liteDir, BACKUP_DIR_PREFIX + START_TIME);
     logger.info("Begin to merge {} to database, start {} end {}.", bakDir, start, end);
     spec.commandLine().getOut()
@@ -545,7 +585,17 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws
 
   private byte[] getDataFromSourceDB(String sourceDir, String dbName, byte[] key)
           throws IOException, RocksDBException {
-    byte[] value = DbTool.getDB(sourceDir, dbName).get(key);
+    DBInterface sourceDb = DbTool.getDB(sourceDir, dbName);
+    DBInterface checkpointDb = getCheckpointDb(sourceDir);
+    // get data from tmp first.
+    byte[] valueFromTmp = checkpointDb.get(Bytes.concat(simpleEncode(dbName), key));
+    byte[] value;
+    if (isEmptyBytes(valueFromTmp)) {
+      value = sourceDb.get(key);
+    } else {
+      value = valueFromTmp.length == 1
+          ? null : Arrays.copyOfRange(valueFromTmp, 1, valueFromTmp.length);
+    }
     if (isEmptyBytes(value)) {
       throw new RuntimeException(String.format("data not found in store, dbName: %s, key: %s",
               dbName, Arrays.toString(key)));
@@ -614,6 +664,19 @@ private long getSecondBlock(String databaseDir) throws RocksDBException, IOExcep
     return num;
   }
 
+  private DBInterface getCheckpointDb(String sourceDir) throws IOException, RocksDBException {
+    List<String> cpList = getCheckpointV2List(sourceDir);
+    DBInterface checkpointDb;
+    if (cpList.size() > 0) {
+      String latestCp = cpList.get(cpList.size() - 1);
+      checkpointDb = DbTool.getDB(
+          sourceDir + "/" + DBUtils.CHECKPOINT_DB_V2, latestCp);
+    } else {
+      checkpointDb = DbTool.getDB(sourceDir, CHECKPOINT_DB);
+    }
+    return checkpointDb;
+  }
+
   @VisibleForTesting
   public static void setRecentBlks(long recentBlks) {
     RECENT_BLKS = recentBlks;

Original file line number	Diff line number	Diff line change
`@@ -108,10 +108,6 @@ public List<WitnessCapsule> getAllWitnesses() {`
`108`	`108`	`return witnessStore.getAllWitnesses();`
`109`	`109`	`}`
`110`	`110`
`111`		`- public List<WitnessCapsule> updateWitnessStandby(List<WitnessCapsule> all) {`
`112`		`- return witnessStore.updateWitnessStandby(all);`
`113`		`- }`
`114`		`-`
`115`	`111`	`public void saveStateFlag(int flag) {`
`116`	`112`	`dynamicPropertiesStore.saveStateFlag(flag);`
`117`	`113`	`}`
Original file line number	Diff line number	Diff line change
`@@ -151,13 +151,11 @@ public void doMaintenance() {`
`151`	`151`	`if (dynamicPropertiesStore.allowChangeDelegation()) {`
`152`	`152`	`long nextCycle = dynamicPropertiesStore.getCurrentCycleNumber() + 1;`
`153`	`153`	`dynamicPropertiesStore.saveCurrentCycleNumber(nextCycle);`
`154`		`- List<WitnessCapsule> all = consensusDelegate.getAllWitnesses();`
`155`		`- all.forEach(witness -> {`
	`154`	`+ consensusDelegate.getAllWitnesses().forEach(witness -> {`
`156`	`155`	`delegationStore.setBrokerage(nextCycle, witness.createDbKey(),`
`157`	`156`	`delegationStore.getBrokerage(witness.createDbKey()));`
`158`	`157`	`delegationStore.setWitnessVote(nextCycle, witness.createDbKey(), witness.getVoteCount());`
`159`	`158`	`});`
`160`		`- consensusDelegate.updateWitnessStandby(all);`
`161`	`159`	`}`
`162`	`160`	`}`
`163`	`161`