Skip to content

Commit 909cd2b

Browse files
halibobo1205halibobo1205
committed
feat(db): update leveldb and rocksdb for arm
1 parent 0dd6259 commit 909cd2b

File tree

26 files changed

+482
-263
lines changed

26 files changed

+482
-263
lines changed

‎chainbase/src/main/java/org/tron/common/storage/OptionsPicker.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package org.tron.common.storage;
2+
3+
import org.tron.common.setting.RocksDbSettings;
4+
import org.tron.common.utils.StorageUtils;
5+
6+
public class OptionsPicker {
7+
8+
protected org.iq80.leveldb.Options getOptionsByDbNameForLevelDB(String dbName) {
9+
return StorageUtils.getOptionsByDbName(dbName);
10+
}
11+
12+
protected org.rocksdb.Options getOptionsByDbNameForRocksDB(String dbName) {
13+
return RocksDbSettings.getOptionsByDbName(dbName);
14+
}
15+
}

‎chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java‎

Lines changed: 33 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.tron.common.storage.rocksdb;
22

3+
import com.google.common.annotations.VisibleForTesting;
34
import com.google.common.collect.Sets;
45
import com.google.common.primitives.Bytes;
56
import java.io.File;
@@ -20,18 +21,14 @@
2021
import java.util.stream.Collectors;
2122
import lombok.NoArgsConstructor;
2223
import lombok.extern.slf4j.Slf4j;
23-
import org.rocksdb.BlockBasedTableConfig;
24-
import org.rocksdb.BloomFilter;
2524
import org.rocksdb.Checkpoint;
26-
import org.rocksdb.DirectComparator;
2725
import org.rocksdb.InfoLogLevel;
2826
import org.rocksdb.Logger;
2927
import org.rocksdb.Options;
3028
import org.rocksdb.ReadOptions;
3129
import org.rocksdb.RocksDB;
3230
import org.rocksdb.RocksDBException;
3331
import org.rocksdb.RocksIterator;
34-
import org.rocksdb.Statistics;
3532
import org.rocksdb.Status;
3633
import org.rocksdb.WriteBatch;
3734
import org.rocksdb.WriteOptions;
@@ -53,36 +50,28 @@
5350
public class RocksDbDataSourceImpl extends DbStat implements DbSourceInter<byte[]>,
5451
Iterable<Map.Entry<byte[], byte[]>>, Instance<RocksDbDataSourceImpl> {
5552

56-
ReadOptions readOpts;
5753
private String dataBaseName;
5854
private RocksDB database;
55+
private Options options;
5956
private volatile boolean alive;
6057
private String parentPath;
6158
private ReadWriteLock resetDbLock = new ReentrantReadWriteLock();
6259
private static final String KEY_ENGINE = "ENGINE";
6360
private static final String ROCKSDB = "ROCKSDB";
64-
private DirectComparator comparator;
6561
private static final org.slf4j.Logger rocksDbLogger = LoggerFactory.getLogger(ROCKSDB);
6662

67-
public RocksDbDataSourceImpl(String parentPath, String name, RocksDbSettings settings,
68-
DirectComparator comparator) {
63+
public RocksDbDataSourceImpl(String parentPath, String name, Options options) {
6964
this.dataBaseName = name;
7065
this.parentPath = parentPath;
71-
this.comparator = comparator;
72-
RocksDbSettings.setRocksDbSettings(settings);
73-
initDB();
74-
}
75-
76-
public RocksDbDataSourceImpl(String parentPath, String name, RocksDbSettings settings) {
77-
this.dataBaseName = name;
78-
this.parentPath = parentPath;
79-
RocksDbSettings.setRocksDbSettings(settings);
66+
this.options = options;
8067
initDB();
8168
}
8269

70+
@VisibleForTesting
8371
public RocksDbDataSourceImpl(String parentPath, String name) {
8472
this.parentPath = parentPath;
8573
this.dataBaseName = name;
74+
this.options = RocksDbSettings.getOptionsByDbName(name);
8675
}
8776

8877
public Path getDbPath() {
@@ -225,10 +214,6 @@ public void initDB() {
225214
throw new RuntimeException(
226215
String.format("failed to check database: %s, engine do not match", dataBaseName));
227216
}
228-
initDB(RocksDbSettings.getSettings());
229-
}
230-
231-
public void initDB(RocksDbSettings settings) {
232217
resetDbLock.writeLock().lock();
233218
try {
234219
if (isAlive()) {
@@ -237,81 +222,40 @@ public void initDB(RocksDbSettings settings) {
237222
if (dataBaseName == null) {
238223
throw new IllegalArgumentException("No name set to the dbStore");
239224
}
225+
options.setLogger(new Logger(options) {
226+
@Override
227+
protected void log(InfoLogLevel infoLogLevel, String logMsg) {
228+
rocksDbLogger.info("{} {}", dataBaseName, logMsg);
229+
}
230+
});
240231

241-
try (Options options = new Options()) {
242-
243-
// most of these options are suggested by https://github.com/facebook/rocksdb/wiki/Set-Up-Options
232+
try {
233+
logger.debug("Opening database {}.", dataBaseName);
234+
final Path dbPath = getDbPath();
244235

245-
// general options
246-
if (settings.isEnableStatistics()) {
247-
options.setStatistics(new Statistics());
248-
options.setStatsDumpPeriodSec(60);
249-
}
250-
options.setCreateIfMissing(true);
251-
options.setIncreaseParallelism(1);
252-
options.setLevelCompactionDynamicLevelBytes(true);
253-
options.setMaxOpenFiles(settings.getMaxOpenFiles());
254-
255-
// general options supported user config
256-
options.setNumLevels(settings.getLevelNumber());
257-
options.setMaxBytesForLevelMultiplier(settings.getMaxBytesForLevelMultiplier());
258-
options.setMaxBytesForLevelBase(settings.getMaxBytesForLevelBase());
259-
options.setMaxBackgroundCompactions(settings.getCompactThreads());
260-
options.setLevel0FileNumCompactionTrigger(settings.getLevel0FileNumCompactionTrigger());
261-
options.setTargetFileSizeMultiplier(settings.getTargetFileSizeMultiplier());
262-
options.setTargetFileSizeBase(settings.getTargetFileSizeBase());
263-
if (comparator != null) {
264-
options.setComparator(comparator);
236+
if (!Files.isSymbolicLink(dbPath.getParent())) {
237+
Files.createDirectories(dbPath.getParent());
265238
}
266-
options.setLogger(new Logger(options) {
267-
@Override
268-
protected void log(InfoLogLevel infoLogLevel, String logMsg) {
269-
rocksDbLogger.info("{} {}", dataBaseName, logMsg);
270-
}
271-
});
272-
273-
// table options
274-
final BlockBasedTableConfig tableCfg;
275-
options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig());
276-
tableCfg.setBlockSize(settings.getBlockSize());
277-
tableCfg.setBlockCache(RocksDbSettings.getCache());
278-
tableCfg.setCacheIndexAndFilterBlocks(true);
279-
tableCfg.setPinL0FilterAndIndexBlocksInCache(true);
280-
tableCfg.setFilter(new BloomFilter(10, false));
281-
282-
// read options
283-
readOpts = new ReadOptions();
284-
readOpts = readOpts.setPrefixSameAsStart(true)
285-
.setVerifyChecksums(false);
286239

287240
try {
288-
logger.debug("Opening database {}.", dataBaseName);
289-
final Path dbPath = getDbPath();
290-
291-
if (!Files.isSymbolicLink(dbPath.getParent())) {
292-
Files.createDirectories(dbPath.getParent());
241+
database = RocksDB.open(options, dbPath.toString());
242+
} catch (RocksDBException e) {
243+
if (Objects.equals(e.getStatus().getCode(), Status.Code.Corruption)) {
244+
logger.error("Database {} corrupted, please delete database directory({}) "
245+
+ "and restart.", dataBaseName, parentPath, e);
246+
} else {
247+
logger.error("Open Database {} failed", dataBaseName, e);
293248
}
294-
295-
try {
296-
database = RocksDB.open(options, dbPath.toString());
297-
} catch (RocksDBException e) {
298-
if (Objects.equals(e.getStatus().getCode(), Status.Code.Corruption)) {
299-
logger.error("Database {} corrupted, please delete database directory({}) " +
300-
"and restart.", dataBaseName, parentPath, e);
301-
} else {
302-
logger.error("Open Database {} failed", dataBaseName, e);
303-
}
304-
throw new TronError(e, TronError.ErrCode.ROCKSDB_INIT);
305-
}
306-
307-
alive = true;
308-
} catch (IOException ioe) {
309-
throw new RuntimeException(
310-
String.format("failed to init database: %s", dataBaseName), ioe);
249+
throw new TronError(e, TronError.ErrCode.ROCKSDB_INIT);
311250
}
312251

313-
logger.debug("Init DB {} done.", dataBaseName);
252+
alive = true;
253+
} catch (IOException ioe) {
254+
throw new RuntimeException(
255+
String.format("failed to init database: %s", dataBaseName), ioe);
314256
}
257+
258+
logger.debug("Init DB {} done.", dataBaseName);
315259
} finally {
316260
resetDbLock.writeLock().unlock();
317261
}
@@ -516,7 +460,8 @@ public boolean deleteDbBakPath(String dir) {
516460

517461
@Override
518462
public RocksDbDataSourceImpl newInstance() {
519-
return new RocksDbDataSourceImpl(parentPath, dataBaseName, RocksDbSettings.getSettings());
463+
return new RocksDbDataSourceImpl(parentPath, dataBaseName,
464+
this.options);
520465
}
521466

522467

‎chainbase/src/main/java/org/tron/common/utils/StorageUtils.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,15 @@ public static String getOutputDirectory() {
5252
}
5353

5454
public static Options getOptionsByDbName(String dbName) {
55+
Options options;
5556
if (hasProperty(dbName)) {
56-
return getProperty(dbName).getDbOptions();
57+
options = getProperty(dbName).getDbOptions();
58+
} else {
59+
options = CommonParameter.getInstance().getStorage().newDefaultDbOptions(dbName);
5760
}
58-
return CommonParameter.getInstance().getStorage().newDefaultDbOptions(dbName);
61+
if ("market_pair_price_to_order".equals(dbName)) {
62+
options.comparator(new MarketOrderPriceComparatorForLevelDB());
63+
}
64+
return options;
5965
}
6066
}

‎chainbase/src/main/java/org/tron/core/db/TronDatabase.java‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@
99
import lombok.Getter;
1010
import lombok.extern.slf4j.Slf4j;
1111
import org.iq80.leveldb.WriteOptions;
12-
import org.rocksdb.DirectComparator;
1312
import org.springframework.beans.factory.annotation.Autowired;
1413
import org.tron.common.parameter.CommonParameter;
14+
import org.tron.common.storage.OptionsPicker;
1515
import org.tron.common.storage.WriteOptionsWrapper;
1616
import org.tron.common.storage.leveldb.LevelDbDataSourceImpl;
1717
import org.tron.common.storage.metric.DbStatService;
@@ -24,7 +24,7 @@
2424
import org.tron.core.exception.ItemNotFoundException;
2525

2626
@Slf4j(topic = "DB")
27-
public abstract class TronDatabase<T> implements ITronChainBase<T> {
27+
public abstract class TronDatabase<T> extends OptionsPicker implements ITronChainBase<T> {
2828

2929
protected DbSourceInter<byte[]> dbSource;
3030
@Getter
@@ -51,8 +51,7 @@ protected TronDatabase(String dbName) {
5151
String parentName = Paths.get(StorageUtils.getOutputDirectoryByDbName(dbName),
5252
CommonParameter.getInstance().getStorage().getDbDirectory()).toString();
5353
dbSource =
54-
new RocksDbDataSourceImpl(parentName, dbName, CommonParameter.getInstance()
55-
.getRocksDBCustomSettings(), getDirectComparator());
54+
new RocksDbDataSourceImpl(parentName, dbName, getOptionsByDbNameForRocksDB(dbName));
5655
}
5756

5857
dbSource.initDB();
@@ -66,14 +65,6 @@ protected void init() {
6665
protected TronDatabase() {
6766
}
6867

69-
protected org.iq80.leveldb.Options getOptionsByDbNameForLevelDB(String dbName) {
70-
return StorageUtils.getOptionsByDbName(dbName);
71-
}
72-
73-
protected DirectComparator getDirectComparator() {
74-
return null;
75-
}
76-
7768
public DbSourceInter<byte[]> getDbSource() {
7869
return dbSource;
7970
}

‎chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@
1616
import lombok.Getter;
1717
import lombok.extern.slf4j.Slf4j;
1818
import org.iq80.leveldb.WriteOptions;
19-
import org.rocksdb.DirectComparator;
2019
import org.springframework.beans.factory.annotation.Autowired;
2120
import org.tron.common.parameter.CommonParameter;
21+
import org.tron.common.storage.OptionsPicker;
2222
import org.tron.common.storage.leveldb.LevelDbDataSourceImpl;
2323
import org.tron.common.storage.metric.DbStatService;
2424
import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl;
@@ -38,7 +38,7 @@
3838

3939

4040
@Slf4j(topic = "DB")
41-
public abstract class TronStoreWithRevoking<T extends ProtoCapsule> implements ITronChainBase<T> {
41+
public abstract class TronStoreWithRevoking<T extends ProtoCapsule> extends OptionsPicker implements ITronChainBase<T> {
4242

4343
@Getter // only for unit test
4444
protected IRevokingDB revokingDB;
@@ -69,22 +69,13 @@ protected TronStoreWithRevoking(String dbName) {
6969
.getInstance().getStorage().getDbDirectory()).toString();
7070
this.db = new RocksDB(
7171
new RocksDbDataSourceImpl(parentPath,
72-
dbName, CommonParameter.getInstance()
73-
.getRocksDBCustomSettings(), getDirectComparator()));
72+
dbName, getOptionsByDbNameForRocksDB(dbName)));
7473
} else {
7574
throw new RuntimeException(String.format("db engine %s is error", dbEngine));
7675
}
7776
this.revokingDB = new Chainbase(new SnapshotRoot(this.db));
7877
}
7978

80-
protected org.iq80.leveldb.Options getOptionsByDbNameForLevelDB(String dbName) {
81-
return StorageUtils.getOptionsByDbName(dbName);
82-
}
83-
84-
protected DirectComparator getDirectComparator() {
85-
return null;
86-
}
87-
8879
protected TronStoreWithRevoking(DB<byte[], byte[]> db) {
8980
this.db = db;
9081
this.revokingDB = new Chainbase(new SnapshotRoot(db));

‎chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.tron.common.parameter.CommonParameter;
3636
import org.tron.common.prometheus.MetricKeys;
3737
import org.tron.common.prometheus.Metrics;
38+
import org.tron.common.storage.OptionsPicker;
3839
import org.tron.common.storage.leveldb.LevelDbDataSourceImpl;
3940
import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl;
4041
import org.tron.common.utils.ByteArray;
@@ -48,7 +49,7 @@
4849
import org.tron.core.store.DynamicPropertiesStore;
4950

5051
@Slf4j(topic = "DB")
51-
public class TxCacheDB implements DB<byte[], byte[]>, Flusher {
52+
public class TxCacheDB extends OptionsPicker implements DB<byte[], byte[]>, Flusher {
5253

5354
// > 65_536(= 2^16) blocks, that is the number of the reference block
5455
private static final long MAX_BLOCK_SIZE = 65536;
@@ -106,7 +107,7 @@ public TxCacheDB(String name, RecentTransactionStore recentTransactionStore,
106107
if ("LEVELDB".equals(dbEngine.toUpperCase())) {
107108
this.persistentStore = new LevelDB(
108109
new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(name),
109-
name, StorageUtils.getOptionsByDbName(name),
110+
name, getOptionsByDbNameForLevelDB(name),
110111
new WriteOptions().sync(CommonParameter.getInstance()
111112
.getStorage().isDbSync())));
112113
} else if ("ROCKSDB".equals(dbEngine.toUpperCase())) {
@@ -115,9 +116,7 @@ public TxCacheDB(String name, RecentTransactionStore recentTransactionStore,
115116
.getInstance().getStorage().getDbDirectory()).toString();
116117

117118
this.persistentStore = new RocksDB(
118-
new RocksDbDataSourceImpl(parentPath,
119-
name, CommonParameter.getInstance()
120-
.getRocksDBCustomSettings()));
119+
new RocksDbDataSourceImpl(parentPath, name, getOptionsByDbNameForRocksDB(name)));
121120
} else {
122121
throw new RuntimeException(String.format("db type: %s is not supported", dbEngine));
123122
}

‎chainbase/src/main/java/org/tron/core/store/MarketPairPriceToOrderStore.java‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,10 @@
33
import java.util.ArrayList;
44
import java.util.Collections;
55
import java.util.List;
6-
import org.iq80.leveldb.Options;
7-
import org.rocksdb.ComparatorOptions;
8-
import org.rocksdb.DirectComparator;
96
import org.springframework.beans.factory.annotation.Autowired;
107
import org.springframework.beans.factory.annotation.Value;
118
import org.springframework.stereotype.Component;
129
import org.tron.common.utils.ByteUtil;
13-
import org.tron.common.utils.MarketOrderPriceComparatorForLevelDB;
14-
import org.tron.common.utils.MarketOrderPriceComparatorForRockDB;
15-
import org.tron.common.utils.StorageUtils;
1610
import org.tron.core.capsule.MarketOrderIdListCapsule;
1711
import org.tron.core.capsule.utils.MarketUtils;
1812
import org.tron.core.db.TronStoreWithRevoking;
@@ -26,20 +20,6 @@ protected MarketPairPriceToOrderStore(@Value("market_pair_price_to_order") Strin
2620
super(dbName);
2721
}
2822

29-
@Override
30-
protected Options getOptionsByDbNameForLevelDB(String dbName) {
31-
Options options = StorageUtils.getOptionsByDbName(dbName);
32-
options.comparator(new MarketOrderPriceComparatorForLevelDB());
33-
return options;
34-
}
35-
36-
//todo: to test later
37-
@Override
38-
protected DirectComparator getDirectComparator() {
39-
ComparatorOptions comparatorOptions = new ComparatorOptions();
40-
return new MarketOrderPriceComparatorForRockDB(comparatorOptions);
41-
}
42-
4323
@Override
4424
public MarketOrderIdListCapsule get(byte[] key) throws ItemNotFoundException {
4525
byte[] value = revokingDB.get(key);

0 commit comments

Comments
 (0)