optionally read rocksdb configfile

Author: fm3

build.sbt

@@ -25,7 +25,7 @@ libraryDependencies ++= Seq(
   "org.scalatest" % "scalatest_2.12" % "3.0.4" % "test",
   "io.grpc" % "grpc-netty" % com.trueaccord.scalapb.compiler.Version.grpcJavaVersion,
   "com.trueaccord.scalapb" %% "scalapb-runtime-grpc" % com.trueaccord.scalapb.compiler.Version.scalapbVersion,
-  "org.rocksdb" % "rocksdbjni" % "5.1.2",
+  "org.rocksdb" % "rocksdbjni" % "5.11.3",
   "com.github.scopt" %% "scopt" % "3.7.0"
 )
 

src/main/scala/com/scalableminds/fossildb/FossilDB.scala

@@ -8,9 +8,10 @@ import fossildb.BuildInfo
 
 import scala.concurrent.ExecutionContext
 
-object ConfigDefaults {val port = 7155; val dataDir = "data"; val backupDir = "backup"; val columnFamilies = List()}
+object ConfigDefaults {val port = 7155; val dataDir = "data"; val backupDir = "backup"; val columnFamilies = List(); val rocksOptionsFile = None}
 case class Config(port: Int = ConfigDefaults.port, dataDir: String = ConfigDefaults.dataDir,
-                  backupDir: String = ConfigDefaults.backupDir, columnFamilies: List[String] = ConfigDefaults.columnFamilies)
+                  backupDir: String = ConfigDefaults.backupDir, columnFamilies: List[String] = ConfigDefaults.columnFamilies,
+                  rocksOptionsFile: Option[String] = ConfigDefaults.rocksOptionsFile)
 
 object FossilDB extends LazyLogging {
   def main(args: Array[String]) = {
@@ -24,7 +25,7 @@ object FossilDB extends LazyLogging {
           logger.info("BuildInfo: (" + BuildInfo + ")")
           logger.info("Config: " + config)
 
-          val storeManager = new StoreManager(Paths.get(config.dataDir), Paths.get(config.backupDir), config.columnFamilies)
+          val storeManager = new StoreManager(Paths.get(config.dataDir), Paths.get(config.backupDir), config.columnFamilies, config.rocksOptionsFile)
 
           val server = new FossilDBServer(storeManager, config.port, ExecutionContext.global)
 
@@ -51,6 +52,9 @@ object FossilDB extends LazyLogging {
 
       opt[Seq[String]]('c', "columnFamilies").required.valueName("<cf1>,<cf2>...").action( (x, c) =>
         c.copy(columnFamilies = x.toList) ).text("column families of the database (created if there is no db yet)")
+
+      opt[String]('r', "rocksOptionsFile").valueName("<filepath>").action( (x, c) =>
+        c.copy(rocksOptionsFile = Some(x)) ).text("rocksdb options file. Default: " + ConfigDefaults.rocksOptionsFile)
     }
 
     parser.parse(args, Config())

src/main/scala/com/scalableminds/fossildb/db/RocksDBStore.scala

@@ -10,13 +10,14 @@ import com.typesafe.scalalogging.LazyLogging
 import org.rocksdb._
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 import scala.concurrent.Future
 
 case class BackupInfo(id: Int, timestamp: Long, size: Long)
 
 case class KeyValuePair[T](key: String, value: T)
 
-class RocksDBManager(dataDir: Path, columnFamilies: List[String]) extends LazyLogging {
+class RocksDBManager(dataDir: Path, columnFamilies: List[String], optionsFilePathOpt: Option[String]) extends LazyLogging {
 
   val (db: RocksDB, columnFamilyHandles) = {
     RocksDB.loadLibrary()
@@ -28,9 +29,21 @@ class RocksDBManager(dataDir: Path, columnFamilies: List[String]) extends LazyLo
       new ColumnFamilyDescriptor(columnFamily, columnOptions)
     }
     val columnFamilyHandles = new util.ArrayList[ColumnFamilyHandle]
-    val options = new DBOptions()
+    var options = new DBOptions()
       .setCreateIfMissing(true)
       .setCreateMissingColumnFamilies(true)
+    var cfListRef: mutable.Buffer[ColumnFamilyDescriptor] = mutable.Buffer()
+    optionsFilePathOpt.map { optionsFilePath =>
+      try {
+        org.rocksdb.OptionsUtil.loadOptionsFromFile(optionsFilePath, Env.getDefault, options, cfListRef.asJava)
+        logger.info("successfully loaded rocksdb options from " + optionsFilePath)
+      } catch {
+        case e: Exception => {
+          logger.error("Failed to load rocksdb options from file " + optionsFilePath + ": " + e.getMessage)
+          System.exit(1)
+        }
+      }
+    }
     logger.info("Opening RocksDB at " + dataDir.toAbsolutePath)
     val db = RocksDB.open(
       options,

src/main/scala/com/scalableminds/fossildb/db/StoreManager.scala

@@ -6,7 +6,7 @@ package com.scalableminds.fossildb.db
 import java.nio.file.Path
 import java.util.concurrent.atomic.AtomicBoolean
 
-class StoreManager(dataDir: Path, backupDir: Path, columnFamilies: List[String]) {
+class StoreManager(dataDir: Path, backupDir: Path, columnFamilies: List[String], rocksdbOptions: Option[String]) {
 
   var rocksDBManager: Option[RocksDBManager] = None
   var stores: Option[Map[String, VersionedKeyValueStore]] = None
@@ -15,7 +15,7 @@ class StoreManager(dataDir: Path, backupDir: Path, columnFamilies: List[String])
 
   def reInitialize = {
     rocksDBManager.map(_.close)
-    rocksDBManager = Some(new RocksDBManager(dataDir, columnFamilies))
+    rocksDBManager = Some(new RocksDBManager(dataDir, columnFamilies, rocksdbOptions))
     stores = Some(columnFamilies.map { cf =>
       val store: VersionedKeyValueStore = new VersionedKeyValueStore(rocksDBManager.get.getStoreForColumnFamily(cf).get)
       (cf -> store)