migrate from derby to h2, derby is retired.

Author: wolfgangasdf

README.md

@@ -72,7 +72,7 @@ plugin for development, just open the project to get started.
 * [Java](https://www.java.com/)
 * [Scala](http://www.scala-lang.org)
 * [Scalafx](http://scalafx.org) as wrapper for [JavaFX](http://docs.oracle.com/javafx) for the graphical user interface
-* [Squeryl](http://squeryl.org) as database ORM & DSL, using [Apache Derby](http://db.apache.org/derby) embedded as backend
+* [Squeryl](http://squeryl.org) as database ORM & DSL, using [H2](https://www.h2database.com/) embedded as backend
 * [Apache Pdfbox](https://pdfbox.apache.org) to access PDF files
 * [JBibtex](https://github.com/jbibtex/jbibtex) to parse and write bibtex and latex
 * [LaTeX2Unicode](https://github.com/tomtung/latex2unicode) 

build.gradle.kts

@@ -63,6 +63,7 @@ dependencies {
     implementation("org.apache.derby:derby:$derbyVersion")
     implementation("org.apache.derby:derbytools:$derbyVersion")
     implementation("org.apache.derby:derbyshared:$derbyVersion")
+    implementation("com.h2database:h2:2.4.240")
     implementation("org.squeryl:squeryl_2.13:0.10.1")
     implementation("org.scala-lang.modules:scala-parser-combinators_2.13:2.4.0")
     implementation("org.apache.pdfbox:pdfbox:3.0.6")

src/main/scala/db/DB.scala

@@ -4,13 +4,13 @@ import java.io.OutputStream
 import java.sql.SQLException
 import java.text.SimpleDateFormat
 import java.time.Instant
-import org.squeryl.adapters.DerbyAdapter
+import org.squeryl.adapters.H2Adapter
 import org.squeryl.dsl._
 import org.squeryl._
 import org.squeryl.annotations.Transient
 import util._
 import util.AppStorage
-import framework.{ApplicationController, Logging}
+import framework.{ApplicationController, Helpers, Logging}
 import org.squeryl.internals.LifecycleEvent
 
 import scala.collection.mutable.ArrayBuffer
@@ -151,8 +151,6 @@ class Topic2Article(val TOPIC: Long = 0, val ARTICLE: Long = 0, var color: Int =
 
 object ReftoolDB extends Schema with Logging {
 
-  val lastschemaversion = 3
-
   val longFieldLength = 10000
   val mediumFieldLength = 4096
 
@@ -181,7 +179,7 @@ object ReftoolDB extends Schema with Logging {
 
   on(settings)(t => declare(
     t.id.is(dbType("varchar(1024)"), named("ID")),
-    t.value.is(dbType("varchar(1024)"), named("VALUE")), t.value.defaultsTo("")
+    t.value.is(dbType("varchar(1024)"), named("CONTENT")), t.value.defaultsTo("")
   ))
 
   on(topics)(t => declare(
@@ -264,18 +262,17 @@ object ReftoolDB extends Schema with Logging {
 
   def dbShutdown(dbpath: String = null): Unit = {
     val dbpath2 = if (dbpath == null) AppStorage.config.dbpath else dbpath
-    try { java.sql.DriverManager.getConnection(s"jdbc:derby:$dbpath2;shutdown=true") } catch {
-      case se: SQLException => if (!se.getSQLState.equals("08006")) throw se
-//      case se: SQLNonTransientConnectionException => if (!se.getSQLState.equals("08006")) throw se
+    try { java.sql.DriverManager.getConnection(s"jdbc:h2:$dbpath2;shutdown=true") } catch {
+      case se: SQLException => if (!se.getSQLState.equals("08006")) throw se // TODO: still needed with H2?
     }
   }
-  def dbGetSchemaVersion: Int = {
-    val res = FileHelper.readString(new MFile(AppStorage.config.dbschemaversionpath))
+  def dbGetSchemaVersion(dbpath: String): Int = {
+    val res = FileHelper.readString(new MFile(dbpath + "/schemaversion.txt"))
     assert(res.nonEmpty, "db schema version file not present!")
     res.get.toInt
   }
-  def dbSetSchemaVersion(v: Int): Unit = {
-    FileHelper.writeString(new MFile(AppStorage.config.dbschemaversionpath), v.toString)
+  def dbSetSchemaVersion(dbpath: String, v: Int): Unit = {
+    FileHelper.writeString(new MFile(dbpath + "/schemaversion.txt"), v.toString)
   }
 
   def getDBstats: String = {
@@ -293,40 +290,51 @@ object ReftoolDB extends Schema with Logging {
 
   def initialize(startwithempty: Boolean): Unit = {
 
-    System.setProperty("derby.stream.error.method", "db.DerbyUtil.getOS")
-    // System.setProperty("derby.language.logStatementText", "true")
-
     val pp = new MFile(AppStorage.config.pdfpath)
     if (!pp.exists) pp.mkdir()
 
-    if (!startwithempty) {
-      if (!new MFile(AppStorage.config.dbpath).exists && new MFile(AppStorage.config.olddbpath).exists) {
+    if (!startwithempty) { // open existing database
+      // upgrade reftool4 -> reftool5
+      if (!new MFile(AppStorage.config.dbpath).exists && !new MFile(AppStorage.config.olddbpath5).exists && new MFile(AppStorage.config.olddbpath).exists) {
+        if (!Helpers.showWarning("About to update your database (4->5), have you backed it up?", "Continue?")) assert(assertion = false, "")
         DBupgrades.upgrade4to5()
-        dbSetSchemaVersion(1)
+        dbSetSchemaVersion(AppStorage.config.olddbpath5, 1)
+        assert(new MFile(AppStorage.config.olddbpath5).exists, "Cannot find database path below data directory!")
+        Helpers.showInformation("Database upgrade finished", s"Please delete the old database folder\n ${AppStorage.config.olddbpath}!")
+      }
+      // upgrade reftool5 derby to schema 4, then migrate to h2, schema 5
+      if (!new MFile(AppStorage.config.dbpath).exists && new MFile(AppStorage.config.olddbpath5).exists) {
+        if (!Helpers.showWarning("About to update your database (derby->h2), have you backed it up?", "Continue?")) assert(assertion = false, "")
+        // upgrade derby schema to version 4 (rename SETTING.VALUE -> CONTENT)
+        assert(dbGetSchemaVersion(AppStorage.config.olddbpath5) <= 4, "derby DB schema version is higher than reftool can handle - old reftool version?")
+        while (dbGetSchemaVersion(AppStorage.config.olddbpath5) != 4) dbSetSchemaVersion(AppStorage.config.olddbpath5, DBupgrades.upgradeSchema(dbGetSchemaVersion(AppStorage.config.olddbpath5)))
+        DBupgradesH2.upgrade5to5h2()
+        dbSetSchemaVersion(AppStorage.config.dbpath, 5)
+        Helpers.showInformation("Database upgrade finished", s"Please delete the old database folder\n ${AppStorage.config.olddbpath5}!")
       }
       assert(new MFile(AppStorage.config.dbpath).exists, "Cannot find database path below data directory!")
       assert(pp.exists, "Cannot find pdf path below data directory")
       // upgrade DB schema if needed
-      assert(dbGetSchemaVersion <= lastschemaversion, "DB schema version is higher than reftool can handle - old reftool version?")
-      while (dbGetSchemaVersion != lastschemaversion) dbSetSchemaVersion(DBupgrades.upgradeSchema(dbGetSchemaVersion))
+      assert(dbGetSchemaVersion(AppStorage.config.dbpath) <= DBupgradesH2.lastschemaversion, "DB schema version is higher than reftool can handle - old reftool version?")
+      if (dbGetSchemaVersion(AppStorage.config.dbpath) != DBupgradesH2.lastschemaversion)
+        if (!Helpers.showWarning("About to update your database, have you backed it up?", "Continue?")) assert(assertion = false, "")
+      while (dbGetSchemaVersion(AppStorage.config.dbpath) != DBupgradesH2.lastschemaversion) dbSetSchemaVersion(AppStorage.config.dbpath, DBupgrades.upgradeSchema(dbGetSchemaVersion(AppStorage.config.dbpath)))
     }
 
     info("Loading database at " + AppStorage.config.dbpath + " ...")
     val dbs = if (!startwithempty)
-      s"jdbc:derby:${AppStorage.config.dbpath};upgrade=true"
+      s"jdbc:h2:${AppStorage.config.dbpath}/h2db;IFEXISTS=TRUE;DB_CLOSE_DELAY=10" //;TRACE_LEVEL_SYSTEM_OUT=2"
     else
-      s"jdbc:derby:${AppStorage.config.dbpath};upgrade=true;create=true"
-
-    Class.forName("org.apache.derby.jdbc.EmbeddedDriver")
+      s"jdbc:h2:${AppStorage.config.dbpath}/h2db"
 
-    SessionFactory.concreteFactory = Some(() => Session.create(java.sql.DriverManager.getConnection(dbs), new DerbyAdapter))
+    SessionFactory.concreteFactory = Some(() => Session.create(java.sql.DriverManager.getConnection(dbs), new H2Adapter))
 
     transaction {
       // Session.currentSession.setLogger( s => debug("sq: " + s) )
-      // ReftoolDB.printDdl
+      // ReftoolDB.printDdl // print schema
       if (startwithempty) {
         ReftoolDB.create
-        dbSetSchemaVersion(lastschemaversion)
+        dbSetSchemaVersion(AppStorage.config.dbpath, DBupgradesH2.lastschemaversion)
       }
       // ensure essential topics are present
       rootTopic = topics.where(t => t.parent === 0).headOption.orNull
@@ -362,6 +370,7 @@ object ReftoolDB extends Schema with Logging {
 
 }
 
+// TODO
 object DerbyUtil extends Logging {
   var sss = ""
   def getOS: OutputStream = (b: Int) => {

src/main/scala/db/DBupgrades.scala

@@ -1,18 +1,27 @@
 package db
 
-import java.sql.Connection
-
+import java.sql.{Connection, SQLException}
 import framework.Logging
 import util.AppStorage
-import ReftoolDB.dbShutdown
 
+// This is only for derby upgrades, derby is now retired
 
 object DBupgrades extends Logging {
 
-  def dbGetConnection: Connection = java.sql.DriverManager.getConnection(s"jdbc:derby:${AppStorage.config.dbpath};upgrade=true")
+  val lastschemaversion = 4
+
+  def dbGetConnection: Connection = java.sql.DriverManager.getConnection(s"jdbc:derby:${AppStorage.config.olddbpath5};upgrade=true")
+
+  def dbShutdown(dbpath: String = null): Unit = {
+    val dbpath2 = if (dbpath == null) AppStorage.config.olddbpath5 else dbpath
+    try { java.sql.DriverManager.getConnection(s"jdbc:derby:$dbpath2;shutdown=true") } catch {
+      case se: SQLException => if (!se.getSQLState.equals("08006")) throw se
+      //      case se: SQLNonTransientConnectionException => if (!se.getSQLState.equals("08006")) throw se
+    }
+  }
 
   def upgradeSchema(oldv: Int): Int = {
-    info("upgrade database from schemaversion " + oldv + " ...")
+    info("upgrade derby database from schemaversion " + oldv + " ...")
     val dbc = dbGetConnection
     val s = dbc.createStatement()
     val newVersion = oldv match {
@@ -23,6 +32,9 @@ object DBupgrades extends Logging {
       case 2 =>
         s.execute("alter table ARTICLES add column LASTTIMESTAMP bigint not null default 0")
         3
+      case 3 =>
+        s.execute("rename column SETTING.VALUE to CONTENT")
+        4
     }
     dbc.close()
     dbShutdown()
@@ -69,7 +81,7 @@ object DBupgrades extends Logging {
     debug("checking old db...")
     dbStats(AppStorage.config.olddbpath, clobx = true)
     debug("importing db...")
-    java.sql.DriverManager.getConnection(s"jdbc:derby:${AppStorage.config.dbpath};createFrom=${AppStorage.config.olddbpath}")
+    java.sql.DriverManager.getConnection(s"jdbc:derby:${AppStorage.config.olddbpath5};createFrom=${AppStorage.config.olddbpath}")
     dbShutdown()
     debug("modify db (hard upgrade if needed)...")
     val dbc = dbGetConnection
@@ -130,7 +142,7 @@ object DBupgrades extends Logging {
     dbc.close()
     dbShutdown()
     debug("checking new db...")
-    dbStats(AppStorage.config.dbpath, clobx = false)
+    dbStats(AppStorage.config.olddbpath5, clobx = false)
     info("Upgrade4to5 finished!")
   }
 

src/main/scala/db/DBupgradesH2.scala

@@ -0,0 +1,87 @@
+package db
+
+import db.SquerylEntrypointForMyApp.transaction
+import framework.Logging
+import org.squeryl.adapters.H2Adapter
+import org.squeryl.{Session, SessionFactory}
+import util.AppStorage
+
+import java.sql.{Connection, ResultSet, Statement}
+
+object DBupgradesH2 extends Logging {
+
+  val lastschemaversion = 5
+
+  def upgrade5to5h2(): Unit = {
+    info("upgrade5to5h2 of " + AppStorage.config.olddbpath5 + " to " + AppStorage.config.dbpath)
+    // create new h2 database schema with squeryl, messes up column order!
+    var h2con: Connection = java.sql.DriverManager.getConnection(s"jdbc:h2:${AppStorage.config.dbpath}/h2db")
+    SessionFactory.concreteFactory = Some(() => Session.create(h2con, new H2Adapter))
+    transaction {
+      ReftoolDB.create
+    }
+    // reconnect
+    h2con = java.sql.DriverManager.getConnection(s"jdbc:h2:${AppStorage.config.dbpath}/h2db")
+    // connect via jdbc to old db
+    val derbycon: Connection = java.sql.DriverManager.getConnection(s"jdbc:derby:${AppStorage.config.olddbpath5}")
+    // copy over data
+    copyTable(derbycon, h2con, "SETTING", 1)
+    copyTable(derbycon, h2con, "TOPICS", 20)
+    copyTable(derbycon, h2con, "ARTICLES", 100)
+    copyTable(derbycon, h2con, "TOPIC2ARTICLE", 100)
+    h2con.close()
+    derbycon.close()
+  }
+
+  // based on https://technology.amis.nl/software-development/java/jdbc-database-table-contents-copy/
+  def copyTable(fromDB: Connection, toDB: Connection, tableName: String, debugmod: Int): Unit = {
+    info(s"copytable: table=$tableName")
+    def surroundWithQuotesAddToBuffer(coltype: String, sb: StringBuffer, o: AnyRef): Unit = {
+      if (coltype == "uniqueidentifier" || coltype == "string" || coltype == "varchar2" || coltype == "varchar" || coltype == "datetime") if (o != null) {
+        sb.append("\'")
+        val string = o.toString
+        val newString = string.replaceAll("\'", "\'\'")
+        sb.append(newString)
+        sb.append("\'")
+      }
+      else sb.append("null")
+      else sb.append(o)
+    }
+
+    var fromStatement: Statement = null
+    var toStatement: Statement = null
+    var resultSet: ResultSet = null
+    fromStatement = fromDB.createStatement
+    toStatement = toDB.createStatement
+    resultSet = fromStatement.executeQuery("select * from " + tableName)
+    // Find out the number of columns
+    val columnCount = resultSet.getMetaData.getColumnCount
+    var rowCount = 0
+    while (resultSet.next()) {
+      rowCount += 1
+      var o: AnyRef = null
+      var i = 0
+      val insertValues = new StringBuffer
+      var insertColumnNames = ""
+      if (insertColumnNames == "") { // only do once
+        for (i <- 1 to columnCount) {
+          insertColumnNames += resultSet.getMetaData.getColumnName(i)
+          if (i != columnCount) insertColumnNames += ","
+        }
+      }
+      for (i <- 1 to columnCount) {
+        o = resultSet.getObject(i)
+        val coltype = resultSet.getMetaData.getColumnTypeName(i).toLowerCase
+        surroundWithQuotesAddToBuffer(coltype, insertValues, o)
+        if (i != columnCount) insertValues.append(", ")
+      }
+      val s = s"insert into $tableName ($insertColumnNames) values ($insertValues)"
+      if (rowCount % debugmod == 0) info(s"copytable row $rowCount : $s")
+      toStatement.executeUpdate(s)
+    }
+    fromStatement.close()
+    toStatement.close()
+    resultSet.close()
+  }
+
+}

src/main/scala/framework/Helpers.scala

@@ -140,6 +140,27 @@ object Helpers extends Logging {
     }.showAndWait()
   }
 
+  // returns true if ok clicked, false for cancel
+  def showWarning(header: String, content: String): Boolean = {
+    val res = new Alert(AlertType.Confirmation) {
+      title = "Confirmation"
+      headerText = header
+      contentText = content
+    }.showAndWait()
+    res match {
+      case Some(ButtonType.OK) => true
+      case _                   => false
+    }
+  }
+
+  def showInformation(header: String, content: String): Unit = {
+    val res = new Alert(AlertType.Information) {
+      title = "Confirmation"
+      headerText = header
+      contentText = content
+    }.showAndWait()
+  }
+
   // https://stackoverflow.com/a/22404140
   import java.net.URISyntaxException
   def getClassBuildTime: Date = {

src/main/scala/util/AppStorage.scala

@@ -18,9 +18,9 @@ class Config extends Logging {
   var autshrinkpdfs = false
   var gspath = ""
 
-  def dbpath: String = datadir + "/db5"
-  def dbschemaversionpath: String = dbpath + "/schemaversion.txt"
+  def dbpath: String = datadir + "/db5h2" // reftool5 with h2 database
   def olddbpath: String = datadir + "/db" // reftool4
+  def olddbpath5: String = datadir + "/db5" // reftool5 before h2
   def pdfpath: String = datadir + "/files"
   val importfolderprefix = "folder-" // after this a number to limit #files in folder
   val csspath: String = getClass.getResource("/reftool.css").toExternalForm

src/main/scala/views/ArticleDetailView.scala

@@ -27,14 +27,7 @@ class ArticleDetailView extends GenericView("articledetailview") with Logging {
 
   override def canClose: Boolean = {
     if (isDirty.value) {
-      val res = new MyAlert(AlertType.Confirmation) {
-        headerText = "Application close requested"
-        contentText = "Press OK to discard changes to Article \n" + article
-      }.showAndWait()
-      res match {
-        case Some(ButtonType.OK) => true
-        case _ => false
-      }
+      Helpers.showWarning("Application close requested", "Press OK to discard changes to Article \n" + article)
     } else true
   }
 

src/main/scala/views/ArticleDocumentsView.scala

@@ -3,7 +3,7 @@ package views
 import db.{Article, Document, ReftoolDB}
 import db.SquerylEntrypointForMyApp._
 import framework.Helpers.{FixedSfxTooltip, MyAlert}
-import framework.{ApplicationController, GenericView, Logging, MyAction}
+import framework.{ApplicationController, GenericView, Helpers, Logging, MyAction}
 import util.FileHelper._
 import util.{FileHelper, ImportHelper, MFile}
 
@@ -80,17 +80,15 @@ class ArticleDocumentsView extends GenericView("articledocumentsview") with Logg
     tooltipString = "Delete selected documents from article and delete document"
     image = new Image(getClass.getResource("/images/delete_obj.gif").toExternalForm)
     action = _ => {
-      new MyAlert(AlertType.Confirmation, "Really deleted selected documents", ButtonType.Yes, ButtonType.No).showAndWait() match {
-        case Some(ButtonType.Yes) =>
-          val ds = new ArrayBuffer[Document] ++ lv.selectionModel.value.getSelectedItems
-          ds.foreach( dd => {
-            getDocumentFileAbs(dd.docPath).delete()
-            lv.getItems.remove(dd)
-          })
-          ApplicationController.showNotification(s"Document deleted!")
-          updateArticle()
-          setArticle(article)
-        case _ =>
+      if (Helpers.showWarning("Really deleted selected documents?", "")) {
+        val ds = new ArrayBuffer[Document] ++ lv.selectionModel.value.getSelectedItems
+        ds.foreach(dd => {
+          getDocumentFileAbs(dd.docPath).delete()
+          lv.getItems.remove(dd)
+        })
+        ApplicationController.showNotification(s"Document deleted!")
+        updateArticle()
+        setArticle(article)
       }
     }
   }