Merge branch 'sys_leveldb' into rm_minisketch-28+k

Author: luke-jr

build-aux/m4/bitcoin_subdir_to_include.m4

@@ -5,14 +5,48 @@ dnl file COPYING or http://www.opensource.org/licenses/mit-license.php.
 dnl BITCOIN_SUBDIR_TO_INCLUDE([CPPFLAGS-VARIABLE-NAME],[SUBDIRECTORY-NAME],[HEADER-FILE])
 dnl SUBDIRECTORY-NAME must end with a path separator
 AC_DEFUN([BITCOIN_SUBDIR_TO_INCLUDE],[
-  if test "$2" = ""; then
+  m4_pushdef([_result_var],[$1])
+  m4_pushdef([_rel_path],[$2])
+  m4_pushdef([_header_file],[$3.h])
+  if test "[]_rel_path" = ""; then
     AC_MSG_RESULT([default])
   else
-    echo "#include <$2$3.h>" >conftest.cpp
-    newinclpath=`${CXXCPP} ${CPPFLAGS} -M conftest.cpp 2>/dev/null | [ tr -d '\\n\\r\\\\' | sed -e 's/^.*[[:space:]:]\(\/[^[:space:]]*\)]$3[\.h[[:space:]].*$/\1/' -e t -e d`]
+    echo '[#]include <'"_rel_path"'/_header_file>' >conftest.cpp
+    newinclpath=$(
+      ${CXXCPP} ${CPPFLAGS} -M conftest.cpp 2>/dev/null |
+      ${SED} -E m4_bpatsubsts([[
+        :build_line
+# If the line doesn't end with a backslash, it is complete; go on to process it
+        /\\$/!b have_complete_line
+# Otherwise, read the next line, and concatenate it to the current one with a space
+        N
+        s/\\\n/ /
+# Then go back and check for a trailing backslash again.
+        t build_line
+
+# When we get here, we have the completed line, with all continuations collapsed.
+        :have_complete_line
+        s/^[^:]*:[[:space:]]*(([^[:space:]\]|\\.)*[[:space:]])*(([^[:space:]\]|\\.)*)(\\|\\\\|\/)?]]patsubst(]_header_file[,[\.],[\\.])[[([[:space:]].*)?$/\3/
+#         ^^^^^^^ The Make line begins with a target (which we don't care about)
+#                ^^^^^^^^^^^^ Ignore any spaces following it
+#                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Match any number of other dependencies
+#                                                              ^^^^^^^^^^^^^^^^^^^^^^ Match any path components for our dependency; note this is reference 3, which we are replacing with
+#                                                                                    ^^^^^^^^^^^^^ Accept the path ending in a backslash, a double-backslash (ie escaped), or a forward slash
+#                                                                                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The filename must match exactly (periods are escaped, since a normal period matches any character in regex)
+#                                                                                                                                      ^^^^^^^^^^^^^^^^^ Filename must be followed by a space, but after that we don't care; we still need to match it all so it gets replaced, however
+# Delete the line, but only if we failed to find the directory (t jumps past the d if we matched)
+        t
+        d
+]],[
+\s*\(#.*\)?$],[],[
+        \(.*\)],[ -e '\1'])
+dnl ^^^^^^^^^^^^^^^^^^^^^^^ Deletes comments and processes sed expressions into -e arguments
+    )
+
     AC_MSG_RESULT([${newinclpath}])
     if test "${newinclpath}" != ""; then
-      eval "$1=\"\$$1\"' -I${newinclpath}'"
+      eval "_result_var=\"\$_result_var\"' -I${newinclpath}'"
     fi
   fi
+  m4_popdef([_result_var],[_rel_path],[_header_file])
 ])

configure.ac

@@ -1075,6 +1075,68 @@ if test "$have_any_system" != "no"; then
   AC_DEFINE([HAVE_SYSTEM], [1], [Define to 1 if std::system or ::wsystem is available.])
 fi
 
+dnl Check for leveldb, only if explicitly requested
+LEVELDB_CPPFLAGS=
+LIBLEVELDB=
+LIBMEMENV=
+AC_ARG_WITH([system-leveldb],
+  [AS_HELP_STRING([--with-system-leveldb],
+  [Build with system LevelDB (default is no; DANGEROUS; NOT SUPPORTED)])],
+  [system_leveldb=$withval],
+  [system_leveldb=no]
+)
+if test x$system_leveldb != xno; then
+  LEVELDB_CPPFLAGS=
+  AC_CHECK_LIB([leveldb],[main],[
+    LIBLEVELDB=-lleveldb
+  ],[
+    AC_MSG_ERROR([leveldb library not found; using --with-system-leveldb is not supported anyway])
+  ])
+  AC_CHECK_HEADER([leveldb/filter_policy.h],[],[
+    AC_MSG_ERROR([LevelDB headers not found; using --with-system-leveldb is not supported anyway])
+  ])
+  AC_CHECK_HEADER([leveldb/helpers/memenv.h],[
+    AC_MSG_CHECKING([for memenv.h path])
+    BITCOIN_SUBDIR_TO_INCLUDE([LEVELDB_CPPFLAGS],[leveldb/helpers/],[memenv])
+  ],[
+    AC_CHECK_HEADER([memenv.h],[],[
+      AC_MSG_ERROR([LevelDB headers not found; using --with-system-leveldb is not supported anyway])
+    ])
+  ])
+
+  AC_MSG_CHECKING([library containing leveldb::NewMemEnv])
+  TEMP_LIBS="$LIBS"
+  TEMP_CPPFLAGS="$CPPFLAGS"
+  CPPFLAGS="$CPPFLAGS $LEVELDB_CPPFLAGS"
+  for searchlib in "" "-lmemenv" ERR; do
+    if test "x$searchlib" = "xERR"; then
+      AC_MSG_RESULT([no])
+      AC_MSG_ERROR([LevelDB's memenv helper not found; using --with-system-leveldb is not supported anyway])
+    fi
+    searchlib="$searchlib $LIBLEVELDB"
+    LIBS="$searchlib $TEMP_LIBS"
+    AC_LINK_IFELSE([AC_LANG_SOURCE([
+        #include <leveldb/env.h>
+        #include <memenv.h>
+
+        int main() {
+            leveldb::Env *myenv = leveldb::NewMemEnv(leveldb::Env::Default());
+            delete myenv;
+        }
+    ])],[
+      AC_MSG_RESULT([$searchlib])
+      LIBMEMENV="$searchlib"
+      break
+    ])
+  done
+  LIBS="$TEMP_LIBS"
+  CPPFLAGS="$TEMP_CPPFLAGS"
+fi
+AM_CONDITIONAL([EMBEDDED_LEVELDB],[test x$system_leveldb = xno])
+AC_SUBST(LEVELDB_CPPFLAGS)
+AC_SUBST(LIBLEVELDB)
+AC_SUBST(LIBMEMENV)
+
 dnl SUPPRESSED_CPPFLAGS=SUPPRESS_WARNINGS([$SOME_CPPFLAGS])
 dnl Replace -I with -isystem in $SOME_CPPFLAGS to suppress warnings from
 dnl headers from its include directories and return the result.

src/Makefile.am

@@ -1110,8 +1110,10 @@ endif
 
 include Makefile.minisketch.include
 
+if EMBEDDED_LEVELDB
 include Makefile.crc32c.include
 include Makefile.leveldb.include
+endif
 
 include Makefile.test_util.include
 include Makefile.test_fuzz.include

src/Makefile.leveldb.include

@@ -8,11 +8,11 @@ LIBMEMENV_INT  = leveldb/libmemenv.la
 noinst_LTLIBRARIES += $(LIBLEVELDB_INT)
 noinst_LTLIBRARIES += $(LIBMEMENV_INT)
 
-LIBLEVELDB = $(LIBLEVELDB_INT) $(LIBCRC32C)
-LIBMEMENV = $(LIBMEMENV_INT)
+LIBLEVELDB += $(LIBLEVELDB_INT) $(LIBCRC32C)
+LIBMEMENV += $(LIBMEMENV_INT)
 
-LEVELDB_CPPFLAGS =
 LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include
+LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv
 
 LEVELDB_CPPFLAGS_INT =
 LEVELDB_CPPFLAGS_INT += -I$(srcdir)/leveldb

src/dbwrapper.cpp

@@ -6,23 +6,26 @@
 
 #include <logging.h>
 #include <random.h>
+#include <node/interface_ui.h>
 #include <serialize.h>
 #include <span.h>
 #include <streams.h>
 #include <util/fs.h>
 #include <util/fs_helpers.h>
 #include <util/strencodings.h>
+#include <util/translation.h>
 
 #include <algorithm>
 #include <cassert>
 #include <cstdarg>
 #include <cstdint>
 #include <cstdio>
+#include <leveldb/c.h>
 #include <leveldb/cache.h>
 #include <leveldb/db.h>
 #include <leveldb/env.h>
 #include <leveldb/filter_policy.h>
-#include <leveldb/helpers/memenv/memenv.h>
+#include <memenv.h>
 #include <leveldb/iterator.h>
 #include <leveldb/options.h>
 #include <leveldb/slice.h>
@@ -51,6 +54,42 @@ static void HandleError(const leveldb::Status& status)
     throw dbwrapper_error(errmsg);
 }
 
+bool dbwrapper_SanityCheck()
+{
+    unsigned long header_version = (leveldb::kMajorVersion << 16) | leveldb::kMinorVersion;
+    unsigned long library_version = (leveldb_major_version() << 16) | leveldb_minor_version();
+
+    if (header_version != library_version) {
+        InitError(Untranslated(strprintf("Compiled with LevelDB %d.%d, but linked with LevelDB %d.%d (incompatible).",
+            leveldb::kMajorVersion, leveldb::kMinorVersion,
+            leveldb_major_version(), leveldb_minor_version()
+        )));
+        return false;
+    }
+
+    return true;
+}
+
+#ifndef WIN32
+namespace leveldb {
+class EnvPosixTestHelper {
+    static void SetReadOnlyMMapLimit(int limit);
+public:
+    static inline void SetReadOnlyMMapLimitForBitcoin(int limit) { SetReadOnlyMMapLimit(limit); }
+};
+}
+
+class BitcoinLevelDBInit {
+public:
+    BitcoinLevelDBInit() {
+        if (sizeof(void*) >= 8) {
+            leveldb::EnvPosixTestHelper::SetReadOnlyMMapLimitForBitcoin(4096);
+        }
+    }
+};
+static BitcoinLevelDBInit g_bitcoin_leveldb_init;
+#endif
+
 class CBitcoinLevelDBLogger : public leveldb::Logger {
 public:
     // This code is adapted from posix_logger.h, which is why it is using vsprintf.

src/dbwrapper.h

@@ -20,6 +20,8 @@
 #include <string>
 #include <vector>
 
+bool dbwrapper_SanityCheck();
+
 static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64;
 static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024;
 

src/kernel/checks.cpp

@@ -4,6 +4,7 @@
 
 #include <kernel/checks.h>
 
+#include <dbwrapper.h>
 #include <random.h>
 #include <util/result.h>
 #include <util/translation.h>
@@ -14,6 +15,10 @@ namespace kernel {
 
 util::Result<void> SanityChecks(const Context&)
 {
+    if (!dbwrapper_SanityCheck()) {
+        return util::Error{Untranslated("Database sanity check failure. Aborting.")};
+    }
+
     if (!Random_SanityCheck()) {
         return util::Error{Untranslated("OS cryptographic RNG sanity check failure. Aborting.")};
     }

src/test/sanity_tests.cpp

@@ -2,6 +2,7 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+#include <dbwrapper.h>
 #include <key.h>
 #include <test/util/setup_common.h>
 
@@ -11,6 +12,7 @@ BOOST_FIXTURE_TEST_SUITE(sanity_tests, BasicTestingSetup)
 
 BOOST_AUTO_TEST_CASE(basic_sanity)
 {
+  BOOST_CHECK_MESSAGE(dbwrapper_SanityCheck() == true, "dbwrapper sanity test");
   BOOST_CHECK_MESSAGE(ECC_InitSanityCheck() == true, "secp256k1 sanity test");
 }