Merge branch 'rm_minisketch-28+k' into rm_historical_relnotes_from_dist

Author: luke-jr

build-aux/m4/bitcoin_subdir_to_include.m4

@@ -5,14 +5,48 @@ dnl file COPYING or http://www.opensource.org/licenses/mit-license.php.
 dnl BITCOIN_SUBDIR_TO_INCLUDE([CPPFLAGS-VARIABLE-NAME],[SUBDIRECTORY-NAME],[HEADER-FILE])
 dnl SUBDIRECTORY-NAME must end with a path separator
 AC_DEFUN([BITCOIN_SUBDIR_TO_INCLUDE],[
-  if test "$2" = ""; then
+  m4_pushdef([_result_var],[$1])
+  m4_pushdef([_rel_path],[$2])
+  m4_pushdef([_header_file],[$3.h])
+  if test "[]_rel_path" = ""; then
     AC_MSG_RESULT([default])
   else
-    echo "#include <$2$3.h>" >conftest.cpp
-    newinclpath=`${CXXCPP} ${CPPFLAGS} -M conftest.cpp 2>/dev/null | [ tr -d '\\n\\r\\\\' | sed -e 's/^.*[[:space:]:]\(\/[^[:space:]]*\)]$3[\.h[[:space:]].*$/\1/' -e t -e d`]
+    echo '[#]include <'"_rel_path"'/_header_file>' >conftest.cpp
+    newinclpath=$(
+      ${CXXCPP} ${CPPFLAGS} -M conftest.cpp 2>/dev/null |
+      ${SED} -E m4_bpatsubsts([[
+        :build_line
+# If the line doesn't end with a backslash, it is complete; go on to process it
+        /\\$/!b have_complete_line
+# Otherwise, read the next line, and concatenate it to the current one with a space
+        N
+        s/\\\n/ /
+# Then go back and check for a trailing backslash again.
+        t build_line
+
+# When we get here, we have the completed line, with all continuations collapsed.
+        :have_complete_line
+        s/^[^:]*:[[:space:]]*(([^[:space:]\]|\\.)*[[:space:]])*(([^[:space:]\]|\\.)*)(\\|\\\\|\/)?]]patsubst(]_header_file[,[\.],[\\.])[[([[:space:]].*)?$/\3/
+#         ^^^^^^^ The Make line begins with a target (which we don't care about)
+#                ^^^^^^^^^^^^ Ignore any spaces following it
+#                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Match any number of other dependencies
+#                                                              ^^^^^^^^^^^^^^^^^^^^^^ Match any path components for our dependency; note this is reference 3, which we are replacing with
+#                                                                                    ^^^^^^^^^^^^^ Accept the path ending in a backslash, a double-backslash (ie escaped), or a forward slash
+#                                                                                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The filename must match exactly (periods are escaped, since a normal period matches any character in regex)
+#                                                                                                                                      ^^^^^^^^^^^^^^^^^ Filename must be followed by a space, but after that we don't care; we still need to match it all so it gets replaced, however
+# Delete the line, but only if we failed to find the directory (t jumps past the d if we matched)
+        t
+        d
+]],[
+\s*\(#.*\)?$],[],[
+        \(.*\)],[ -e '\1'])
+dnl ^^^^^^^^^^^^^^^^^^^^^^^ Deletes comments and processes sed expressions into -e arguments
+    )
+
     AC_MSG_RESULT([${newinclpath}])
     if test "${newinclpath}" != ""; then
-      eval "$1=\"\$$1\"' -I${newinclpath}'"
+      eval "_result_var=\"\$_result_var\"' -I${newinclpath}'"
     fi
   fi
+  m4_popdef([_result_var],[_rel_path],[_header_file])
 ])

ci/test/03_test_script.sh

@@ -143,7 +143,6 @@ du -sh "${PREVIOUS_RELEASES_DIR}"
 if [[ $HOST = *-mingw32 ]]; then
   # Generate all binaries, so that they can be wrapped
   make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1
-  make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1
   "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh"
 fi
 

configure.ac

@@ -1075,6 +1075,68 @@ if test "$have_any_system" != "no"; then
   AC_DEFINE([HAVE_SYSTEM], [1], [Define to 1 if std::system or ::wsystem is available.])
 fi
 
+dnl Check for leveldb, only if explicitly requested
+LEVELDB_CPPFLAGS=
+LIBLEVELDB=
+LIBMEMENV=
+AC_ARG_WITH([system-leveldb],
+  [AS_HELP_STRING([--with-system-leveldb],
+  [Build with system LevelDB (default is no; DANGEROUS; NOT SUPPORTED)])],
+  [system_leveldb=$withval],
+  [system_leveldb=no]
+)
+if test x$system_leveldb != xno; then
+  LEVELDB_CPPFLAGS=
+  AC_CHECK_LIB([leveldb],[main],[
+    LIBLEVELDB=-lleveldb
+  ],[
+    AC_MSG_ERROR([leveldb library not found; using --with-system-leveldb is not supported anyway])
+  ])
+  AC_CHECK_HEADER([leveldb/filter_policy.h],[],[
+    AC_MSG_ERROR([LevelDB headers not found; using --with-system-leveldb is not supported anyway])
+  ])
+  AC_CHECK_HEADER([leveldb/helpers/memenv.h],[
+    AC_MSG_CHECKING([for memenv.h path])
+    BITCOIN_SUBDIR_TO_INCLUDE([LEVELDB_CPPFLAGS],[leveldb/helpers/],[memenv])
+  ],[
+    AC_CHECK_HEADER([memenv.h],[],[
+      AC_MSG_ERROR([LevelDB headers not found; using --with-system-leveldb is not supported anyway])
+    ])
+  ])
+
+  AC_MSG_CHECKING([library containing leveldb::NewMemEnv])
+  TEMP_LIBS="$LIBS"
+  TEMP_CPPFLAGS="$CPPFLAGS"
+  CPPFLAGS="$CPPFLAGS $LEVELDB_CPPFLAGS"
+  for searchlib in "" "-lmemenv" ERR; do
+    if test "x$searchlib" = "xERR"; then
+      AC_MSG_RESULT([no])
+      AC_MSG_ERROR([LevelDB's memenv helper not found; using --with-system-leveldb is not supported anyway])
+    fi
+    searchlib="$searchlib $LIBLEVELDB"
+    LIBS="$searchlib $TEMP_LIBS"
+    AC_LINK_IFELSE([AC_LANG_SOURCE([
+        #include <leveldb/env.h>
+        #include <memenv.h>
+
+        int main() {
+            leveldb::Env *myenv = leveldb::NewMemEnv(leveldb::Env::Default());
+            delete myenv;
+        }
+    ])],[
+      AC_MSG_RESULT([$searchlib])
+      LIBMEMENV="$searchlib"
+      break
+    ])
+  done
+  LIBS="$TEMP_LIBS"
+  CPPFLAGS="$TEMP_CPPFLAGS"
+fi
+AM_CONDITIONAL([EMBEDDED_LEVELDB],[test x$system_leveldb = xno])
+AC_SUBST(LEVELDB_CPPFLAGS)
+AC_SUBST(LIBLEVELDB)
+AC_SUBST(LIBMEMENV)
+
 dnl SUPPRESSED_CPPFLAGS=SUPPRESS_WARNINGS([$SOME_CPPFLAGS])
 dnl Replace -I with -isystem in $SOME_CPPFLAGS to suppress warnings from
 dnl headers from its include directories and return the result.

contrib/guix/libexec/build.sh

@@ -196,7 +196,8 @@ GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz"
 # Create the source tarball if not already there
 if [ ! -e "$GIT_ARCHIVE" ]; then
     mkdir -p "$(dirname "$GIT_ARCHIVE")"
-    git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD
+    CONFIG_SITE="${BASEPREFIX}/${HOST}/share/config.site" \
+    contrib/guix/libexec/make_release_tarball.sh "${GIT_ARCHIVE}" "${DISTNAME}"
 fi
 
 mkdir -p "$OUTDIR"
@@ -239,8 +240,6 @@ mkdir -p "$DISTSRC"
     # Extract the source tarball
     tar --strip-components=1 -xf "${GIT_ARCHIVE}"
 
-    ./autogen.sh
-
     # Configure this DISTSRC for $HOST
     # shellcheck disable=SC2086
     env CONFIG_SITE="${BASEPREFIX}/${HOST}/share/config.site" \

contrib/guix/libexec/make_release_tarball.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# A helper script to generate source release tarball
+
+export LC_ALL=C
+set -ex
+
+[ "$#" -ge 2 ]
+
+GIT_ARCHIVE="$1"
+DISTNAME="$2"
+
+git archive --prefix="${DISTNAME}/" HEAD | tar -xp --exclude '*minisketch*'
+
+# Generate correct build info file from git, before we lose git
+GIT_BUILD_INFO="$(share/genbuild.sh /dev/stdout)"
+sed 's/\/\/ No build information available/'"${GIT_BUILD_INFO}"'/' -i "${DISTNAME}/share/genbuild.sh"
+
+cd "${DISTNAME}"
+
+./autogen.sh
+./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking
+make distclean
+
+cd ..
+tar \
+  --format=ustar \
+  --exclude autom4te.cache \
+  --exclude .deps \
+  --exclude .git \
+  --sort=name \
+  --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 \
+  --mtime="${REFERENCE_DATETIME}" \
+  -c "${DISTNAME}" | \
+  gzip -9n \
+  >"${GIT_ARCHIVE}"
+
+rm -rf "${DISTNAME}"

share/genbuild.sh

@@ -18,9 +18,14 @@ else
     exit 1
 fi
 
+# This checks that we are actually part of the intended git repository, and not just getting info about some unrelated git repository that the code happens to be in a directory under
+git_check_in_repo() {
+    ! { git status --porcelain -uall --ignored "$@" 2>/dev/null || echo '??'; } | grep -q '?'
+}
+
 GIT_TAG=""
 GIT_COMMIT=""
-if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ]; then
+if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ] && git_check_in_repo share/genbuild.sh; then
     # clean 'dirty' status of touched files that haven't been modified
     git diff >/dev/null 2>/dev/null
 
@@ -40,7 +45,9 @@ if [ -n "$GIT_TAG" ]; then
 elif [ -n "$GIT_COMMIT" ]; then
     NEWINFO="#define BUILD_GIT_COMMIT \"$GIT_COMMIT\""
 else
-    NEWINFO="// No build information available"
+    # NOTE: The NEWINFO line below this comment gets replaced by a string-match in contrib/guix/libexec/make_release_tarball.sh
+    # If changing it, update the script too!
+    NEWINFO='// No build information available'
 fi
 
 # only update build.h if necessary

src/Makefile.am

@@ -231,7 +231,6 @@ BITCOIN_CORE_H = \
   node/mempool_persist_args.h \
   node/miner.h \
   node/mini_miner.h \
-  node/minisketchwrapper.h \
   node/peerman_args.h \
   node/protocol_version.h \
   node/psbt.h \
@@ -436,7 +435,6 @@ libbitcoin_node_a_SOURCES = \
   node/mempool_persist_args.cpp \
   node/miner.cpp \
   node/mini_miner.cpp \
-  node/minisketchwrapper.cpp \
   node/peerman_args.cpp \
   node/psbt.cpp \
   node/timeoffsets.cpp \
@@ -1108,10 +1106,10 @@ endif
 	 echo "};"; \
 	} > "$@.new" && mv -f "$@.new" "$@"
 
-include Makefile.minisketch.include
-
+if EMBEDDED_LEVELDB
 include Makefile.crc32c.include
 include Makefile.leveldb.include
+endif
 
 include Makefile.test_util.include
 include Makefile.test_fuzz.include

src/Makefile.leveldb.include

@@ -8,11 +8,11 @@ LIBMEMENV_INT  = leveldb/libmemenv.la
 noinst_LTLIBRARIES += $(LIBLEVELDB_INT)
 noinst_LTLIBRARIES += $(LIBMEMENV_INT)
 
-LIBLEVELDB = $(LIBLEVELDB_INT) $(LIBCRC32C)
-LIBMEMENV = $(LIBMEMENV_INT)
+LIBLEVELDB += $(LIBLEVELDB_INT) $(LIBCRC32C)
+LIBMEMENV += $(LIBMEMENV_INT)
 
-LEVELDB_CPPFLAGS =
 LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include
+LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv
 
 LEVELDB_CPPFLAGS_INT =
 LEVELDB_CPPFLAGS_INT += -I$(srcdir)/leveldb

src/Makefile.test.include

@@ -114,7 +114,6 @@ BITCOIN_TESTS =\
   test/miner_tests.cpp \
   test/miniminer_tests.cpp \
   test/miniscript_tests.cpp \
-  test/minisketch_tests.cpp \
   test/multisig_tests.cpp \
   test/net_peer_connection_tests.cpp \
   test/net_peer_eviction_tests.cpp \
@@ -343,7 +342,6 @@ test_fuzz_fuzz_SOURCES = \
  test/fuzz/merkleblock.cpp \
  test/fuzz/message.cpp \
  test/fuzz/miniscript.cpp \
- test/fuzz/minisketch.cpp \
  test/fuzz/mini_miner.cpp \
  test/fuzz/muhash.cpp \
  test/fuzz/multiplication_overflow.cpp \

src/dbwrapper.cpp

@@ -6,23 +6,26 @@
 
 #include <logging.h>
 #include <random.h>
+#include <node/interface_ui.h>
 #include <serialize.h>
 #include <span.h>
 #include <streams.h>
 #include <util/fs.h>
 #include <util/fs_helpers.h>
 #include <util/strencodings.h>
+#include <util/translation.h>
 
 #include <algorithm>
 #include <cassert>
 #include <cstdarg>
 #include <cstdint>
 #include <cstdio>
+#include <leveldb/c.h>
 #include <leveldb/cache.h>
 #include <leveldb/db.h>
 #include <leveldb/env.h>
 #include <leveldb/filter_policy.h>
-#include <leveldb/helpers/memenv/memenv.h>
+#include <memenv.h>
 #include <leveldb/iterator.h>
 #include <leveldb/options.h>
 #include <leveldb/slice.h>
@@ -51,6 +54,42 @@ static void HandleError(const leveldb::Status& status)
     throw dbwrapper_error(errmsg);
 }
 
+bool dbwrapper_SanityCheck()
+{
+    unsigned long header_version = (leveldb::kMajorVersion << 16) | leveldb::kMinorVersion;
+    unsigned long library_version = (leveldb_major_version() << 16) | leveldb_minor_version();
+
+    if (header_version != library_version) {
+        InitError(Untranslated(strprintf("Compiled with LevelDB %d.%d, but linked with LevelDB %d.%d (incompatible).",
+            leveldb::kMajorVersion, leveldb::kMinorVersion,
+            leveldb_major_version(), leveldb_minor_version()
+        )));
+        return false;
+    }
+
+    return true;
+}
+
+#ifndef WIN32
+namespace leveldb {
+class EnvPosixTestHelper {
+    static void SetReadOnlyMMapLimit(int limit);
+public:
+    static inline void SetReadOnlyMMapLimitForBitcoin(int limit) { SetReadOnlyMMapLimit(limit); }
+};
+}
+
+class BitcoinLevelDBInit {
+public:
+    BitcoinLevelDBInit() {
+        if (sizeof(void*) >= 8) {
+            leveldb::EnvPosixTestHelper::SetReadOnlyMMapLimitForBitcoin(4096);
+        }
+    }
+};
+static BitcoinLevelDBInit g_bitcoin_leveldb_init;
+#endif
+
 class CBitcoinLevelDBLogger : public leveldb::Logger {
 public:
     // This code is adapted from posix_logger.h, which is why it is using vsprintf.