Merge #36: A few improvements to prepare for subtree in Bitcoin Core

1e96b67 Improve MinGW building (Pieter Wuille)
67a67c2 Hide non-API library symbols (Pieter Wuille)
281f811 Add MSVC CountBits implementation using _BitScanReverse (Pieter Wuille)
b21917d Work around clmul msan bug in clang < 11 (Pieter Wuille)
1261e9e CPUID support on MSVC (Pieter Wuille)
ec49c07 A few fixes to prepare for MSVC building (Pieter Wuille)

Pull request description:

  This includes various build system and compatibility improvements discovered in the process of subtreeing this into Bitcoin Core (see bitcoin/bitcoin#21859).

  * A few compatibility issues suggested [here](bitcoin/bitcoin#21859 (comment)) by sipsorcery.
  * Support for __builtin_clz equivalent on MSVC
  * Support for CPUID detection on MSVC
  * Prevent internal symbols from being exported
  * Make old MinGW CI work (-all-static instead of -static needed).

ACKs for top commit:
  gmaxwell:
    utACK 1e96b67   (I also tested it, but not with mingw or msvc, so it doesn't count)

Tree-SHA512: 83d457f39940ba9b6161a45181a8fb8853b793355eac61b7327f48c03e4261fadf06e7b05668e2f138236e520530ec2fd5ddff68afeccad3e4a3d3f184c5e25e

Author: sipa

.cirrus.yml

@@ -146,7 +146,6 @@ task:
   env:
     EXEC_CMD: wine
     HOST: x86_64-w64-mingw32
-    CXXFLAGS: -O2 -static -static-libgcc -static-libstdc++
     BUILD:
   << : *MERGE_BASE
   test_script:

Makefile.am

@@ -65,19 +65,17 @@ endif
 bench_SOURCES = $(MINISKETCH_BENCH_SOURCES_INT)
 bench_CPPFLAGS = $(AM_CPPFLAGS) $(RELEASE_DEFINES)
 bench_LDADD = $(LIBMINISKETCH)
-bench_LDFLAGS = $(AM_LDFLAGS) -static
+bench_LDFLAGS = $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
 
 test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT)
 test_CPPFLAGS = $(AM_CPPFLAGS) $(RELEASE_DEFINES)
-test_CXXFLAGS = $(AM_CXXFLAGS)
 test_LDADD = $(LIBMINISKETCH)
-test_LDFLAGS = $(AM_LDFLAGS) -static
+test_LDFLAGS = $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
 
 test_verify_SOURCES = $(MINISKETCH_TEST_SOURCES_INT)
 test_verify_CPPFLAGS = $(AM_CPPFLAGS) $(VERIFY_DEFINES)
-test_verify_CXXFLAGS = $(AM_CXXFLAGS)
 test_verify_LDADD = $(LIBMINISKETCH_VERIFY)
-test_verify_LDFLAGS = $(AM_LDFLAGS) -static
+test_verify_LDFLAGS = $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
 
 EXTRA_DIST=
 EXTRA_DIST += LICENSE

build-aux/m4/ax_check_link_flag.m4

@@ -0,0 +1,53 @@
+# ===========================================================================
+#    https://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_CHECK_LINK_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT])
+#
+# DESCRIPTION
+#
+#   Check whether the given FLAG works with the linker or gives an error.
+#   (Warnings, however, are ignored)
+#
+#   ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on
+#   success/failure.
+#
+#   If EXTRA-FLAGS is defined, it is added to the linker's default flags
+#   when the check is done.  The check is thus made with the flags: "LDFLAGS
+#   EXTRA-FLAGS FLAG".  This can for example be used to force the linker to
+#   issue an error when a bad flag is given.
+#
+#   INPUT gives an alternative input source to AC_LINK_IFELSE.
+#
+#   NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this
+#   macro in sync with AX_CHECK_{PREPROC,COMPILE}_FLAG.
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Guido U. Draheim <[email protected]>
+#   Copyright (c) 2011 Maarten Bosmans <[email protected]>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved.  This file is offered as-is, without any
+#   warranty.
+
+#serial 6
+
+AC_DEFUN([AX_CHECK_LINK_FLAG],
+[AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF
+AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_ldflags_$4_$1])dnl
+AC_CACHE_CHECK([whether the linker accepts $1], CACHEVAR, [
+  ax_check_save_flags=$LDFLAGS
+  LDFLAGS="$LDFLAGS $4 $1"
+  AC_LINK_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])],
+    [AS_VAR_SET(CACHEVAR,[yes])],
+    [AS_VAR_SET(CACHEVAR,[no])])
+  LDFLAGS=$ax_check_save_flags])
+AS_VAR_IF(CACHEVAR,yes,
+  [m4_default([$2], :)],
+  [m4_default([$3], :)])
+AS_VAR_POPDEF([CACHEVAR])dnl
+])dnl AX_CHECK_LINK_FLAGS

configure.ac

@@ -14,6 +14,15 @@ AC_LANG([C++])
 
 AC_PATH_PROG(CCACHE,ccache)
 
+saved_CXXFLAGS="$CXXFLAGS"
+CXXFLAGS="-fvisibility=hidden $CXXFLAGS"
+AC_MSG_CHECKING([if ${CXX} supports -fvisibility=hidden])
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])],
+    [ AC_MSG_RESULT([yes]) ],
+    [ AC_MSG_RESULT([no])
+      CXXFLAGS="$saved_CXXFLAGS"
+    ])
+
 AC_ARG_ENABLE([ccache],
   [AS_HELP_STRING([--disable-ccache],
   [do not use ccache for building (default is to use if found)])],
@@ -88,6 +97,19 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [
   ]
 )
 
+AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]],[LDFLAGS="-Wl,--exclude-libs,ALL $LDFLAGS"])
+
+case $host in
+  *mingw*)
+     dnl -static is interpreted by libtool, where it has a different meaning.
+     dnl In libtool-speak, it's -all-static.
+     AX_CHECK_LINK_FLAG([[-static]],[LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static"])
+     ;;
+  *)
+     AX_CHECK_LINK_FLAG([[-static]],[LIBTOOL_APP_LDFLAGS="-static"])
+     ;;
+esac
+
 AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]])
 ## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all
 ## unknown options if any other warning is produced. Test the -Wfoo case, and
@@ -125,6 +147,7 @@ AC_SUBST(WARN_CXXFLAGS)
 AC_SUBST(NOWARN_CXXFLAGS)
 AC_SUBST(VERIFY_DEFINES)
 AC_SUBST(RELEASE_DEFINES)
+AC_SUBST(LIBTOOL_APP_LDFLAGS)
 AM_CONDITIONAL([ENABLE_CLMUL],[test x$enable_clmul = xyes])
 AM_CONDITIONAL([USE_BENCHMARK], [test x"$use_benchmark" = x"yes"])
 AM_CONDITIONAL([USE_TESTS], [test x"$use_tests" != x"no"])

include/minisketch.h

@@ -3,7 +3,26 @@
 
 #include <stdint.h>
 #include <stdlib.h>
-#include <unistd.h>
+
+#ifdef _MSC_VER
+#  include <compat.h>
+#else
+#  include <unistd.h>
+#endif
+
+#ifndef MINISKETCH_API
+# if defined(_WIN32)
+#  ifdef MINISKETCH_BUILD
+#   define MINISKETCH_API __declspec(dllexport)
+#  else
+#   define MINISKETCH_API
+#  endif
+# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(MINISKETCH_BUILD)
+#  define MINISKETCH_API __attribute__ ((visibility ("default")))
+# else
+#  define MINISKETCH_API
+# endif
+#endif
 
 #ifdef __cplusplus
 #  if __cplusplus >= 201103L
@@ -21,7 +40,7 @@ extern "C" {
 typedef struct minisketch minisketch;
 
 /** Determine whether support for elements of `bits` bits was compiled in. */
-int minisketch_bits_supported(uint32_t bits);
+MINISKETCH_API int minisketch_bits_supported(uint32_t bits);
 
 /** Determine the maximum number of implementations available.
  *
@@ -32,13 +51,13 @@ int minisketch_bits_supported(uint32_t bits);
  * function call, inclusive. Note that not every combination of implementation
  * and element size may exist (see further).
 */
-uint32_t minisketch_implementation_max(void);
+MINISKETCH_API uint32_t minisketch_implementation_max(void);
 
 /** Determine if the a combination of bits and implementation number is available.
  *
  * Returns 1 if it is, 0 otherwise.
  */
-int minisketch_implementation_supported(uint32_t bits, uint32_t implementation);
+MINISKETCH_API int minisketch_implementation_supported(uint32_t bits, uint32_t implementation);
 
 /** Construct a sketch for a given element size, implementation and capacity.
  *
@@ -49,16 +68,16 @@ int minisketch_implementation_supported(uint32_t bits, uint32_t implementation);
  *
  * If the result is not NULL, it must be destroyed using minisketch_destroy.
  */
-minisketch* minisketch_create(uint32_t bits, uint32_t implementation, size_t capacity);
+MINISKETCH_API minisketch* minisketch_create(uint32_t bits, uint32_t implementation, size_t capacity);
 
 /** Get the element size of a sketch in bits. */
-uint32_t minisketch_bits(const minisketch* sketch);
+MINISKETCH_API uint32_t minisketch_bits(const minisketch* sketch);
 
 /** Get the capacity of a sketch. */
-size_t minisketch_capacity(const minisketch* sketch);
+MINISKETCH_API size_t minisketch_capacity(const minisketch* sketch);
 
 /** Get the implementation of a sketch. */
-uint32_t minisketch_implementation(const minisketch* sketch);
+MINISKETCH_API uint32_t minisketch_implementation(const minisketch* sketch);
 
 /** Set the seed for randomizing algorithm choices to a fixed value.
  *
@@ -71,28 +90,28 @@ uint32_t minisketch_implementation(const minisketch* sketch);
  * When seed is -1, a fixed internal value with predictable behavior is
  * used. It is only intended for testing.
  */
-void minisketch_set_seed(minisketch* sketch, uint64_t seed);
+MINISKETCH_API void minisketch_set_seed(minisketch* sketch, uint64_t seed);
 
 /** Clone a sketch.
  *
  * The result must be destroyed using minisketch_destroy.
  */
-minisketch* minisketch_clone(const minisketch* sketch);
+MINISKETCH_API minisketch* minisketch_clone(const minisketch* sketch);
 
 /** Destroy a sketch.
  *
  * The pointer that was passed in may not be used anymore afterwards.
  */
-void minisketch_destroy(minisketch* sketch);
+MINISKETCH_API void minisketch_destroy(minisketch* sketch);
 
 /** Compute the size in bytes for serializing a given sketch. */
-size_t minisketch_serialized_size(const minisketch* sketch);
+MINISKETCH_API size_t minisketch_serialized_size(const minisketch* sketch);
 
 /** Serialize a sketch to bytes. */
-void minisketch_serialize(const minisketch* sketch, unsigned char* output);
+MINISKETCH_API void minisketch_serialize(const minisketch* sketch, unsigned char* output);
 
 /** Deserialize a sketch from bytes. */
-void minisketch_deserialize(minisketch* sketch, const unsigned char* input);
+MINISKETCH_API void minisketch_deserialize(minisketch* sketch, const unsigned char* input);
 
 /** Add an element to a sketch.
  * 
@@ -107,7 +126,7 @@ void minisketch_deserialize(minisketch* sketch, const unsigned char* input);
  *
  * Note that adding the same element a second time removes it again.
  */
-void minisketch_add_uint64(minisketch* sketch, uint64_t element);
+MINISKETCH_API void minisketch_add_uint64(minisketch* sketch, uint64_t element);
 
 /** Merge the elements of another sketch into this sketch.
  *
@@ -125,7 +144,7 @@ void minisketch_add_uint64(minisketch* sketch, uint64_t element);
  * of two sketches with the same element size and capacity by performing a bitwise XOR
  * of the serializations.
  */
-size_t minisketch_merge(minisketch* sketch, const minisketch* other_sketch);
+MINISKETCH_API size_t minisketch_merge(minisketch* sketch, const minisketch* other_sketch);
 
 /** Decode a sketch.
  *
@@ -134,7 +153,7 @@ size_t minisketch_merge(minisketch* sketch, const minisketch* other_sketch);
  *
  * The return value is the number of decoded elements, or -1 if decoding failed.
  */
-ssize_t minisketch_decode(const minisketch* sketch, size_t max_elements, uint64_t* output);
+MINISKETCH_API ssize_t minisketch_decode(const minisketch* sketch, size_t max_elements, uint64_t* output);
 
 /** Compute the capacity needed to achieve a certain rate of false positives.
  *
@@ -148,7 +167,7 @@ ssize_t minisketch_decode(const minisketch* sketch, size_t max_elements, uint64_
  * function computes the necessary capacity. It is only guaranteed to be
  * accurate up to fpbits=256.
  */
-size_t minisketch_compute_capacity(uint32_t bits, size_t max_elements, uint32_t fpbits);
+MINISKETCH_API size_t minisketch_compute_capacity(uint32_t bits, size_t max_elements, uint32_t fpbits);
 
 /** Compute what max_elements can be decoded for a certain rate of false positives.
  *
@@ -166,7 +185,7 @@ size_t minisketch_compute_capacity(uint32_t bits, size_t max_elements, uint32_t
  * chance of 1 in 2^18.5). Therefore, minisketch_compute_max_elements with
  * capacity=9 will return 9.
  */
-size_t minisketch_compute_max_elements(uint32_t bits, size_t capacity, uint32_t fpbits);
+MINISKETCH_API size_t minisketch_compute_max_elements(uint32_t bits, size_t capacity, uint32_t fpbits);
 
 #ifdef __cplusplus
 }

src/fields/clmul_common_impl.h

@@ -8,14 +8,27 @@
 #define _MINISKETCH_FIELDS_CLMUL_COMMON_IMPL_H_ 1
 
 #include <stdint.h>
-#include <x86intrin.h>
+#include <immintrin.h>
 
 #include "../int_utils.h"
 #include "../lintrans.h"
 
 namespace {
 
-template<typename I, int BITS, I MOD> I MulWithClMulReduce(I a, I b)
+// The memory sanitizer in clang < 11 cannot reason through _mm_clmulepi64_si128 calls.
+// Disable memory sanitization in the functions using them for those compilers.
+#if defined(__clang__) && (__clang_major__ < 11)
+#  if defined(__has_feature)
+#    if __has_feature(memory_sanitizer)
+#      define NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory")))
+#    endif
+#  endif
+#endif
+#ifndef NO_SANITIZE_MEMORY
+#  define NO_SANITIZE_MEMORY
+#endif
+
+template<typename I, int BITS, I MOD> NO_SANITIZE_MEMORY I MulWithClMulReduce(I a, I b)
 {
     static constexpr I MASK = Mask<BITS, I>();
 
@@ -52,7 +65,7 @@ template<typename I, int BITS, I MOD> I MulWithClMulReduce(I a, I b)
     }
 }
 
-template<typename I, int BITS, int POS> I MulTrinomial(I a, I b)
+template<typename I, int BITS, int POS> NO_SANITIZE_MEMORY I MulTrinomial(I a, I b)
 {
     static constexpr I MASK = Mask<BITS, I>();
 

src/int_utils.h

@@ -13,6 +13,10 @@
 #include <algorithm>
 #include <type_traits>
 
+#ifdef _MSC_VER
+#  include <intrin.h>
+#endif
+
 template<int bits>
 static constexpr inline uint64_t Rot(uint64_t x) { return (x << bits) | (x >> (64 - bits)); }
 
@@ -135,6 +139,17 @@ static inline int CountBits(I val, int max) {
     } else {
         return std::numeric_limits<unsigned long long>::digits - __builtin_clzll(val);
     }
+#elif _MSC_VER
+    (void)max;
+    unsigned long index;
+    unsigned char ret;
+    if (std::numeric_limits<I>::digits <= 32) {
+        ret = _BitScanReverse(&index, val);
+    } else {
+        ret = _BitScanReverse64(&index, val);
+    }
+    if (!ret) return 0;
+    return index;
 #else
     while (max && (val >> (max - 1) == 0)) --max;
     return max;