Merge #17595: guix: Enable building for x86_64-w64-mingw32 target

a35e323 guix: Appease travis. (Carl Dong)
0b66d22 guix: Use gcc-9 for mingw-w64 instead of 8 (Carl Dong)
ba0b99b guix: Don't set MINGW_HAS_SECURE_API CFLAG in depends (Carl Dong)
93439a7 guix: Bump to upstream commit with mingw-w64 changes (Carl Dong)
35a9679 guix: Check mingw symbols, improve SSP fix docs (Carl Dong)
449d8fe guix: Expand on INT trap message (Carl Dong)
3f1f03c guix: Spelling fixes (Carl Dong)
ff821dd guix: Reinstate make-ssp-fixed-gcc (Carl Dong)
360a9e0 guix: Bump time-machine for mingw-w64 patches (Carl Dong)
93e41b7 guix: Use gcc-8 for mingw-w64 instead of 7 (Carl Dong)
ef4f7e4 guix: Set the well-known timezone env var (Carl Dong)
acf4b3b guix: Make x86_64-w64-mingw32 builds reproducible (Carl Dong)
c4cce00 guix: Remove dead links from README. (Carl Dong)
df953a4 guix: Appease shellcheck. (Carl Dong)
91897c9 guix: Improve guix-build.sh documentation (Carl Dong)
570d769 guix: Build support for Windows (Carl Dong)

Pull request description:

  ~~Based on: bitcoin/bitcoin#16519
  Based on: #17933 (Time Machines are... shall we say... superior 😁)

  This PR allows us to perform Guix builds for the `x86_64-w64-mingw32` target. We do this _without_ splitting up the build script like we do in Gitian by using this newfangled alien technology called `case` statements. (This is WIP and might be changed to `if` statements soon)

ACKs for top commit:
  fanquake:
    ACK a35e323 2/3

Tree-SHA512: c471951c23eb2cda919a71285d8b8f2580cb20f09d5db17b53e13dbd8813e01b3e7a83ea848e4913fd0f2bc12c6c133c5f76b54e65c0d89fed4dfd2e0be19875

Author: fanquake

contrib/guix/README.md

@@ -220,8 +220,6 @@ repository and will likely put one up soon.
 [guix/env-setup]: https://www.gnu.org/software/guix/manual/en/html_node/Build-Environment-Setup.html
 [guix/substitutes]: https://www.gnu.org/software/guix/manual/en/html_node/Substitutes.html
 [guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html
-[guix/inferiors]: https://www.gnu.org/software/guix/manual/en/html_node/Inferiors.html
-[guix/channels]: https://www.gnu.org/software/guix/manual/en/html_node/Channels.html
 [guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html
 
 [debian/guix-package]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=850644

contrib/guix/guix-build.sh

@@ -13,33 +13,106 @@ make -C "${PWD}/depends" -j"$MAX_JOBS" download ${V:+V=1} ${SOURCES_PATH:+SOURCE
 # Determine the reference time used for determinism (overridable by environment)
 SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}"
 
+# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility
+# across time.
 time-machine() {
     guix time-machine --url=https://github.com/dongcarl/guix.git \
-                      --commit=b3a7c72c8b2425f8ddb0fc6e3b1caeed40f86dee \
+                      --commit=b066c25026f21fb57677aa34692a5034338e7ee3 \
                       -- "$@"
 }
 
-# Deterministically build Bitcoin Core for HOSTs (overriable by environment)
-for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu}; do
+# Function to be called when building for host ${1} and the user interrupts the
+# build
+int_trap() {
+cat << EOF
+** INT received while building ${1}, you may want to clean up the relevant
+   output, deploy, and distsrc-* directories before rebuilding
+
+Hint: To blow everything away, you may want to use:
+
+  $ git clean -xdff --exclude='/depends/SDKs/*'
+
+Specifically, this will remove all files without an entry in the index,
+excluding the SDK directory. Practically speaking, this means that all ignored
+and untracked files and directories will be wiped, allowing you to start anew.
+EOF
+}
+
+# Deterministically build Bitcoin Core for HOSTs (overridable by environment)
+# shellcheck disable=SC2153
+for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32}; do
 
     # Display proper warning when the user interrupts the build
-    trap 'echo "** INT received while building ${host}, you may want to clean up the relevant output and distsrc-* directories before rebuilding"' INT
-
-    # Run the build script 'contrib/guix/libexec/build.sh' in the build
-    # container specified by 'contrib/guix/manifest.scm'
-    # shellcheck disable=SC2086
-    time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \
-                             --container \
-                             --pure \
-                             --no-cwd \
-                             --share="$PWD"=/bitcoin \
-                             ${SOURCES_PATH:+--share="$SOURCES_PATH"} \
-                             ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \
-                             -- env HOST="$host" \
-                                    MAX_JOBS="$MAX_JOBS" \
-                                    SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \
-                                    ${V:+V=1} \
-                                    ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \
-                                  bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh"
+    trap 'int_trap ${host}' INT
+
+    (
+        # Required for 'contrib/guix/manifest.scm' to output the right manifest
+        # for the particular $HOST we're building for
+        export HOST="$host"
+
+        # Run the build script 'contrib/guix/libexec/build.sh' in the build
+        # container specified by 'contrib/guix/manifest.scm'.
+        #
+        # Explanation of `guix environment` flags:
+        #
+        #   --container        run command within an isolated container
+        #
+        #     Running in an isolated container minimizes build-time differences
+        #     between machines and improves reproducibility
+        #
+        #   --pure             unset existing environment variables
+        #
+        #     Same rationale as --container
+        #
+        #   --no-cwd           do not share current working directory with an
+        #                      isolated container
+        #
+        #     When --container is specified, the default behavior is to share
+        #     the current working directory with the isolated container at the
+        #     same exact path (e.g. mapping '/home/satoshi/bitcoin/' to
+        #     '/home/satoshi/bitcoin/'). This means that the $PWD inside the
+        #     container becomes a source of irreproducibility. --no-cwd disables
+        #     this behaviour.
+        #
+        #   --share=SPEC       for containers, share writable host file system
+        #                      according to SPEC
+        #
+        #   --share="$PWD"=/bitcoin
+        #
+        #                     maps our current working directory to /bitcoin
+        #                     inside the isolated container, which we later cd
+        #                     into.
+        #
+        #     While we don't want to map our current working directory to the
+        #     same exact path (as this introduces irreproducibility), we do want
+        #     it to be at a _fixed_ path _somewhere_ inside the isolated
+        #     container so that we have something to build. '/bitcoin' was
+        #     chosen arbitrarily.
+        #
+        #   ${SOURCES_PATH:+--share="$SOURCES_PATH"}
+        #
+        #                     make the downloaded depends sources path available
+        #                     inside the isolated container
+        #
+        #     The isolated container has no network access as it's in a
+        #     different network namespace from the main machine, so we have to
+        #     make the downloaded depends sources available to it. The sources
+        #     should have been downloaded prior to this invocation.
+        #
+        # shellcheck disable=SC2086
+        time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \
+                                 --container \
+                                 --pure \
+                                 --no-cwd \
+                                 --share="$PWD"=/bitcoin \
+                                 ${SOURCES_PATH:+--share="$SOURCES_PATH"} \
+                                 ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \
+                                 -- env HOST="$host" \
+                                        MAX_JOBS="$MAX_JOBS" \
+                                        SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \
+                                        ${V:+V=1} \
+                                        ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \
+                                      bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh"
+    )
 
 done

contrib/guix/libexec/build.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 export LC_ALL=C
 set -e -o pipefail
+export TZ=UTC
 
 # Check that environment variables assumed to be set by the environment are set
 echo "Building for platform triple ${HOST:?not set} with reference timestamp ${SOURCE_DATE_EPOCH:?not set}..."
@@ -36,23 +37,41 @@ store_path() {
               --expression='s|"[[:space:]]*$||'
 }
 
-# Determine output paths to use in CROSS_* environment variables
-CROSS_GLIBC="$(store_path glibc-cross-${HOST})"
-CROSS_GLIBC_STATIC="$(store_path glibc-cross-${HOST} static)"
-CROSS_KERNEL="$(store_path linux-libre-headers-cross-${HOST})"
-CROSS_GCC="$(store_path gcc-cross-${HOST})"
-CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories...
-CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one)
-
 # Set environment variables to point Guix's cross-toolchain to the right
 # includes/libs for $HOST
-#
-# NOTE: CROSS_C_INCLUDE_PATH is missing ${CROSS_GCC_LIB}/include-fixed, because
-# the limits.h in it is missing a '#include_next <limits.h>'
-#
-export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include"
-export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}"
-export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib"
+case "$HOST" in
+    *mingw*)
+        # Determine output paths to use in CROSS_* environment variables
+        CROSS_GLIBC="$(store_path "mingw-w64-x86_64-winpthreads")"
+        CROSS_GCC="$(store_path "gcc-cross-${HOST}")"
+        CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories...
+        CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one)
+
+        NATIVE_GCC="$(store_path gcc-glibc-2.27-toolchain)"
+        export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64"
+        export CPATH="${NATIVE_GCC}/include"
+
+        export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include"
+        export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}"
+        export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib"
+        ;;
+    *linux*)
+        CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")"
+        CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)"
+        CROSS_KERNEL="$(store_path "linux-libre-headers-cross-${HOST}")"
+        CROSS_GCC="$(store_path "gcc-cross-${HOST}")"
+        CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories...
+        CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one)
+
+        # NOTE: CROSS_C_INCLUDE_PATH is missing ${CROSS_GCC_LIB}/include-fixed, because
+        # the limits.h in it is missing a '#include_next <limits.h>'
+        export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include"
+        export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}"
+        export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib"
+        ;;
+    *)
+        exit 1 ;;
+esac
 
 # Sanity check CROSS_*_PATH directories
 IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}"
@@ -74,16 +93,20 @@ export GUIX_LD_WRAPPER_DISABLE_RPATH=yes
 [ -e /usr/bin/env ]  || ln -s --no-dereference "$(command -v env)"  /usr/bin/env
 
 # Determine the correct value for -Wl,--dynamic-linker for the current $HOST
-glibc_dynamic_linker=$(
-    case "$HOST" in
-        i686-linux-gnu)      echo /lib/ld-linux.so.2 ;;
-        x86_64-linux-gnu)    echo /lib64/ld-linux-x86-64.so.2 ;;
-        arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;;
-        aarch64-linux-gnu)   echo /lib/ld-linux-aarch64.so.1 ;;
-        riscv64-linux-gnu)   echo /lib/ld-linux-riscv64-lp64d.so.1 ;;
-        *)                   exit 1 ;;
-    esac
-)
+case "$HOST" in
+    *linux*)
+        glibc_dynamic_linker=$(
+            case "$HOST" in
+                i686-linux-gnu)      echo /lib/ld-linux.so.2 ;;
+                x86_64-linux-gnu)    echo /lib64/ld-linux-x86-64.so.2 ;;
+                arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;;
+                aarch64-linux-gnu)   echo /lib/ld-linux-aarch64.so.1 ;;
+                riscv64-linux-gnu)   echo /lib/ld-linux-riscv64-lp64d.so.1 ;;
+                *)                   exit 1 ;;
+            esac
+        )
+        ;;
+esac
 
 # Environment variables for determinism
 export QT_RCC_TEST=1
@@ -136,11 +159,27 @@ DISTNAME="$(basename "$SOURCEDIST" '.tar.gz')"
 # Binary Tarball Building #
 ###########################
 
-# Similar flags to Gitian
-CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests"
-HOST_CFLAGS="-O2 -g -ffile-prefix-map=${PWD}=."
-HOST_CXXFLAGS="-O2 -g -ffile-prefix-map=${PWD}=."
-HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++"
+# CONFIGFLAGS
+CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests"
+case "$HOST" in
+    *linux*) CONFIGFLAGS+=" --enable-glibc-back-compat" ;;
+esac
+
+# CFLAGS
+HOST_CFLAGS="-O2 -g"
+case "$HOST" in
+    *linux*)  HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;;
+    *mingw*)  HOST_CFLAGS+=" -fno-ident" ;;
+esac
+
+# CXXFLAGS
+HOST_CXXFLAGS="$HOST_CFLAGS"
+
+# LDFLAGS
+case "$HOST" in
+    *linux*)  HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++" ;;
+    *mingw*)  HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;;
+esac
 
 # Make $HOST-specific native binaries from depends available in $PATH
 export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}"
@@ -160,7 +199,7 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}"
                     ${CONFIGFLAGS} \
                     CFLAGS="${HOST_CFLAGS}" \
                     CXXFLAGS="${HOST_CXXFLAGS}" \
-                    LDFLAGS="${HOST_LDFLAGS}"
+                    ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"}
 
     sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool
 
@@ -169,9 +208,21 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}"
 
     # Perform basic ELF security checks on a series of executables.
     make -C src --jobs=1 check-security ${V:+V=1}
-    # Check that executables only contain allowed gcc, glibc and libstdc++
-    # version symbols for Linux distro back-compatibility.
-    make -C src --jobs=1 check-symbols  ${V:+V=1}
+
+    case "$HOST" in
+        *linux*|*mingw*)
+            # Check that executables only contain allowed gcc, glibc and libstdc++
+            # version symbols for Linux distro back-compatibility.
+            make -C src --jobs=1 check-symbols  ${V:+V=1}
+            ;;
+    esac
+
+    # Make the os-specific installers
+    case "$HOST" in
+        *mingw*)
+            make deploy ${V:+V=1}
+            ;;
+    esac
 
     # Setup the directory where our Bitcoin Core build for HOST will be
     # installed. This directory will also later serve as the input for our
@@ -180,9 +231,21 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}"
     mkdir -p "${INSTALLPATH}"
     # Install built Bitcoin Core to $INSTALLPATH
     make install DESTDIR="${INSTALLPATH}" ${V:+V=1}
+
+    case "$HOST" in
+        *mingw*)
+            cp -f --target-directory="$OUTDIR" ./*-setup-unsigned.exe
+            ;;
+    esac
     (
         cd installed
 
+        case "$HOST" in
+            *mingw*)
+                mv --target-directory="$DISTNAME"/lib/ "$DISTNAME"/bin/*.dll
+                ;;
+        esac
+
         # Prune libtool and object archives
         find . -name "lib*.la" -delete
         find . -name "lib*.a" -delete
@@ -196,19 +259,60 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}"
             find "${DISTNAME}/lib" -type f -print0
         } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg
 
-        cp "${DISTSRC}/doc/README.md" "${DISTNAME}/"
+        case "$HOST" in
+            *mingw*)
+                cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt"
+                ;;
+            *linux*)
+                cp "${DISTSRC}/doc/README.md" "${DISTNAME}/"
+                ;;
+        esac
 
         # Finally, deterministically produce {non-,}debug binary tarballs ready
         # for release
-        find "${DISTNAME}" -not -name "*.dbg" -print0 \
-            | sort --zero-terminated \
-            | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \
-            | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \
-            || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 )
-        find "${DISTNAME}" -name "*.dbg" -print0 \
-            | sort --zero-terminated \
-            | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \
-            | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \
-            || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 )
+        case "$HOST" in
+            *mingw*)
+                find "${DISTNAME}" -not -name "*.dbg" -print0 \
+                    | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}"
+                find "${DISTNAME}" -not -name "*.dbg" \
+                    | sort \
+                    | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" \
+                    || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" && exit 1 )
+                find "${DISTNAME}" -name "*.dbg" -print0 \
+                    | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}"
+                find "${DISTNAME}" -name "*.dbg" \
+                    | sort \
+                    | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \
+                    || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" && exit 1 )
+                ;;
+            *linux*)
+                find "${DISTNAME}" -not -name "*.dbg" -print0 \
+                    | sort --zero-terminated \
+                    | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \
+                    | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \
+                    || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 )
+                find "${DISTNAME}" -name "*.dbg" -print0 \
+                    | sort --zero-terminated \
+                    | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \
+                    | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \
+                    || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 )
+                ;;
+        esac
     )
 )
+
+case "$HOST" in
+    *mingw*)
+        cp -rf --target-directory=. contrib/windeploy
+        (
+            cd ./windeploy
+            mkdir unsigned
+            cp --target-directory=unsigned/ "$OUTDIR"/bitcoin-*-setup-unsigned.exe
+            find . -print0 \
+                | sort --zero-terminated \
+                | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \
+                | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \
+                || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 )
+        )
+        ;;
+esac