Windows binary release

CI: use both cores on GitHub Actions and set timeouts for the local testnet tests

Author: stefantalpalaru

.github/workflows/ci.yml

@@ -122,6 +122,21 @@ jobs:
             echo "CFLAGS=$CFLAGS" >> $GITHUB_ENV
           fi
 
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+          echo "ncpu=$ncpu" >> $GITHUB_ENV
+
       - name: Install build dependencies (Linux i386)
         if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
         run: |
@@ -218,23 +233,6 @@ jobs:
         shell: bash
         working-directory: nim-beacon-chain
         run: |
-          ncpu=
-          ext=
-          case '${{ runner.os }}' in
-          'Linux')
-            ncpu=$(nproc)
-            ;;
-          'macOS')
-            ncpu=$(sysctl -n hw.ncpu)
-            ;;
-          'Windows')
-            ncpu=$NUMBER_OF_PROCESSORS
-            ext=.exe
-            ;;
-          esac
-          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
-          echo "::set-output name=ncpu::$ncpu"
-
           make -j$ncpu ARCH_OVERRIDE=$PLATFORM CI_CACHE=NimBinaries V=1 update
 
       - name: Get latest fixtures commit hash
@@ -284,11 +282,11 @@ jobs:
         shell: bash
         working-directory: nim-beacon-chain
         run: |
-          ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000  --base-metrics-port 8008 -- --verify-finalization --discv5:no
+          ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000  --base-metrics-port 8008 --timeout 600 -- --verify-finalization --discv5:no
 
       - name: Run nim-beacon-chain testnet1 (mainnet)
         if: matrix.target.TEST_KIND == 'finalization-mainnet'
         shell: bash
         working-directory: nim-beacon-chain
         run: |
-          ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000 --base-metrics-port 8008 -- --verify-finalization --discv5:no
+          ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000 --base-metrics-port 8008 --timeout 2400 -- --verify-finalization --discv5:no

.github/workflows/release.yml

@@ -7,7 +7,7 @@ name: Upload Release Asset
 
 jobs:
   build-amd64:
-    name: AMD64 release asset
+    name: Linux AMD64 release asset
     runs-on: ubuntu-latest
     steps:
       - name: Create release
@@ -54,7 +54,7 @@ jobs:
           docker push statusim/nimbus-eth2:amd64-${TAG}
           docker push statusim/nimbus-eth2:amd64-latest
   build-arm64:
-    name: ARM64 release asset
+    name: Linux ARM64 release asset
     runs-on: ubuntu-latest
     steps:
       - name: Install packages
@@ -91,7 +91,7 @@ jobs:
           asset_name: ${{ steps.make_dist.outputs.linux_arm64_archive }}
           asset_content_type: application/gzip
   build-arm:
-    name: ARM release asset
+    name: Linux ARM release asset
     runs-on: ubuntu-latest
     steps:
       - name: Install packages
@@ -127,4 +127,34 @@ jobs:
           asset_path: ./dist/${{ steps.make_dist.outputs.linux_arm_archive }}
           asset_name: ${{ steps.make_dist.outputs.linux_arm_archive }}
           asset_content_type: application/gzip
+  build-win64:
+    name: Windows AMD64 release asset
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Build project
+        id: make_dist
+        run: |
+          make dist-win64
+          cd dist
+          echo "::set-output name=windows_amd64_archive::"$(echo nimbus-eth2_Windows_amd64_*.tar.gz)
+      - name: Fetch latest release
+        id: fetch_release
+        uses: InsonusK/[email protected]
+        with:
+          myToken: ${{ github.token }}
+          view_top: 1
+      - name: Check release version
+        run: |
+          [[ "refs/tags/${{ steps.fetch_release.outputs.tag_name }}" == "${{ github.ref }}" ]]
+      - name: Upload release asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: https://uploads.github.com/repos/${{ github.repository }}/releases/${{ steps.fetch_release.outputs.id }}/assets{?name,label}
+          asset_path: ./dist/${{ steps.make_dist.outputs.windows_amd64_archive }}
+          asset_name: ${{ steps.make_dist.outputs.windows_amd64_archive }}
+          asset_content_type: application/gzip
 

Makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020 Status Research & Development GmbH. Licensed under
+# Copyright (c) 2019-2021 Status Research & Development GmbH. Licensed under
 # either of:
 # - Apache License, version 2.0
 # - MIT license
@@ -81,6 +81,7 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 	dist-amd64 \
 	dist-arm64 \
 	dist-arm \
+	dist-win64 \
 	dist \
 	benchmarks
 
@@ -396,10 +397,15 @@ dist-arm:
 	MAKE="$(MAKE)" \
 		scripts/make_dist.sh arm
 
+dist-win64:
+	MAKE="$(MAKE)" \
+		scripts/make_dist.sh win64
+
 dist:
 	$(MAKE) dist-amd64
 	$(MAKE) dist-arm64
 	$(MAKE) dist-arm
+	$(MAKE) dist-win64
 
 #- this simple test will show any missing dynamically-linked Glibc symbols in the target distro
 dist-test:

beacon_chain/network_metadata.nim

@@ -1,3 +1,9 @@
+# Copyright (c) 2020-2021 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
 import
   tables, strutils, os,
   stew/shims/macros, nimcrypto/hash,
@@ -68,7 +74,7 @@ type
       incompatibilityDesc*: string
 
 const
-  eth2testnetsDir = currentSourcePath.parentDir / ".." / "vendor" / "eth2-testnets"
+  eth2testnetsDir = currentSourcePath.parentDir.replace('\\', '/') & "/../vendor/eth2-testnets"
 
 const presetValueLoaders = genExpr(nnkBracket):
   for constName in PresetValue:
@@ -107,12 +113,12 @@ proc loadEth2NetworkMetadata*(path: string): Eth2NetworkMetadata
                              {.raises: [CatchableError, Defect].} =
   try:
     let
-      genesisPath = path / "genesis.ssz"
-      genesisDepositsSnapshotPath = path / "genesis_deposit_contract_snapshot.ssz"
-      configPath = path / "config.yaml"
-      depositContractPath = path / "deposit_contract.txt"
-      depositContractBlockPath = path / "deposit_contract_block.txt"
-      bootstrapNodesPath = path / "bootstrap_nodes.txt"
+      genesisPath = path & "/genesis.ssz"
+      genesisDepositsSnapshotPath = path & "/genesis_deposit_contract_snapshot.ssz"
+      configPath = path & "/config.yaml"
+      depositContractPath = path & "/deposit_contract.txt"
+      depositContractBlockPath = path & "/deposit_contract_block.txt"
+      bootstrapNodesPath = path & "/bootstrap_nodes.txt"
 
       runtimePreset = if fileExists(configPath):
         extractRuntimePreset(configPath, readPresetFile(configPath))
@@ -164,27 +170,27 @@ proc loadEth2NetworkMetadata*(path: string): Eth2NetworkMetadata
                         incompatibilityDesc: err.msg)
 
 const
-  mainnetMetadataDir = eth2testnetsDir / "shared" / "mainnet"
+  mainnetMetadataDir = eth2testnetsDir & "/shared/mainnet"
 
   mainnetMetadata* = when const_preset == "mainnet":
     Eth2NetworkMetadata(
       incompatible: false, # TODO: This can be more accurate if we verify
                            # that there are no constant overrides
       eth1Network: some mainnet,
       runtimePreset: mainnetRuntimePreset,
-      bootstrapNodes: readFile(mainnetMetadataDir / "bootstrap_nodes.txt").splitLines,
+      bootstrapNodes: readFile(mainnetMetadataDir & "/bootstrap_nodes.txt").splitLines,
       depositContractAddress: Eth1Address.fromHex "0x00000000219ab540356cBB839Cbe05303d7705Fa",
       depositContractDeployedAt: BlockHashOrNumber.init "11052984",
-      genesisData: readFile(mainnetMetadataDir / "genesis.ssz"),
-      genesisDepositsSnapshot: readFile(mainnetMetadataDir / "genesis_deposit_contract_snapshot.ssz"))
+      genesisData: readFile(mainnetMetadataDir & "/genesis.ssz"),
+      genesisDepositsSnapshot: readFile(mainnetMetadataDir & "/genesis_deposit_contract_snapshot.ssz"))
   else:
     Eth2NetworkMetadata(
       incompatible: true,
       incompatibilityDesc: "This build is compiled with the " & const_preset & " const preset. " &
                            "It's not compatible with mainnet")
 
 template eth2testnet(path: string): Eth2NetworkMetadata =
-  loadEth2NetworkMetadata(eth2testnetsDir / path)
+  loadEth2NetworkMetadata(eth2testnetsDir & "/" & path)
 
 const
   pyrmontMetadata* = eth2testnet "shared/pyrmont"

beacon_chain/spec/keystore.nim

@@ -217,7 +217,7 @@ macro wordListArray*(filename: static string,
                      minWordLen: static int = 0,
                      maxWordLen: static int = high(int)): untyped =
   result = newTree(nnkBracket)
-  var words = slurp(filename).splitLines()
+  var words = slurp(filename.replace('\\', '/')).splitLines()
   for word in words:
     if word.len >= minWordLen and word.len <= maxWordLen:
       result.add newCall("cstring", newLit(word))

docker/dist/Dockerfile.win64

@@ -0,0 +1,18 @@
+# The build is reproducible only if this base image stays the same.
+FROM statusteam/nimbus_beacon_node:dist_base_20210202230118_win64@sha256:47b8ea60a35bd45355f0490677c04a9b42df56b5041cbf64da8ad233511a55d5
+
+SHELL ["/bin/bash", "-c"]
+
+ARG USER_ID
+ARG GROUP_ID
+
+RUN addgroup --gid ${GROUP_ID} user; \
+	adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user;
+
+USER user
+
+STOPSIGNAL SIGINT
+
+COPY "entry_point.sh" "/home/user/"
+ENTRYPOINT ["/home/user/entry_point.sh", "Windows_amd64"]
+

docker/dist/README-Windows.md

@@ -0,0 +1,7 @@
+# Windows-specific requirements
+
+Run the wrapper scripts from a [Git for Windows](https://gitforwindows.org/) Bash shell.
+
+If you run the beacon node binary directly, prefix it with "winpty -- ". It
+will increase the chance of Ctrl+C working inside that "mintty" terminal emulator.
+

docker/dist/base_image/Dockerfile.win64

@@ -0,0 +1,23 @@
+# This Docker image can change from one build to another, because the upstream
+# Debian/Ubuntu package index is continuously updated and we have to run
+# `apt-get update` in here.
+#
+# The only way to make this a part of our reproducible build system is to build
+# it once, upload it to Docker Hub and make sure it's being pulled regularly so
+# it's not deleted after 6 months of inactivity.
+
+FROM ubuntu:20.04
+
+SHELL ["/bin/bash", "-c"]
+
+ENV DEBIAN_FRONTEND=noninteractive TZ="Etc/UTC"
+RUN \
+ apt-get -qq update \
+ && apt-get -qq -y install git gnupg software-properties-common lsb &>/dev/null \
+ && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 86B72ED9 \
+ && add-apt-repository "deb [arch=amd64] https://pkg.mxe.cc/repos/apt $(lsb_release -sc) main" \
+ && apt-get -qq update \
+ && apt-get -qq -y install mxe-x86-64-w64-mingw32.static-cc &>/dev/null \
+ && apt-get -qq clean \
+ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+

docker/dist/base_image/Makefile

@@ -7,9 +7,11 @@ IMAGE_NAME := statusteam/nimbus_beacon_node:$(IMAGE_TAG)
 	build-amd64 \
 	build-arm64 \
 	build-arm \
+	build-win64 \
 	push-amd64 \
 	push-arm64 \
-	push-arm
+	push-arm \
+	push-win64
 
 build-amd64:
 	$(CURDIR)/make_base_image.sh amd64 "$(IMAGE_NAME)"
@@ -20,6 +22,9 @@ build-arm64:
 build-arm:
 	$(CURDIR)/make_base_image.sh arm "$(IMAGE_NAME)_arm"
 
+build-win64:
+	$(CURDIR)/make_base_image.sh win64 "$(IMAGE_NAME)_win64"
+
 # You probably don't want to recreate and push these base images to Docker Hub,
 # because when older images expire and get deleted, it will no longer be possible
 # to reproduce old releases.
@@ -33,3 +38,6 @@ build-arm:
 #push-arm: build-arm
 #	docker push $(IMAGE_NAME)_arm
 
+#push-win64: build-win64
+#	docker push $(IMAGE_NAME)_win64
+

docker/dist/base_image/make_base_image.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
+# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under
 # either of:
 # - Apache License, version 2.0
 # - MIT license
@@ -23,7 +23,7 @@ fi
 ARCH="${1}"
 DOCKER_TAG="${2}"
 
-if [[ "${ARCH}" == "amd64" ]]; then
+if [[ "${ARCH}" == "amd64" || "${ARCH}" == "win64" ]]; then
   USE_QEMU=0
 else
   USE_QEMU=1
@@ -63,8 +63,6 @@ DOCKER_BUILDKIT=1 \
   docker build \
   -t ${DOCKER_TAG} \
   --progress=plain \
-  --build-arg USER_ID=$(id -u) \
-  --build-arg GROUP_ID=$(id -g) \
   ${DOCKER_EXTRA_ARGS} \
   -f Dockerfile.${ARCH} .