ci: Migrate CI scripts to CMake

Author: hebasto

.github/workflows/ci.yml

@@ -67,12 +67,12 @@ jobs:
           echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD $EXCLUDE_MERGE_BASE_ANCESTORS | head -1)" >> "$GITHUB_ENV"
       - run: |
           sudo apt-get update
-          sudo apt-get install clang ccache build-essential libtool autotools-dev automake pkg-config bsdmainutils python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libnatpmp-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y
+          sudo apt-get install clang ccache build-essential cmake pkg-config bsdmainutils python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libnatpmp-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y
       - name: Compile and run tests
         run: |
           # Run tests on commits after the last merge commit and before the PR head commit
           # Use clang++, because it is a bit faster and uses less memory than g++
-          git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && ./autogen.sh && CC=clang CXX=clang++ ./configure --with-incompatible-bdb && make clean && make -j $(nproc) check && ./test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.TEST_BASE }}
+          git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && CC=clang CXX=clang++ cmake -B build -DBUILD_GUI=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWITH_BDB=ON -DWITH_NATPMP=ON -DWITH_MINIUPNPC=ON -DWITH_USDT=ON && cmake --build build -j $(nproc) && ctest --test-dir build -j $(nproc) && ./build/test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.TEST_BASE }}
 
   macos-native-x86_64:
     name: 'macOS 13 native, x86_64, no depends, sqlite only, gui'
@@ -139,18 +139,17 @@ jobs:
     if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request'
 
     env:
-      CCACHE_MAXSIZE: '200M'
-      CI_CCACHE_VERSION: '4.7.5'
-      CI_QT_CONF: '-release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml'
-      CI_QT_DIR: 'qt-everywhere-src-5.15.11'
-      CI_QT_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.11/single/qt-everywhere-opensource-src-5.15.11.zip'
       PYTHONUTF8: 1
       TEST_RUNNER_TIMEOUT_FACTOR: 40
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Remove non-MSVC tool installations
+        run: |
+          Remove-Item -Path "$env:ProgramFiles/CMake" -Recurse -Force
+
       - name: Configure Developer Command Prompt for Microsoft Visual C++
         # Using microsoft/setup-msbuild is not enough.
         uses: ilammy/msvc-dev-cmd@v1
@@ -159,141 +158,60 @@ jobs:
 
       - name: Get tool information
         run: |
+          cmake -version | Tee-Object -FilePath "cmake_version"
+          Write-Output "---"
           msbuild -version | Tee-Object -FilePath "msbuild_version"
           $env:VCToolsVersion | Tee-Object -FilePath "toolset_version"
-          $env:CI_QT_URL | Out-File -FilePath "qt_url"
-          $env:CI_QT_CONF | Out-File -FilePath "qt_conf"
           py -3 --version
           Write-Host "PowerShell version $($PSVersionTable.PSVersion.ToString())"
 
-      - name: Restore static Qt cache
-        id: static-qt-cache
-        uses: actions/cache/restore@v4
-        with:
-          path: C:\Qt_static
-          key: ${{ github.job }}-static-qt-${{ hashFiles('msbuild_version', 'qt_url', 'qt_conf') }}
-
-      - name: Build static Qt. Download
-        if: steps.static-qt-cache.outputs.cache-hit != 'true'
-        shell: cmd
-        run: |
-          curl --location --output C:\qt-src.zip %CI_QT_URL%
-          choco install --yes --no-progress jom
-
-      - name: Build static Qt. Expand source archive
-        if: steps.static-qt-cache.outputs.cache-hit != 'true'
-        shell: cmd
-        run: tar -xf C:\qt-src.zip -C C:\
-
-      - name: Build static Qt. Create build directory
-        if: steps.static-qt-cache.outputs.cache-hit != 'true'
-        run: |
-          Rename-Item -Path "C:\$env:CI_QT_DIR" -NewName "C:\qt-src"
-          New-Item -ItemType Directory -Path "C:\qt-src\build"
-
-      - name: Build static Qt. Configure
-        if: steps.static-qt-cache.outputs.cache-hit != 'true'
-        working-directory: C:\qt-src\build
-        shell: cmd
-        run: ..\configure %CI_QT_CONF% -prefix C:\Qt_static
-
-      - name: Build static Qt. Build
-        if: steps.static-qt-cache.outputs.cache-hit != 'true'
-        working-directory: C:\qt-src\build
-        shell: cmd
-        run: jom
-
-      - name: Build static Qt. Install
-        if: steps.static-qt-cache.outputs.cache-hit != 'true'
-        working-directory: C:\qt-src\build
-        shell: cmd
-        run: jom install
-
-      - name: Save static Qt cache
-        if: steps.static-qt-cache.outputs.cache-hit != 'true'
-        uses: actions/cache/save@v4
-        with:
-          path: C:\Qt_static
-          key: ${{ github.job }}-static-qt-${{ hashFiles('msbuild_version', 'qt_url', 'qt_conf') }}
-
-      - name: Ccache installation cache
-        id: ccache-installation-cache
-        uses: actions/cache@v4
-        with:
-          path: |
-            C:\ProgramData\chocolatey\lib\ccache
-            C:\ProgramData\chocolatey\bin\ccache.exe
-            C:\ccache\cl.exe
-          key: ${{ github.job }}-ccache-installation-${{ env.CI_CCACHE_VERSION }}
-
-      - name: Install Ccache
-        if: steps.ccache-installation-cache.outputs.cache-hit != 'true'
-        run: |
-          choco install --yes --no-progress ccache --version=$env:CI_CCACHE_VERSION
-          New-Item -ItemType Directory -Path "C:\ccache"
-          Copy-Item -Path "$env:ChocolateyInstall\lib\ccache\tools\ccache-$env:CI_CCACHE_VERSION-windows-x86_64\ccache.exe" -Destination "C:\ccache\cl.exe"
-
-      - name: Restore Ccache cache
-        id: ccache-cache
-        uses: actions/cache/restore@v4
-        with:
-          path: ~/AppData/Local/ccache
-          key: ${{ github.job }}-ccache-${{ github.run_id }}
-          restore-keys: ${{ github.job }}-ccache-
-
       - name: Using vcpkg with MSBuild
         run: |
           Set-Location "$env:VCPKG_INSTALLATION_ROOT"
+          Add-Content -Path "triplets\x64-windows.cmake" -Value "set(VCPKG_BUILD_TYPE release)"
           Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)"
-          .\vcpkg.exe --vcpkg-root "$env:VCPKG_INSTALLATION_ROOT" integrate install
-          git rev-parse HEAD | Tee-Object -FilePath "$env:GITHUB_WORKSPACE\vcpkg_commit"
 
       - name: vcpkg tools cache
         uses: actions/cache@v4
         with:
           path: C:/vcpkg/downloads/tools
           key: ${{ github.job }}-vcpkg-tools
 
-      - name: vcpkg binary cache
-        uses: actions/cache@v4
+      - name: Restore vcpkg binary cache
+        uses: actions/cache/restore@v4
+        id: vcpkg-binary-cache
         with:
           path: ~/AppData/Local/vcpkg/archives
-          key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('vcpkg_commit', 'msbuild_version', 'toolset_version', 'build_msvc/vcpkg.json') }}
-
-      - name: Generate project files
-        run: py -3 build_msvc\msvc-autogen.py
+          key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }}
 
-      - name: Build
-        shell: cmd
+      - name: Generate build system
         run: |
-          ccache --zero-stats
-          msbuild build_msvc\bitcoin.sln -property:CLToolPath=C:\ccache;CLToolExe=cl.exe;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo
-
-      - name: Ccache stats
-        run: ccache --show-stats
+          cmake -B build --preset vs2022-static -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_INSTALLATION_ROOT\scripts\buildsystems\vcpkg.cmake" -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWERROR=ON
 
-      - name: Save Ccache cache
+      - name: Save vcpkg binary cache
         uses: actions/cache/save@v4
-        if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true'
+        if: github.event_name != 'pull_request' && steps.vcpkg-binary-cache.outputs.cache-hit != 'true'
         with:
-          path: ~/AppData/Local/ccache
-          # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
-          key: ${{ github.job }}-ccache-${{ github.run_id }}
-
-      - name: Run unit tests
-        run: src\test_bitcoin.exe -l test_suite
-
-      - name: Run benchmarks
-        run: src\bench_bitcoin.exe -sanity-check
+          path: ~/AppData/Local/vcpkg/archives
+          key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }}
 
-      - name: Run util tests
-        run: py -3 test\util\test_runner.py
+      - name: Build
+        working-directory: build
+        run: |
+          cmake --build . -j $env:NUMBER_OF_PROCESSORS --config Release
 
-      - name: Run rpcauth test
-        run: py -3 test\util\rpcauth-test.py
+      - name: Run test suite
+        working-directory: build
+        run: |
+          ctest -j $env:NUMBER_OF_PROCESSORS -C Release
 
       - name: Run functional tests
+        working-directory: build
         env:
+          BITCOIND: '${{ github.workspace }}\build\src\Release\bitcoind.exe'
+          BITCOINCLI: '${{ github.workspace }}\build\src\Release\bitcoin-cli.exe'
+          BITCOINUTIL: '${{ github.workspace }}\build\src\Release\bitcoin-util.exe'
+          BITCOINWALLET: '${{ github.workspace }}\build\src\Release\bitcoin-wallet.exe'
           TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }}
         shell: cmd
         run: py -3 test\functional\test_runner.py --jobs %NUMBER_OF_PROCESSORS% --ci --quiet --tmpdirprefix=%RUNNER_TEMP% --combinedlogslen=99999999 --timeout-factor=%TEST_RUNNER_TIMEOUT_FACTOR% %TEST_RUNNER_EXTRA%
@@ -306,8 +224,9 @@ jobs:
           git log -1
 
       - name: Run fuzz binaries
+        working-directory: build
         env:
-          BITCOINFUZZ: "${{ github.workspace}}\\src\\fuzz.exe"
+          BITCOINFUZZ: '${{ github.workspace }}\build\src\test\fuzz\Release\fuzz.exe'
         shell: cmd
         run: py -3 test\fuzz\test_runner.py --par %NUMBER_OF_PROCESSORS% --loglevel DEBUG %RUNNER_TEMP%\qa-assets\fuzz_seed_corpus
 

ci/test/00_setup_env.sh

@@ -61,8 +61,6 @@ export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
 export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache}
 # Folder where the build result is put (bin and lib).
 export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out}
-# Folder where the build is done (dist and out-of-tree build).
-export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build}
 # The folder for previous release binaries.
 # This folder exists only on the ci guest, and on the ci host as a volume.
 export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases}

ci/test/00_setup_env_arm.sh

@@ -17,4 +17,4 @@ export RUN_FUNCTIONAL_TESTS=false
 export GOAL="install"
 # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1"
 # This could be removed once the ABI change warning does not show up by default
-export BITCOIN_CONFIG="--enable-reduce-exports CXXFLAGS='-Wno-psabi -Wno-error=maybe-uninitialized'"
+export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DCMAKE_CXX_FLAGS='-Wno-psabi -Wno-error=maybe-uninitialized'"

ci/test/00_setup_env_i686_centos.sh

@@ -13,5 +13,5 @@ export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc
 export PIP_PACKAGES="pyzmq"
 export GOAL="install"
 export NO_WERROR=1  # Suppress error: #warning _FORTIFY_SOURCE > 2 is treated like 2 on this platform [-Werror=cpp]
-export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports"
+export BITCOIN_CONFIG="-DWITH_ZMQ=ON -DBUILD_GUI=ON -DREDUCE_EXPORTS=ON"
 export CONFIG_SHELL="/bin/dash"

ci/test/00_setup_env_i686_multiprocess.sh

@@ -13,6 +13,11 @@ export PACKAGES="llvm clang g++-multilib"
 export DEP_OPTS="DEBUG=1 MULTIPROCESS=1"
 export GOAL="install"
 export TEST_RUNNER_EXTRA="--v2transport"
-export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' \
-CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='-Wno-error=documentation'"
+export BITCOIN_CONFIG="\
+ -DCMAKE_BUILD_TYPE=Debug \
+ -DCMAKE_C_COMPILER='clang;-m32' \
+ -DCMAKE_CXX_COMPILER='clang++;-m32' \
+ -DCMAKE_CXX_FLAGS='-Wno-error=documentation' \
+ -DAPPEND_CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' \
+"
 export BITCOIND=bitcoin-node  # Used in functional tests

ci/test/00_setup_env_mac_cross.sh

@@ -17,4 +17,4 @@ export XCODE_BUILD_ID=15A240d
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="deploy"
-export BITCOIN_CONFIG="--with-gui --enable-reduce-exports"
+export BITCOIN_CONFIG="-DBUILD_GUI=ON -DREDUCE_EXPORTS=ON"

ci/test/00_setup_env_mac_native.sh

@@ -11,7 +11,7 @@ export HOST=x86_64-apple-darwin
 # Therefore, `--break-system-packages` is needed.
 export PIP_PACKAGES="--break-system-packages zmq"
 export GOAL="install"
-export BITCOIN_CONFIG="--with-gui --with-miniupnpc --with-natpmp --enable-reduce-exports"
+export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_MINIUPNPC=ON -DWITH_NATPMP=ON -DREDUCE_EXPORTS=ON"
 export CI_OS_NAME="macos"
 export NO_DEPENDS=1
 export OSX_SDK=""

ci/test/00_setup_env_native_asan.sh

@@ -22,8 +22,13 @@ export CONTAINER_NAME=ci_native_asan
 export PACKAGES="systemtap-sdt-dev clang-18 llvm-18 libclang-rt-18-dev python3-zmq qtbase5-dev qttools5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}"
 export NO_DEPENDS=1
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \
-CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \
---with-sanitizers=address,float-divide-by-zero,integer,undefined \
-CC='clang-18 -ftrivial-auto-var-init=pattern' CXX='clang++-18 -ftrivial-auto-var-init=pattern'"
+export BITCOIN_CONFIG="\
+ -DWITH_USDT=ON -DWITH_ZMQ=ON -DWARN_INCOMPATIBLE_BDB=OFF -DBUILD_GUI=ON \
+ -DSANITIZERS=address,float-divide-by-zero,integer,undefined \
+ -DCMAKE_C_COMPILER=clang-18 \
+ -DCMAKE_CXX_COMPILER=clang++-18 \
+ -DCMAKE_C_FLAGS='-ftrivial-auto-var-init=pattern' \
+ -DCMAKE_CXX_FLAGS='-ftrivial-auto-var-init=pattern' \
+ -DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \
+"
 export CCACHE_MAXSIZE=300M

ci/test/00_setup_env_native_fuzz.sh

@@ -15,7 +15,13 @@ export RUN_FUNCTIONAL_TESTS=false
 export RUN_FUZZ_TESTS=true
 export GOAL="install"
 export CI_CONTAINER_CAP="--cap-add SYS_PTRACE"  # If run with (ASan + LSan), the container needs access to ptrace (https://github.com/google/sanitizers/issues/764)
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,float-divide-by-zero,integer \
-CC='clang-18 -ftrivial-auto-var-init=pattern' CXX='clang++-18 -ftrivial-auto-var-init=pattern'"
+export BITCOIN_CONFIG="\
+ -DBUILD_FOR_FUZZING=ON \
+ -DSANITIZERS=fuzzer,address,undefined,float-divide-by-zero,integer \
+ -DCMAKE_C_COMPILER=clang-18 \
+ -DCMAKE_CXX_COMPILER=clang++-18 \
+ -DCMAKE_C_FLAGS='-ftrivial-auto-var-init=pattern' \
+ -DCMAKE_CXX_FLAGS='-ftrivial-auto-var-init=pattern' \
+"
 export CCACHE_MAXSIZE=200M
 export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-18"

ci/test/00_setup_env_native_fuzz_with_msan.sh

@@ -18,7 +18,11 @@ export PACKAGES="ninja-build"
 export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export GOAL="install"
 # _FORTIFY_SOURCE is not compatible with MSAN.
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE -U_FORTIFY_SOURCE'"
+export BITCOIN_CONFIG="\
+ -DBUILD_FOR_FUZZING=ON \
+ -DSANITIZERS=fuzzer,memory \
+ -DAPPEND_CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE -U_FORTIFY_SOURCE' \
+"
 export USE_MEMORY_SANITIZER="true"
 export RUN_UNIT_TESTS="false"
 export RUN_FUNCTIONAL_TESTS="false"