Merge bitcoin/bitcoin#28173: ci: Run Windows native task on GitHub Actions

fanquake · fanquake · commit db57574aa87c · 2023-08-28T11:27:29.000+01:00
6a43372 ci: Run "Win64 native" job on GitHub Actions (Hennadii Stepanov) Pull request description: From bitcoin/bitcoin#28098: > Thus, someone would have to sponsor an amount of roughly 5kUSD/mo for those two tasks. > If the goal is to stay on a free plan, I think the only option is GitHub Actions CI. Historical context: - bitcoin/bitcoin#17697 - bitcoin/bitcoin#17803 - bitcoin/bitcoin#18031 Security concerns: - bitcoin/bitcoin#28098 (comment) - bitcoin/bitcoin#28098 (comment) `GITHUB_TOKEN` permissions (from the build log in my personal repo): ``` 2023-07-27T07:30:17.8313534Z ##[group]GITHUB_TOKEN Permissions 2023-07-27T07:30:17.8314113Z Contents: read 2023-07-27T07:30:17.8314608Z Metadata: read 2023-07-27T07:30:17.8314957Z Packages: read 2023-07-27T07:30:17.8315233Z ##[endgroup] ``` Comparison of resources: | Resource | Current, Cirrus CI | Suggested, GitHub Actions | |---|:-:|:-:| | CPU | 6 | 2 | | RAM, GB | 12 | 7 | The `TEST_RUNNER_TIMEOUT_FACTOR` variable is set to the current default value for all CI tasks: https://github.com/bitcoin/bitcoin/blob/64440bb733896a7a2caf902825e0406cb993e666/ci/test/00_setup_env.sh#L48 Top commit has no ACKs. Tree-SHA512: ddfdaf7a1e4793a64ac0cd20f116b29608dd06f15b062769ac70b3ea2fb82775aa96aa79c7b768efefec4338aaa5b57d267b592f62d0e8d5d94ecc11001a165d
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -101,102 +101,6 @@ task:
   env:
     FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh"
 
-task:
-  name: "Win64 native [vs2022]"
-  << : *FILTER_TEMPLATE
-  windows_container:
-    cpu: 6
-    memory: 12G
-    image: cirrusci/windowsservercore:visualstudio2022
-  timeout_in: 120m
-  env:
-    PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Current\Bin;%PATH%'
-    PYTHONUTF8: 1
-    CI_VCPKG_TAG: '2023.01.09'
-    VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads'
-    VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives'
-    CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache'
-    WRAPPED_CL: 'C:\Users\ContainerAdministrator\AppData\Local\Temp\cirrus-ci-build\ci\test\wrapped-cl.bat'
-    QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip'
-    QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.5.zip'
-    QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.5'
-    QTBASEDIR: 'C:\Qt_static'
-    x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"'
-    QT_CONFIGURE_COMMAND: '..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml'
-    IgnoreWarnIntDirInTempDetected: 'true'
-  merge_script:
-    - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL pull/$env:CIRRUS_PR/merge; git reset --hard FETCH_HEAD; }
-  msvc_qt_built_cache:
-    folder: "%QTBASEDIR%"
-    reupload_on_changes: false
-    fingerprint_script:
-      - echo %QT_DOWNLOAD_URL% %QT_CONFIGURE_COMMAND%
-      - msbuild -version
-    populate_script:
-      - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip
-      - mkdir C:\jom
-      - tar -xf C:\jom.zip -C C:\jom
-      - curl -L -o %QT_LOCAL_PATH% %QT_DOWNLOAD_URL%
-      - tar -xf %QT_LOCAL_PATH% -C C:\
-      - '%x64_NATIVE_TOOLS%'
-      - cd %QT_SOURCE_DIR%
-      - mkdir build
-      - cd build
-      - '%QT_CONFIGURE_COMMAND% -prefix %QTBASEDIR%'
-      - jom
-      - jom install
-  vcpkg_tools_cache:
-    folder: '%VCPKG_DOWNLOADS%\tools'
-    reupload_on_changes: false
-    fingerprint_script:
-      - echo %CI_VCPKG_TAG%
-      - msbuild -version
-  vcpkg_binary_cache:
-    folder: '%VCPKG_DEFAULT_BINARY_CACHE%'
-    reupload_on_changes: true
-    fingerprint_script:
-      - echo %CI_VCPKG_TAG%
-      - type build_msvc\vcpkg.json
-      - msbuild -version
-    populate_script:
-      - mkdir %VCPKG_DEFAULT_BINARY_CACHE%
-  ccache_cache:
-    folder: '%CCACHE_DIR%'
-  install_tools_script:
-    - choco install --yes --no-progress ccache --version=4.7.4
-    - choco install --yes --no-progress python3 --version=3.9.6
-    - pip install zmq
-    - ccache --version
-    - python -VV
-  install_vcpkg_script:
-    - cd ..
-    - git clone --quiet https://github.com/microsoft/vcpkg.git
-    - cd vcpkg
-    - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG%
-    - .\bootstrap-vcpkg -disableMetrics
-    - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake
-    - .\vcpkg integrate install
-    - .\vcpkg version
-  build_script:
-    - '%x64_NATIVE_TOOLS%'
-    - cd %CIRRUS_WORKING_DIR%
-    - ccache --zero-stats
-    - python build_msvc\msvc-autogen.py
-    - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL%;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo
-    - ccache --show-stats
-  check_script:
-    - src\test_bitcoin.exe -l test_suite
-    - src\bench_bitcoin.exe --sanity-check
-    - python test\util\test_runner.py
-    - python test\util\rpcauth-test.py
-  functional_tests_script:
-    # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted".
-    # See: https://learn.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance
-    - netsh int ipv4 set dynamicport tcp start=1025 num=64511
-    - netsh int ipv6 set dynamicport tcp start=1025 num=64511
-    # Exclude feature_dbcrash for now due to timeout
-    - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=99999999 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash
-
 task:
   name: 'ARM, unit tests, no functional tests'
   << : *GLOBAL_TASK_TEMPLATE
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -68,3 +68,199 @@ jobs:
           path: ${{ env.CCACHE_DIR }}
           # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
           key: ${{ github.job }}-ccache-${{ github.run_id }}
+
+  win64-native:
+    name: 'Win64 native, VS 2022'
+    # Use latest image, but hardcode version to avoid silent upgrades (and breaks).
+    # See: https://github.com/actions/runner-images#available-images.
+    runs-on: windows-2022
+
+    # No need to run on the read-only mirror, unless it is a PR.
+    if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request'
+
+    env:
+      CCACHE_MAXSIZE: '200M'
+      CI_CCACHE_VERSION: '4.7.5'
+      CI_QT_CONF: '-release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml'
+      CI_QT_DIR: 'qt-everywhere-src-5.15.5'
+      CI_QT_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip'
+      PYTHONUTF8: 1
+      TEST_RUNNER_TIMEOUT_FACTOR: 40
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Fix Visual Studio installation
+        # See: https://github.com/actions/runner-images/issues/7832#issuecomment-1617585694.
+        run: |
+          Set-Location "C:\Program Files (x86)\Microsoft Visual Studio\Installer\"
+          $InstallPath = "C:\Program Files\Microsoft Visual Studio\2022\Enterprise"
+          $componentsToRemove= @(
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.ARM"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.ARM.Spectre"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.ARM64"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.ARM64.Spectre"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.x86.x64"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.x86.x64.Spectre"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.Spectre"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.ARM"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.ARM.Spectre"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.ARM64"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.ARM64.Spectre"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.Spectre"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.ARM"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.ARM.Spectre"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.ARM64"
+            "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.ARM64.Spectre"
+          )
+          [string]$workloadArgs = $componentsToRemove | ForEach-Object {" --remove " +  $_}
+          $Arguments = ('/c', "vs_installer.exe", 'modify', '--installPath', "`"$InstallPath`"",$workloadArgs, '--quiet', '--norestart', '--nocache')
+          # should be run twice
+          $process = Start-Process -FilePath cmd.exe -ArgumentList $Arguments -Wait -PassThru -WindowStyle Hidden
+          $process = Start-Process -FilePath cmd.exe -ArgumentList $Arguments -Wait -PassThru -WindowStyle Hidden
+
+      - name: Configure Developer Command Prompt for Microsoft Visual C++
+        # Using microsoft/setup-msbuild is not enough.
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: x64
+
+      - name: Check MSBuild and Qt
+        run: |
+          msbuild -version | Out-File -FilePath "$env:GITHUB_WORKSPACE\msbuild_version"
+          Get-Content -Path "$env:GITHUB_WORKSPACE\msbuild_version"
+          $env:CI_QT_URL | Out-File -FilePath "$env:GITHUB_WORKSPACE\qt_url"
+          $env:CI_QT_CONF | Out-File -FilePath "$env:GITHUB_WORKSPACE\qt_conf"
+
+      - name: Restore static Qt cache
+        id: static-qt-cache
+        uses: actions/cache/restore@v3
+        with:
+          path: C:\Qt_static
+          key: ${{ github.job }}-static-qt-${{ hashFiles('msbuild_version', 'qt_url', 'qt_conf') }}
+
+      - name: Build static Qt. Download
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        shell: cmd
+        run: |
+          curl --location --output C:\qt-src.zip %CI_QT_URL%
+          choco install --yes --no-progress jom
+
+      - name: Build static Qt. Expand source archive
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        shell: cmd
+        run: tar -xf C:\qt-src.zip -C C:\
+
+      - name: Build static Qt. Create build directory
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        run: |
+          Rename-Item -Path "C:\$env:CI_QT_DIR" -NewName "C:\qt-src"
+          New-Item -ItemType Directory -Path "C:\qt-src\build"
+
+      - name: Build static Qt. Configure
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        working-directory: C:\qt-src\build
+        shell: cmd
+        run: ..\configure %CI_QT_CONF% -prefix C:\Qt_static
+
+      - name: Build static Qt. Build
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        working-directory: C:\qt-src\build
+        shell: cmd
+        run: jom
+
+      - name: Build static Qt. Install
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        working-directory: C:\qt-src\build
+        shell: cmd
+        run: jom install
+
+      - name: Save static Qt cache
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v3
+        with:
+          path: C:\Qt_static
+          key: ${{ github.job }}-static-qt-${{ hashFiles('msbuild_version', 'qt_url', 'qt_conf') }}
+
+      - name: Ccache installation cache
+        id: ccache-installation-cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            C:\ProgramData\chocolatey\lib\ccache
+            C:\ProgramData\chocolatey\bin\ccache.exe
+            C:\ccache\cl.exe
+          key: ${{ github.job }}-ccache-installation-${{ env.CI_CCACHE_VERSION }}
+
+      - name: Install Ccache
+        if: steps.ccache-installation-cache.outputs.cache-hit != 'true'
+        run: |
+          choco install --yes --no-progress ccache --version=$env:CI_CCACHE_VERSION
+          New-Item -ItemType Directory -Path "C:\ccache"
+          Copy-Item -Path "$env:ChocolateyInstall\lib\ccache\tools\ccache-$env:CI_CCACHE_VERSION-windows-x86_64\ccache.exe" -Destination "C:\ccache\cl.exe"
+
+      - name: Restore Ccache cache
+        uses: actions/cache/restore@v3
+        with:
+          path: ~/AppData/Local/ccache
+          key: ${{ github.job }}-ccache-${{ github.run_id }}
+          restore-keys: ${{ github.job }}-ccache-
+
+      - name: Using vcpkg with MSBuild
+        run: |
+          Set-Location "$env:VCPKG_INSTALLATION_ROOT"
+          Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)"
+          vcpkg --vcpkg-root "$env:VCPKG_INSTALLATION_ROOT" integrate install
+          git rev-parse HEAD | Out-File -FilePath "$env:GITHUB_WORKSPACE\vcpkg_commit"
+          Get-Content -Path "$env:GITHUB_WORKSPACE\vcpkg_commit"
+
+      - name: vcpkg tools cache
+        uses: actions/cache@v3
+        with:
+          path: C:/vcpkg/downloads/tools
+          key: ${{ github.job }}-vcpkg-tools
+
+      - name: vcpkg binary cache
+        uses: actions/cache@v3
+        with:
+          path: ~/AppData/Local/vcpkg/archives
+          key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('vcpkg_commit', 'msbuild_version', 'build_msvc/vcpkg.json') }}
+
+      - name: Generate project files
+        run: py -3 build_msvc\msvc-autogen.py
+
+      - name: Build
+        shell: cmd
+        run: |
+          ccache --zero-stats
+          msbuild build_msvc\bitcoin.sln -property:CLToolPath=C:\ccache;CLToolExe=cl.exe;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo
+
+      - name: Ccache stats
+        run: ccache --show-stats
+
+      - name: Save Ccache cache
+        uses: actions/cache/save@v3
+        if: github.event_name != 'pull_request'
+        with:
+          path: ~/AppData/Local/ccache
+          # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
+          key: ${{ github.job }}-ccache-${{ github.run_id }}
+
+      - name: Run unit tests
+        run: src\test_bitcoin.exe -l test_suite
+
+      - name: Run benchmarks
+        run: src\bench_bitcoin.exe -sanity-check
+
+      - name: Run util tests
+        run: py -3 test\util\test_runner.py
+
+      - name: Run rpcauth test
+        run: py -3 test\util\rpcauth-test.py
+
+      - name: Run functional tests
+        shell: cmd
+        run: py -3 test\functional\test_runner.py --ci --quiet --tmpdirprefix=%RUNNER_TEMP% --combinedlogslen=99999999 --timeout-factor=%TEST_RUNNER_TIMEOUT_FACTOR% --extended --exclude feature_dbcrash
diff --git a/ci/test/wrapped-cl.bat b/ci/test/wrapped-cl.bat
