long-test.yml

name: Long Test

on:
  pull_request:
    types:
      - labeled
      - synchronize
      - opened
      - reopened
  schedule:
    - cron: "0 0 * * 1-5"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

permissions: read-all

jobs:
  long-asan:
    if: &check_trigger_conditions ${{ (github.event.action == 'labeled' && github.event.label.name == 'run-long-test') || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-long-test')) || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }}
    name: "ASAN"
    runs-on:
      [
        self-hosted,
        1ES.Pool=gha-vmss-d16av5-ci,
        "JobId=long_asan-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
      ]
    container:
      image: mcr.microsoft.com/azurelinux/base/core:3.0
      options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE

    steps:
      - name: "Checkout dependencies"
        shell: bash
        run: |
          set -ex
          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
          tdnf -y update
          tdnf -y install ca-certificates git

      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: "Install dependencies"
        shell: bash
        run: |
          set -ex
          ./scripts/setup-ci.sh

      - name: "Build Debug (Long Test)"
        run: |
          set -ex
          git config --global --add safe.directory /__w/CCF/CCF
          mkdir build
          cd build
          cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON -DSAN=ON -DUSE_SNMALLOC=OFF ..
          ninja

      - name: "Test"
        run: |
          set +x
          cd build
          ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark"

      - name: "Upload logs"
        if: success() || failure()
        uses: actions/upload-artifact@v7
        with:
          name: logs-asan
          path: |
            build/workspace/*/*.config.json
            build/workspace/*/out
            build/workspace/*/err
            build/workspace/*/*.ledger/*
            build/workspace/*/stack_trace
          if-no-files-found: ignore

  long-tsan:
    if: *check_trigger_conditions
    name: "TSAN"
    runs-on:
      [
        self-hosted,
        1ES.Pool=gha-vmss-d16av5-ci,
        "JobId=long_tsan-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
      ]
    container:
      image: mcr.microsoft.com/azurelinux/base/core:3.0
      options: --user root --publish-all --privileged

    steps:
      - name: "Checkout dependencies"
        shell: bash
        run: |
          set -ex
          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
          tdnf -y update
          tdnf -y install ca-certificates git

      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: "Install dependencies"
        shell: bash
        run: |
          set -ex
          ./scripts/setup-ci.sh

      - name: "Build Debug (Long Test)"
        run: |
          set -ex
          git config --global --add safe.directory /__w/CCF/CCF
          mkdir build
          cd build
          cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=OFF -DTSAN=ON -DUSE_SNMALLOC=OFF -DWORKER_THREADS=2 ..
          ninja

      - name: "Test"
        run: |
          set +x
          cd build
          setarch -R ./tests.sh --output-on-failure --timeout 3600 -LE "benchmark" -E "recovery|regression|governance|membership"

      - name: "Upload logs"
        if: success() || failure()
        uses: actions/upload-artifact@v7
        with:
          name: logs-tsan
          path: |
            build/workspace/*/*.config.json
            build/workspace/*/out
            build/workspace/*/err
            build/workspace/*/*.ledger/*
            build/workspace/*/stack_trace
          if-no-files-found: ignore

  # All e2e tests without sanitizers in debug mode; needed because:
  # - Sanitizer builds may slightly differ.
  # - Test durations may also differ, which is important for -L "suite" because they have fixed timeouts.
  long-e2e-debug:
    if: *check_trigger_conditions
    name: Long e2e - Debug
    runs-on:
      [
        self-hosted,
        1ES.Pool=gha-vmss-d16av5-ci,
        "JobId=long_e2e_debug-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
      ]
    container:
      image: mcr.microsoft.com/azurelinux/base/core:3.0
      options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE

    steps:
      - name: "Checkout dependencies"
        shell: bash
        run: |
          set -ex
          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
          tdnf -y update
          tdnf -y install ca-certificates git

      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: "Install dependencies"
        shell: bash
        run: |
          set -ex
          ./scripts/setup-ci.sh
          # This run requires libc++ to run with hardening mode
          tdnf -y install libcxx-devel libunwind-devel

      - name: "Build"
        run: |
          git config --global --add safe.directory /__w/CCF/CCF
          mkdir build
          cd build
          # Use libc++ to enable hardening/bounds checking during tests
          cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON -DUSE_LIBCXX=ON -DUSE_SNMALLOC=OFF ..
          ninja

      - name: "Test"
        run: |
          set +x
          cd build
          ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark"

      - name: "Upload logs"
        if: success() || failure()
        uses: actions/upload-artifact@v7
        with:
          name: logs-e2e-debug
          path: |
            build/workspace/*/*.config.json
            build/workspace/*/out
            build/workspace/*/err
            build/workspace/*/*.ledger/*
            build/workspace/*/stack_trace
          if-no-files-found: ignore

  # All e2e tests in release mode (same as release build).
  long-e2e-release:
    if: *check_trigger_conditions
    name: Long e2e - Release
    runs-on:
      [
        self-hosted,
        1ES.Pool=gha-vmss-d16av5-ci,
        "JobId=long_e2e_release-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
      ]
    container:
      image: mcr.microsoft.com/azurelinux/base/core:3.0
      options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE

    steps:
      - name: "Checkout dependencies"
        shell: bash
        run: |
          set -ex
          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
          tdnf -y update
          tdnf -y install ca-certificates git

      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: "Install dependencies"
        shell: bash
        run: |
          set -ex
          ./scripts/setup-ci.sh

      - name: "Build"
        run: |
          git config --global --add safe.directory /__w/CCF/CCF
          mkdir build
          cd build
          cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCLIENT_PROTOCOLS_TEST=ON -DLONG_TESTS=ON ..
          ninja

      - name: "Test"
        run: |
          set +x
          cd build
          ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark"

      - name: "Upload logs"
        if: success() || failure()
        uses: actions/upload-artifact@v7
        with:
          name: logs-e2e-release
          path: |
            build/workspace/*/*.config.json
            build/workspace/*/out
            build/workspace/*/err
            build/workspace/*/*.ledger/*
            build/workspace/*/stack_trace
          if-no-files-found: ignore

  # Relatively short-running tests suites combined together to reduce executors
  # required, and to benefit from shared setup.
  # - End-to-end shuffled suite
  # - Long LTS tests
  # - Ensure snmalloc linkage
  e2e-long-combined:
    if: *check_trigger_conditions
    name: Long Shuffled/LTS/Snmalloc
    runs-on:
      [
        self-hosted,
        1ES.Pool=gha-vmss-d16av5-ci,
        "JobId=long_e2e_shuffled-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
      ]
    container:
      image: mcr.microsoft.com/azurelinux/base/core:3.0
      options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE

    steps:
      - name: "Checkout dependencies"
        shell: bash
        run: |
          set -ex
          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
          tdnf -y update
          tdnf -y install ca-certificates git

      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: "Install dependencies"
        shell: bash
        run: |
          set -ex
          ./scripts/setup-ci.sh

      - name: "Build"
        run: |
          git config --global --add safe.directory /__w/CCF/CCF
          mkdir build
          cd build
          cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON -DSHUFFLE_SUITE=ON -DWORKER_THREADS=2 ..
          ninja

      - name: "Test Long LTS"
        run: |
          set +x
          cd build
          ./tests.sh --output-on-failure --timeout 1600 -R "lts"

      - name: "Test shuffled"
        run: |
          set +x
          cd build
          ./tests.sh --output-on-failure --timeout 1600 -R "suite"

      - name: "Check snmalloc disabled"
        run: |
          set -ex
          cd build
          cmake -GNinja -DUSE_SNMALLOC=OFF ..
          ninja samples/apps/basic/basic
          ! ../scripts/ensure-snmalloc.sh samples/apps/basic/basic

      - name: "Check snmalloc enabled"
        run: |
          set -ex
          cd build
          cmake -GNinja -DUSE_SNMALLOC=ON ..
          ninja samples/apps/basic/basic
          ../scripts/ensure-snmalloc.sh samples/apps/basic/basic

      - name: "Upload logs"
        if: success() || failure()
        uses: actions/upload-artifact@v7
        with:
          name: e2e-long-combined
          path: |
            build/workspace/*/*.config.json
            build/workspace/*/out
            build/workspace/*/err
            build/workspace/*/*.ledger/*
            build/workspace/*/stack_trace
          if-no-files-found: ignore