diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml deleted file mode 100644 index acb64034b9..0000000000 --- a/.github/workflows/bandit.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Bandit static analysis (for Python code) -name: Bandit -on: - push: - branches-ignore: - - 'dependabot/**' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - bandit: - name: Bandit - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ (matrix.os == 'ubuntu-latest' && github.repository_owner == 'oneapi-src') && 'intel-ubuntu-22.04' || matrix.os }} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install Bandit - run: python3 -m pip install bandit - - # Run Bandit recursively, but omit _deps directory (with 3rd party code) - - name: Run Bandit - run: python3 -m bandit -r . -x '/_deps/' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6e128b6037..3d9bfc29b4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -29,8 +29,13 @@ jobs: sudo apt-get update sudo apt-get install -y doxygen - - name: Install pip requirements - run: python3 -m pip install -r third_party/requirements.txt + # Latest distros do not allow global pip installation + - name: Install Python requirements in venv + run: | + python3 -m venv .venv + . .venv/bin/activate + echo "$PATH" >> $GITHUB_PATH + python3 -m pip install -r third_party/requirements.txt - name: Setup PATH for python run: echo "$HOME/.local/bin" >> $GITHUB_PATH diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index 50f8f1aa9d..b8df73975f 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -1,4 +1,5 @@ -# Checks required for a PR to merge. This workflow mostly call other workflows. +# Run checks required for a PR to merge and verify if post-merge commit is valid. +# This workflow only call other workflows. name: PR/push on: @@ -15,101 +16,54 @@ permissions: contents: read jobs: - CodeStyle: - name: Coding style - runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install apt packages - run: | - sudo apt-get update - sudo apt-get install -y black cmake clang-format-15 cmake-format libhwloc-dev - - - name: Configure CMake - run: > - cmake - -B ${{github.workspace}}/build - -DUMF_FORMAT_CODE_STYLE=ON - -DUMF_BUILD_TESTS=OFF - -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF - -DUMF_BUILD_CUDA_PROVIDER=OFF - -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=OFF - - - name: Check C/C++ formatting - run: cmake --build build --target clang-format-check - - - name: Check CMake formatting - run: | - cmake --build build --target cmake-format-apply - git diff --exit-code - - - name: Check Python formatting - run: cmake --build build --target black-format-check - + CodeChecks: + uses: ./.github/workflows/reusable_checks.yml DocsBuild: - name: Build docs - runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install doxygen - run: | - sudo apt-get update - sudo apt-get install -y doxygen - - - name: Install pip requirements - run: python3 -m pip install -r third_party/requirements.txt - - - name: Setup PATH for python - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Build the documentation - working-directory: scripts - run: python3 generate_docs.py - - Spellcheck: - uses: ./.github/workflows/spellcheck.yml + uses: ./.github/workflows/reusable_docs_build.yml FastBuild: name: Fast builds - needs: [Spellcheck, CodeStyle] - uses: ./.github/workflows/fast.yml + needs: [CodeChecks, DocsBuild] + uses: ./.github/workflows/reusable_fast.yml Build: name: Basic builds needs: [FastBuild] - uses: ./.github/workflows/basic.yml + uses: ./.github/workflows/reusable_basic.yml DevDax: needs: [FastBuild] - uses: ./.github/workflows/dax.yml + uses: ./.github/workflows/reusable_dax.yml Sanitizers: needs: [FastBuild] - uses: ./.github/workflows/sanitizers.yml + uses: ./.github/workflows/reusable_sanitizers.yml Qemu: needs: [FastBuild] - uses: ./.github/workflows/qemu.yml + uses: ./.github/workflows/reusable_qemu.yml Benchmarks: needs: [Build] - uses: ./.github/workflows/benchmarks.yml + uses: ./.github/workflows/reusable_benchmarks.yml ProxyLib: needs: [Build] - uses: ./.github/workflows/proxy_lib.yml + uses: ./.github/workflows/reusable_proxy_lib.yml GPU: needs: [Build] - uses: ./.github/workflows/gpu.yml + uses: ./.github/workflows/reusable_gpu.yml Valgrind: needs: [Build] - uses: ./.github/workflows/valgrind.yml + uses: ./.github/workflows/reusable_valgrind.yml MultiNuma: needs: [Build] - uses: ./.github/workflows/multi_numa.yml + uses: ./.github/workflows/reusable_multi_numa.yml Coverage: needs: [Build, DevDax, GPU, MultiNuma, Qemu, ProxyLib] - uses: ./.github/workflows/coverage.yml + uses: ./.github/workflows/reusable_coverage.yml + CodeQL: + needs: [Build] + permissions: + contents: read + security-events: write + uses: ./.github/workflows/reusable_codeql.yml + Trivy: + needs: [Build] + permissions: + contents: read + security-events: write + uses: ./.github/workflows/reusable_trivy.yml diff --git a/.github/workflows/basic.yml b/.github/workflows/reusable_basic.yml similarity index 100% rename from .github/workflows/basic.yml rename to .github/workflows/reusable_basic.yml diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/reusable_benchmarks.yml similarity index 100% rename from .github/workflows/benchmarks.yml rename to .github/workflows/reusable_benchmarks.yml diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml new file mode 100644 index 0000000000..e3e264b0db --- /dev/null +++ b/.github/workflows/reusable_checks.yml @@ -0,0 +1,62 @@ +# Basic checks on the code, incl. coding style, spelling, bandit analysis. +# TODO: add license check +name: Basic checks + +on: workflow_call + +permissions: + contents: read + +jobs: + CodeChecks: + name: Basic code checks + runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} + + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y black cmake clang-format-15 cmake-format libhwloc-dev + + # Latest distros do not allow global pip installation + - name: Install Python requirements in venv + run: | + python3 -m venv .venv + . .venv/bin/activate + echo "$PATH" >> $GITHUB_PATH + python3 -m pip install bandit + + - name: Configure CMake + run: > + cmake + -B ${{github.workspace}}/build + -DUMF_FORMAT_CODE_STYLE=ON + -DUMF_BUILD_TESTS=OFF + -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF + -DUMF_BUILD_CUDA_PROVIDER=OFF + -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=OFF + + - name: Check C/C++ formatting + run: cmake --build build --target clang-format-check + + - name: Check CMake formatting + run: | + cmake --build build --target cmake-format-apply + git diff --exit-code + + - name: Check Python formatting + run: cmake --build build --target black-format-check + + - name: Run a spell check + uses: crate-ci/typos@b63f421581dce830bda2f597a678cb7776b41877 # v1.18.2 + with: + config: ./.github/workflows/.spellcheck-conf.toml + + # Run Bandit recursively, but omit _deps directory (with 3rd party code) and python's venv + - name: Run Bandit + run: python3 -m bandit -r . -x '/_deps/,/.venv/' diff --git a/.github/workflows/codeql.yml b/.github/workflows/reusable_codeql.yml similarity index 74% rename from .github/workflows/codeql.yml rename to .github/workflows/reusable_codeql.yml index 4a8f3ceb57..e764563103 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/reusable_codeql.yml @@ -1,19 +1,7 @@ # CodeQL static analysis name: CodeQL -# Due to lower score on Scorecard we're running this separately from -# "PR/push" workflow. For some reason permissions weren't properly set -# or recognized (by Scorecard). If Scorecard changes its behavior we can -# go back to use 'workflow_call' trigger. -on: - push: - branches-ignore: - - 'dependabot/**' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true +on: workflow_call permissions: contents: read @@ -56,27 +44,34 @@ jobs: with: languages: cpp - - name: Initialize vcpkg - if: ${{ matrix.os == 'windows-latest' }} + - name: "[Win] Initialize vcpkg" + if: matrix.os == 'windows-latest' uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 with: vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289 vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg vcpkgJsonGlob: '**/vcpkg.json' - - name: Install dependencies - if: ${{ matrix.os == 'windows-latest' }} - run: vcpkg install - shell: pwsh # Specifies PowerShell as the shell for running the script. + - name: "[Win] Install dependencies" + if: matrix.os == 'windows-latest' + run: | + vcpkg install + python3 -m pip install -r third_party/requirements.txt - - name: Install apt packages + - name: "[Lin] Install apt packages" if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update sudo apt-get install -y cmake clang libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev - - name: Install pip packages - run: python3 -m pip install -r third_party/requirements.txt + # Latest distros do not allow global pip installation + - name: "[Lin] Install Python requirements in venv" + if: matrix.os == 'ubuntu-latest' + run: | + python3 -m venv .venv + . .venv/bin/activate + echo "$PATH" >> $GITHUB_PATH + python3 -m pip install -r third_party/requirements.txt - name: Configure CMake run: > diff --git a/.github/workflows/coverage.yml b/.github/workflows/reusable_coverage.yml similarity index 91% rename from .github/workflows/coverage.yml rename to .github/workflows/reusable_coverage.yml index 08816c37a4..b71836fa6d 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/reusable_coverage.yml @@ -1,4 +1,4 @@ -# Coverage build +# Coverage build - gather artifacts from other builds and merge them into a single report name: Coverage on: workflow_call @@ -20,7 +20,7 @@ jobs: with: fetch-depth: 0 - - name: Install dependencies (ubuntu-latest) + - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y lcov diff --git a/.github/workflows/dax.yml b/.github/workflows/reusable_dax.yml similarity index 100% rename from .github/workflows/dax.yml rename to .github/workflows/reusable_dax.yml diff --git a/.github/workflows/reusable_docs_build.yml b/.github/workflows/reusable_docs_build.yml new file mode 100644 index 0000000000..269560c674 --- /dev/null +++ b/.github/workflows/reusable_docs_build.yml @@ -0,0 +1,34 @@ +name: Docs build + +on: workflow_call + +permissions: + contents: read + +jobs: + DocsBuild: + name: Docs build + runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} + + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + + - name: Install doxygen + run: | + sudo apt-get update + sudo apt-get install -y doxygen + + # Latest distros do not allow global pip installation + - name: Install Python requirements in venv + run: | + python3 -m venv .venv + . .venv/bin/activate + echo "$PATH" >> $GITHUB_PATH + python3 -m pip install -r third_party/requirements.txt + + - name: Build the documentation + working-directory: scripts + run: python3 generate_docs.py diff --git a/.github/workflows/fast.yml b/.github/workflows/reusable_fast.yml similarity index 92% rename from .github/workflows/fast.yml rename to .github/workflows/reusable_fast.yml index 997c4441c9..e25de68a1b 100644 --- a/.github/workflows/fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -133,16 +133,6 @@ jobs: working-directory: ${{env.BUILD_DIR}} run: ctest --output-on-failure --test-dir test -C Release - - name: check /DEPENDENTLOADFLAG (Windows only) - if: matrix.os == 'windows-latest' - run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{env.BUILD_DIR}}/bin/Release/umf.dll - shell: pwsh - - - name: check /DEPENDENTLOADFLAG in umf_proxy.dll - if: matrix.os == 'windows-latest' - run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{env.BUILD_DIR}}/src/proxy_lib/Release/umf_proxy.dll - shell: pwsh - # TODO: We could add some script to verify metadata of dll's (selected fields, perhaps) # ref. https://superuser.com/questions/381276/what-are-some-nice-command-line-ways-to-inspect-dll-exe-details - name: Print metadata of our dll's diff --git a/.github/workflows/gpu.yml b/.github/workflows/reusable_gpu.yml similarity index 100% rename from .github/workflows/gpu.yml rename to .github/workflows/reusable_gpu.yml diff --git a/.github/workflows/multi_numa.yml b/.github/workflows/reusable_multi_numa.yml similarity index 100% rename from .github/workflows/multi_numa.yml rename to .github/workflows/reusable_multi_numa.yml diff --git a/.github/workflows/proxy_lib.yml b/.github/workflows/reusable_proxy_lib.yml similarity index 100% rename from .github/workflows/proxy_lib.yml rename to .github/workflows/reusable_proxy_lib.yml diff --git a/.github/workflows/qemu.yml b/.github/workflows/reusable_qemu.yml similarity index 92% rename from .github/workflows/qemu.yml rename to .github/workflows/reusable_qemu.yml index 7834a8b318..7d6724cdd6 100644 --- a/.github/workflows/qemu.yml +++ b/.github/workflows/reusable_qemu.yml @@ -29,7 +29,14 @@ jobs: sudo apt-get update sudo apt-get install -y qemu-system genisoimage qemu-utils \ libvirt-clients libvirt-daemon-system libvirt-daemon virtinst bridge-utils - pip install -r umf/scripts/qemu/requirements.txt + + # Latest distros do not allow global pip installation + - name: Install Python requirements in venv + run: | + python3 -m venv .venv + . .venv/bin/activate + echo "$PATH" >> $GITHUB_PATH + python3 -m pip install -r umf/scripts/qemu/requirements.txt - name: Add user to kvm group run: sudo usermod -a -G kvm,libvirt $USER diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/reusable_sanitizers.yml similarity index 100% rename from .github/workflows/sanitizers.yml rename to .github/workflows/reusable_sanitizers.yml diff --git a/.github/workflows/trivy.yml b/.github/workflows/reusable_trivy.yml similarity index 69% rename from .github/workflows/trivy.yml rename to .github/workflows/reusable_trivy.yml index 21a76d0cd0..c10229276c 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/reusable_trivy.yml @@ -1,22 +1,7 @@ # Runs linter for Docker files name: Trivy -# Due to lower score on Scorecard we're running this separately from -# "PR/push" workflow. For some reason permissions weren't properly set -# or recognized (by Scorecard). If Scorecard changes its behavior we can -# use 'workflow_call' trigger. -on: - push: - branches-ignore: - - 'dependabot/**' - pull_request: - paths: - - '.github/docker/*Dockerfile' - - '.github/workflows/trivy.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true +on: workflow_call permissions: contents: read diff --git a/.github/workflows/valgrind.yml b/.github/workflows/reusable_valgrind.yml similarity index 69% rename from .github/workflows/valgrind.yml rename to .github/workflows/reusable_valgrind.yml index 40d4e65352..86ceb68c68 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/reusable_valgrind.yml @@ -1,3 +1,4 @@ +# Run tests with valgrind intstrumentation tools: memcheck, drd, helgrind name: Valgrind on: workflow_call @@ -8,10 +9,6 @@ permissions: jobs: valgrind: name: Valgrind - strategy: - fail-fast: false - matrix: - tool: ['memcheck', 'drd', 'helgrind'] runs-on: ubuntu-latest steps: @@ -42,5 +39,11 @@ jobs: - name: Build run: cmake --build ${{github.workspace}}/build --config Debug -j$(nproc) - - name: Run tests under valgrind - run: ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{github.workspace}}/build ${{matrix.tool}} + - name: Run tests with 'memcheck' + run: ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{github.workspace}}/build memcheck + + - name: Run tests with 'drd' + run: ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{github.workspace}}/build drd + + - name: Run tests with 'helgrind' + run: ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{github.workspace}}/build helgrind diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml deleted file mode 100644 index dbd6f1c8e6..0000000000 --- a/.github/workflows/spellcheck.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Checks spelling issues in the repo -name: SpellCheck - -on: workflow_call - -permissions: - contents: read - -jobs: - analyze: - name: Run spell check - runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} - - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Run a spell check - uses: crate-ci/typos@b63f421581dce830bda2f597a678cb7776b41877 # v1.18.2 - with: - config: ./.github/workflows/.spellcheck-conf.toml diff --git a/test/test_valgrind.sh b/test/test_valgrind.sh index 4b8b25b3bc..262a867841 100755 --- a/test/test_valgrind.sh +++ b/test/test_valgrind.sh @@ -145,6 +145,8 @@ for test in $(ls -1 umf_test-*); do fi || true done +rm -rf ${BUILD_DIR}/test/cpuid + [ $ANY_TEST_FAILED -eq 0 ] && echo PASSED && exit 0 echo