From 59952be8ed3a949607a0f5422234b2fee17e7e5d Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 24 Sep 2024 15:05:05 +0200 Subject: [PATCH 1/5] Add running standalone examples Signed-off-by: Lukasz Dorau --- .github/workflows/gpu.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/gpu.yml b/.github/workflows/gpu.yml index 279f977b1b..6eae1498a0 100644 --- a/.github/workflows/gpu.yml +++ b/.github/workflows/gpu.yml @@ -100,6 +100,10 @@ jobs: working-directory: ${{env.BUILD_DIR}} run: ctest --output-on-failure --test-dir examples -C ${{env.BUILD_TYPE}} + - name: Run standalone examples + working-directory: ${{env.BUILD_DIR}} + run: ctest --output-on-failure -R umf_standalone_examples -C ${{env.BUILD_TYPE}} + - name: Run benchmarks working-directory: ${{env.BUILD_DIR}} run: ctest --output-on-failure --test-dir benchmark -C ${{env.BUILD_TYPE}} --exclude-regex umf-bench-multithreaded @@ -163,6 +167,10 @@ jobs: working-directory: ${{env.BUILD_DIR}} run: ctest --output-on-failure --test-dir examples -C ${{env.BUILD_TYPE}} + - name: Run standalone examples + working-directory: ${{env.BUILD_DIR}} + run: ctest --output-on-failure -R umf_standalone_examples -C ${{env.BUILD_TYPE}} + - name: Run benchmarks working-directory: ${{env.BUILD_DIR}} run: ctest --output-on-failure --test-dir benchmark -C ${{env.BUILD_TYPE}} --exclude-regex umf-bench-multithreaded From 058dcdd5ae0016db2fbbf89d18fb0883a2f7eb1d Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 24 Sep 2024 15:16:06 +0200 Subject: [PATCH 2/5] VERBOSE --- .github/workflows/gpu.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/gpu.yml b/.github/workflows/gpu.yml index 6eae1498a0..ed15209fc3 100644 --- a/.github/workflows/gpu.yml +++ b/.github/workflows/gpu.yml @@ -94,19 +94,19 @@ jobs: - name: Run tests working-directory: ${{env.BUILD_DIR}} - run: ctest -C ${{env.BUILD_TYPE}} --output-on-failure --test-dir test + run: ctest -C ${{env.BUILD_TYPE}} -V --test-dir test - name: Run examples working-directory: ${{env.BUILD_DIR}} - run: ctest --output-on-failure --test-dir examples -C ${{env.BUILD_TYPE}} + run: ctest -V --test-dir examples -C ${{env.BUILD_TYPE}} - name: Run standalone examples working-directory: ${{env.BUILD_DIR}} - run: ctest --output-on-failure -R umf_standalone_examples -C ${{env.BUILD_TYPE}} + run: ctest -V -R umf_standalone_examples -C ${{env.BUILD_TYPE}} - name: Run benchmarks working-directory: ${{env.BUILD_DIR}} - run: ctest --output-on-failure --test-dir benchmark -C ${{env.BUILD_TYPE}} --exclude-regex umf-bench-multithreaded + run: ctest -V --test-dir benchmark -C ${{env.BUILD_TYPE}} --exclude-regex umf-bench-multithreaded gpu-CUDA: name: Build @@ -161,16 +161,16 @@ jobs: - name: Run tests working-directory: ${{env.BUILD_DIR}} - run: ctest -C ${{env.BUILD_TYPE}} --output-on-failure --test-dir test + run: ctest -C ${{env.BUILD_TYPE}} -V --test-dir test - name: Run examples working-directory: ${{env.BUILD_DIR}} - run: ctest --output-on-failure --test-dir examples -C ${{env.BUILD_TYPE}} + run: ctest -V --test-dir examples -C ${{env.BUILD_TYPE}} - name: Run standalone examples working-directory: ${{env.BUILD_DIR}} - run: ctest --output-on-failure -R umf_standalone_examples -C ${{env.BUILD_TYPE}} + run: ctest -V -R umf_standalone_examples -C ${{env.BUILD_TYPE}} - name: Run benchmarks working-directory: ${{env.BUILD_DIR}} - run: ctest --output-on-failure --test-dir benchmark -C ${{env.BUILD_TYPE}} --exclude-regex umf-bench-multithreaded + run: ctest -V --test-dir benchmark -C ${{env.BUILD_TYPE}} --exclude-regex umf-bench-multithreaded From 9cd9eb4b50892f4e2a9ed870af0c48780d863163 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 24 Sep 2024 15:18:21 +0200 Subject: [PATCH 3/5] DEBUG CI Signed-off-by: Lukasz Dorau --- .github/workflows/bandit.yml | 35 ----------- .github/workflows/codeql.yml | 94 ---------------------------- .github/workflows/docs.yml | 64 ------------------- .github/workflows/gpu.yml | 6 +- .github/workflows/pr_push.yml | 112 ---------------------------------- .github/workflows/trivy.yml | 57 ----------------- 6 files changed, 5 insertions(+), 363 deletions(-) delete mode 100644 .github/workflows/bandit.yml delete mode 100644 .github/workflows/codeql.yml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/pr_push.yml delete mode 100644 .github/workflows/trivy.yml diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml deleted file mode 100644 index acb64034b9..0000000000 --- a/.github/workflows/bandit.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Bandit static analysis (for Python code) -name: Bandit -on: - push: - branches-ignore: - - 'dependabot/**' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - bandit: - name: Bandit - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ (matrix.os == 'ubuntu-latest' && github.repository_owner == 'oneapi-src') && 'intel-ubuntu-22.04' || matrix.os }} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install Bandit - run: python3 -m pip install bandit - - # Run Bandit recursively, but omit _deps directory (with 3rd party code) - - name: Run Bandit - run: python3 -m bandit -r . -x '/_deps/' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index a444234205..0000000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,94 +0,0 @@ -# CodeQL static analysis -name: CodeQL - -# Due to lower score on Scorecard we're running this separately from -# "PR/push" workflow. For some reason permissions weren't properly set -# or recognized (by Scorecard). If Scorecard changes its behavior we can -# go back to use 'workflow_call' trigger. -on: - push: - branches-ignore: - - 'dependabot/**' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -env: - BUILD_DIR : "${{github.workspace}}/build" - INSTL_DIR : "${{github.workspace}}/../install-dir" - -jobs: - analyze: - name: Analyze - permissions: - security-events: write - env: - VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - include: - - os: ubuntu-latest - # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command - extra_build_option: '-DCMAKE_BUILD_TYPE=Release' - - os: windows-latest - runs-on: ${{matrix.os}} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Initialize CodeQL - uses: github/codeql-action/init@b7bf0a3ed3ecfa44160715d7c442788f65f0f923 # v3.23.2 - with: - languages: cpp - - - name: Initialize vcpkg - if: ${{ matrix.os == 'windows-latest' }} - uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 - with: - vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289 - vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg - vcpkgJsonGlob: '**/vcpkg.json' - - - name: Install dependencies - if: ${{ matrix.os == 'windows-latest' }} - run: vcpkg install - shell: pwsh # Specifies PowerShell as the shell for running the script. - - - name: Install apt packages - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt-get update - sudo apt-get install -y cmake clang libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev - - - name: Install pip packages - run: python3 -m pip install -r third_party/requirements.txt - - - name: Configure CMake - run: > - cmake - -B ${{env.BUILD_DIR}} - ${{matrix.extra_build_option}} - -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" - -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" - -DUMF_FORMAT_CODE_STYLE=OFF - -DUMF_DEVELOPER_MODE=ON - -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON - -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON - -DUMF_BUILD_CUDA_PROVIDER=ON - -DUMF_TESTS_FAIL_ON_SKIP=ON - - - name: Build - run: cmake --build ${{env.BUILD_DIR}} --config Release -j - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@b7bf0a3ed3ecfa44160715d7c442788f65f0f923 # v3.23.2 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 6e128b6037..0000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,64 +0,0 @@ -# Deploy documentation to GitHub Pages -name: GitHubPages - -on: - push: - branches: ["main"] - -# Cancel previous in-progress workflow, only the latest run is relevant -concurrency: - group: "docs" - cancel-in-progress: true - -permissions: - contents: read - -jobs: - build: - name: Build docs - runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install doxygen - run: | - sudo apt-get update - sudo apt-get install -y doxygen - - - name: Install pip requirements - run: python3 -m pip install -r third_party/requirements.txt - - - name: Setup PATH for python - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Build the documentation - working-directory: scripts - run: python3 generate_docs.py - - - name: Upload artifact - uses: actions/upload-pages-artifact@0252fc4ba7626f0298f0cf00902a25c6afc77fa8 # v3.0.0 - with: - path: docs/html - - deploy: - name: Deploy docs to GitHub Pages - needs: build - - permissions: - pages: write - id-token: write - - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - - runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} - - steps: - - name: Deploy the documentation to GitHub Pages - id: deployment - uses: actions/deploy-pages@87c3283f01cd6fe19a0ab93a23b2f6fcba5a8e42 # v4.0.3 diff --git a/.github/workflows/gpu.yml b/.github/workflows/gpu.yml index ed15209fc3..1c14ef4853 100644 --- a/.github/workflows/gpu.yml +++ b/.github/workflows/gpu.yml @@ -4,7 +4,11 @@ name: GPU -on: [workflow_call] +on: + push: + branches-ignore: + - 'dependabot/**' + pull_request: permissions: contents: read diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml deleted file mode 100644 index 02b7adf9f8..0000000000 --- a/.github/workflows/pr_push.yml +++ /dev/null @@ -1,112 +0,0 @@ -# Checks required for a PR to merge. This workflow mostly call other workflows. -name: PR/push - -on: - push: - branches-ignore: - - 'dependabot/**' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - CodeStyle: - name: Coding style - runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install apt packages - run: | - sudo apt-get update - sudo apt-get install -y black cmake clang-format-15 cmake-format libhwloc-dev - - - name: Configure CMake - run: > - cmake - -B ${{github.workspace}}/build - -DUMF_FORMAT_CODE_STYLE=ON - -DUMF_BUILD_TESTS=OFF - -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF - -DUMF_BUILD_CUDA_PROVIDER=OFF - -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=OFF - - - name: Check C/C++ formatting - run: cmake --build build --target clang-format-check - - - name: Check CMake formatting - run: | - cmake --build build --target cmake-format-apply - git diff --exit-code - - - name: Check Python formatting - run: cmake --build build --target black-format-check - - DocsBuild: - name: Build docs - runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install doxygen - run: | - sudo apt-get update - sudo apt-get install -y doxygen - - - name: Install pip requirements - run: python3 -m pip install -r third_party/requirements.txt - - - name: Setup PATH for python - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Build the documentation - working-directory: scripts - run: python3 generate_docs.py - - Spellcheck: - uses: ./.github/workflows/spellcheck.yml - FastBuild: - name: Fast builds - needs: [Spellcheck, CodeStyle] - uses: ./.github/workflows/fast.yml - Build: - name: Basic builds - needs: [FastBuild] - uses: ./.github/workflows/basic.yml - DevDax: - needs: [FastBuild] - uses: ./.github/workflows/devdax.yml - Sanitizers: - needs: [FastBuild] - uses: ./.github/workflows/sanitizers.yml - Qemu: - needs: [FastBuild] - uses: ./.github/workflows/qemu.yml - Benchmarks: - needs: [Build] - uses: ./.github/workflows/benchmarks.yml - ProxyLib: - needs: [Build] - uses: ./.github/workflows/proxy_lib.yml - GPU: - needs: [Build] - uses: ./.github/workflows/gpu.yml - Valgrind: - needs: [Build] - uses: ./.github/workflows/valgrind.yml - MultiNuma: - needs: [Build] - uses: ./.github/workflows/multi_numa.yml diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml deleted file mode 100644 index 21a76d0cd0..0000000000 --- a/.github/workflows/trivy.yml +++ /dev/null @@ -1,57 +0,0 @@ -# Runs linter for Docker files -name: Trivy - -# Due to lower score on Scorecard we're running this separately from -# "PR/push" workflow. For some reason permissions weren't properly set -# or recognized (by Scorecard). If Scorecard changes its behavior we can -# use 'workflow_call' trigger. -on: - push: - branches-ignore: - - 'dependabot/**' - pull_request: - paths: - - '.github/docker/*Dockerfile' - - '.github/workflows/trivy.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - trivy: - name: Trivy - runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} - permissions: - security-events: write - - steps: - - name: Clone the git repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Run Trivy - uses: aquasecurity/trivy-action@84384bd6e777ef152729993b8145ea352e9dd3ef # v0.17.0 - with: - scan-type: 'config' - hide-progress: false - format: 'sarif' - output: 'trivy-results.sarif' - exit-code: 1 # Fail if issue found - # file with suppressions: .trivyignore (in root dir) - - - name: Print report and trivyignore file - run: | - echo "### Trivy ignore content:" - cat .trivyignore - echo "### Trivy report:" - cat trivy-results.sarif - - - name: Upload results - uses: github/codeql-action/upload-sarif@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3.24.0 - with: - sarif_file: 'trivy-results.sarif' From ab35b238eabe770280bfe8d92abc99ec627ad43a Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 24 Sep 2024 15:25:08 +0200 Subject: [PATCH 4/5] Remove benchmarks Signed-off-by: Lukasz Dorau --- .github/workflows/gpu.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/gpu.yml b/.github/workflows/gpu.yml index 1c14ef4853..e0f55edc92 100644 --- a/.github/workflows/gpu.yml +++ b/.github/workflows/gpu.yml @@ -106,11 +106,7 @@ jobs: - name: Run standalone examples working-directory: ${{env.BUILD_DIR}} - run: ctest -V -R umf_standalone_examples -C ${{env.BUILD_TYPE}} - - - name: Run benchmarks - working-directory: ${{env.BUILD_DIR}} - run: ctest -V --test-dir benchmark -C ${{env.BUILD_TYPE}} --exclude-regex umf-bench-multithreaded + run: ctest -V -R standalone -C ${{env.BUILD_TYPE}} gpu-CUDA: name: Build @@ -174,7 +170,3 @@ jobs: - name: Run standalone examples working-directory: ${{env.BUILD_DIR}} run: ctest -V -R umf_standalone_examples -C ${{env.BUILD_TYPE}} - - - name: Run benchmarks - working-directory: ${{env.BUILD_DIR}} - run: ctest -V --test-dir benchmark -C ${{env.BUILD_TYPE}} --exclude-regex umf-bench-multithreaded From bc70f614bbad7ab277fe5da2a17ba6f4ab93deb4 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 24 Sep 2024 15:49:46 +0200 Subject: [PATCH 5/5] Test where standalone examples are run Signed-off-by: Lukasz Dorau --- examples/ipc_level_zero/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/ipc_level_zero/CMakeLists.txt b/examples/ipc_level_zero/CMakeLists.txt index 18b9f542ee..28557b0d31 100644 --- a/examples/ipc_level_zero/CMakeLists.txt +++ b/examples/ipc_level_zero/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright (C) 2024 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +message(FATAL_ERROR "QQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ") cmake_minimum_required(VERSION 3.14.0 FATAL_ERROR) project(umf_example_ipc_level_zero LANGUAGES C)