From 0c7f0250a5bd77408b2a9f5905cf63b35555c385 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 10 Jan 2025 12:47:03 +0100 Subject: [PATCH 1/9] Use Ubuntu 22.04 in pre-commit and building docs --- .github/workflows/build-sphinx.yml | 2 +- .github/workflows/pre-commit.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-sphinx.yml b/.github/workflows/build-sphinx.yml index 2cba3f30a3d9..0f7516a01701 100644 --- a/.github/workflows/build-sphinx.yml +++ b/.github/workflows/build-sphinx.yml @@ -25,7 +25,7 @@ jobs: build-and-deploy: name: Build and Deploy Docs - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: # Needed to cancel any previous runs that are not completed for a given workflow diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 59b10427b09c..b9ca3f361e88 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -9,7 +9,7 @@ permissions: read-all jobs: pre-commit: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Set up clang-format run: | From 15d25120b52c4a6c2ed12dee19365eb94d4cf805 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 10 Jan 2025 12:47:46 +0100 Subject: [PATCH 2/9] Use fail-fast instead of continue-on-error in nightly run --- .github/workflows/cron-run-tests.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index 8d4d3745ee02..528321570c42 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -33,12 +33,11 @@ jobs: actions: write strategy: + fail-fast: false matrix: python: ['3.9', '3.10', '3.11', '3.12', '3.13'] runner: [ubuntu-22.04, ubuntu-24.04, windows-2019] - continue-on-error: false - steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@85880fa0301c86cca9da44039ee3bb12d3bedbfa # 0.12.1 From fb3d2af9501c0c523b080eadbb23d6c5fc9e929f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 10 Jan 2025 13:23:12 +0100 Subject: [PATCH 3/9] Rerun tests in nightly workflow based on condition --- .github/workflows/cron-run-tests.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index 528321570c42..d17795ab5c02 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -15,7 +15,6 @@ env: PACKAGE_NAME: dpnp CHANNELS: '-c dppy/label/dev -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels' TEST_ENV_NAME: test - RERUN_TESTS_ON_FAILURE: 'true' RUN_TESTS_MAX_ATTEMPTS: 2 jobs: @@ -94,14 +93,14 @@ jobs: python -c "import dpnp; print(dpnp.__version__)" - name: Run tests - if: env.RERUN_TESTS_ON_FAILURE != 'true' + id: run_tests run: | python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests env: SYCL_CACHE_PERSISTENT: 1 - name: ReRun tests on Linux - if: env.RERUN_TESTS_ON_FAILURE == 'true' && matrix.runner != 'windows-2019' + if: steps.run_tests.outcome == 'failure' && matrix.runner != 'windows-2019' id: run_tests_linux uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 with: @@ -118,7 +117,7 @@ jobs: SYCL_CACHE_PERSISTENT: 1 - name: ReRun tests on Windows - if: env.RERUN_TESTS_ON_FAILURE == 'true' && matrix.runner == 'windows-2019' + if: steps.run_tests.outcome == 'failure' && matrix.runner == 'windows-2019' id: run_tests_win uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 with: From 7e15e74f2123a0285d30367fd24189f07b1964b8 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 10 Jan 2025 13:40:32 +0100 Subject: [PATCH 4/9] GH automatically appends python and runner OS to job title --- .github/workflows/check-mkl-interfaces.yaml | 4 ++-- .github/workflows/conda-package.yml | 8 ++++---- .github/workflows/cron-run-tests.yaml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/check-mkl-interfaces.yaml b/.github/workflows/check-mkl-interfaces.yaml index 3ee687bf5d47..9dfeeda7e91a 100644 --- a/.github/workflows/check-mkl-interfaces.yaml +++ b/.github/workflows/check-mkl-interfaces.yaml @@ -28,7 +28,7 @@ env: jobs: test_by_tag: - name: Run on ['${{ matrix.os }}', python='${{ matrix.python }}'] with oneMKL tag + name: Run tests with oneMKL tag strategy: matrix: @@ -121,7 +121,7 @@ jobs: SYCL_CACHE_PERSISTENT: 1 test_by_branch: - name: Run on ['${{ matrix.os }}', python='${{ matrix.python }}'] with oneMKL develop branch + name: Run tests with oneMKL develop branch strategy: matrix: diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 23ed4fe904e1..d03f037ee8df 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -25,7 +25,7 @@ env: jobs: build: - name: Build ['${{ matrix.os }}', python='${{ matrix.python }}'] + name: Build strategy: matrix: @@ -96,7 +96,7 @@ jobs: path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl test_linux: - name: Test ['ubuntu-latest', python='${{ matrix.python }}'] + name: Test needs: build @@ -199,7 +199,7 @@ jobs: python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests test_windows: - name: Test ['windows-2019', python='${{ matrix.python }}'] + name: Test needs: build @@ -335,7 +335,7 @@ jobs: python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests upload: - name: Upload ['${{ matrix.os }}', python='${{ matrix.python }}'] + name: Upload needs: [test_linux, test_windows] diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index d17795ab5c02..ec49466aff4b 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -19,7 +19,7 @@ env: jobs: test: - name: Test ['${{ matrix.runner }}', python='${{ matrix.python }}'] + name: Test runs-on: ${{ matrix.runner }} From 23c4907d938dc5d113cd61eeb3c1183bb246145e Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 10 Jan 2025 13:42:52 +0100 Subject: [PATCH 5/9] ReSetup miniconda on failure --- .github/workflows/conda-package.yml | 6 ++++-- .github/workflows/cron-run-tests.yaml | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index d03f037ee8df..499de400c68b 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -100,7 +100,7 @@ jobs: needs: build - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} defaults: run: @@ -109,6 +109,7 @@ jobs: strategy: matrix: python: ['3.9', '3.10', '3.11', '3.12', '3.13'] + os: [ubuntu-latest] continue-on-error: true @@ -203,7 +204,7 @@ jobs: needs: build - runs-on: windows-2019 + runs-on: ${{ matrix.os }} defaults: run: @@ -212,6 +213,7 @@ jobs: strategy: matrix: python: ['3.9', '3.10', '3.11', '3.12', '3.13'] + os: [windows-2019] continue-on-error: true diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index ec49466aff4b..3ebe4485f5a8 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -55,6 +55,19 @@ jobs: echo "Latest tag is ${{ steps.find_latest_tag.outputs.tag }}" - name: Setup miniconda + id: setup_miniconda + continue-on-error: true + uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0 + with: + miniforge-version: latest + use-mamba: 'true' + channels: conda-forge + conda-remove-defaults: 'true' + python-version: ${{ matrix.python }} + activate-environment: ${{ env.TEST_ENV_NAME }} + + - name: ReSetup miniconda + if: steps.setup_miniconda.outcome == 'failure' uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0 with: miniforge-version: latest @@ -94,6 +107,7 @@ jobs: - name: Run tests id: run_tests + continue-on-error: true run: | python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests env: From 74715ae9c0949ac66f1a1c01d3271ea6e4110a3f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 10 Jan 2025 16:09:18 +0100 Subject: [PATCH 6/9] Align with changes in dpctl::tensor::offset_utils::device_allocate_and_pack --- .../elementwise_functions.hpp | 63 ++++++------------- 1 file changed, 19 insertions(+), 44 deletions(-) diff --git a/dpnp/backend/extensions/elementwise_functions/elementwise_functions.hpp b/dpnp/backend/extensions/elementwise_functions/elementwise_functions.hpp index f1e22968aa20..324cc488f68a 100644 --- a/dpnp/backend/extensions/elementwise_functions/elementwise_functions.hpp +++ b/dpnp/backend/extensions/elementwise_functions/elementwise_functions.hpp @@ -218,28 +218,21 @@ std::pair std::vector host_tasks{}; host_tasks.reserve(2); - const auto &ptr_size_event_triple_ = device_allocate_and_pack( + auto ptr_size_event_triple_ = device_allocate_and_pack( q, host_tasks, simplified_shape, simplified_src_strides, simplified_dst_strides); - py::ssize_t *shape_strides = std::get<0>(ptr_size_event_triple_); - const sycl::event ©_shape_ev = std::get<2>(ptr_size_event_triple_); - - if (shape_strides == nullptr) { - throw std::runtime_error("Device memory allocation failed"); - } + auto shape_strides_owner = std::move(std::get<0>(ptr_size_event_triple_)); + const auto ©_shape_ev = std::get<2>(ptr_size_event_triple_); + const py::ssize_t *shape_strides = shape_strides_owner.get(); sycl::event strided_fn_ev = strided_fn(q, src_nelems, nd, shape_strides, src_data, src_offset, dst_data, dst_offset, depends, {copy_shape_ev}); // async free of shape_strides temporary - auto ctx = q.get_context(); - sycl::event tmp_cleanup_ev = q.submit([&](sycl::handler &cgh) { - cgh.depends_on(strided_fn_ev); - using dpctl::tensor::alloc_utils::sycl_free_noexcept; - cgh.host_task( - [ctx, shape_strides]() { sycl_free_noexcept(shape_strides, ctx); }); - }); + sycl::event tmp_cleanup_ev = dpctl::tensor::alloc_utils::async_smart_free( + q, {strided_fn_ev}, shape_strides_owner); + host_tasks.push_back(tmp_cleanup_ev); return std::make_pair( @@ -543,30 +536,21 @@ std::pair py_binary_ufunc( } using dpctl::tensor::offset_utils::device_allocate_and_pack; - const auto &ptr_sz_event_triple_ = device_allocate_and_pack( + auto ptr_sz_event_triple_ = device_allocate_and_pack( exec_q, host_tasks, simplified_shape, simplified_src1_strides, simplified_src2_strides, simplified_dst_strides); + auto shape_strides_owner = std::move(std::get<0>(ptr_sz_event_triple_)); + auto ©_shape_ev = std::get<2>(ptr_sz_event_triple_); - py::ssize_t *shape_strides = std::get<0>(ptr_sz_event_triple_); - const sycl::event ©_shape_ev = std::get<2>(ptr_sz_event_triple_); - - if (shape_strides == nullptr) { - throw std::runtime_error("Unable to allocate device memory"); - } + const py::ssize_t *shape_strides = shape_strides_owner.get(); sycl::event strided_fn_ev = strided_fn( exec_q, src_nelems, nd, shape_strides, src1_data, src1_offset, src2_data, src2_offset, dst_data, dst_offset, depends, {copy_shape_ev}); // async free of shape_strides temporary - auto ctx = exec_q.get_context(); - - sycl::event tmp_cleanup_ev = exec_q.submit([&](sycl::handler &cgh) { - cgh.depends_on(strided_fn_ev); - using dpctl::tensor::alloc_utils::sycl_free_noexcept; - cgh.host_task( - [ctx, shape_strides]() { sycl_free_noexcept(shape_strides, ctx); }); - }); + sycl::event tmp_cleanup_ev = dpctl::tensor::alloc_utils::async_smart_free( + exec_q, {strided_fn_ev}, shape_strides_owner); host_tasks.push_back(tmp_cleanup_ev); @@ -796,30 +780,21 @@ std::pair } using dpctl::tensor::offset_utils::device_allocate_and_pack; - const auto &ptr_sz_event_triple_ = device_allocate_and_pack( + auto ptr_sz_event_triple_ = device_allocate_and_pack( exec_q, host_tasks, simplified_shape, simplified_rhs_strides, simplified_lhs_strides); + auto shape_strides_owner = std::move(std::get<0>(ptr_sz_event_triple_)); + auto copy_shape_ev = std::get<2>(ptr_sz_event_triple_); - py::ssize_t *shape_strides = std::get<0>(ptr_sz_event_triple_); - const sycl::event ©_shape_ev = std::get<2>(ptr_sz_event_triple_); - - if (shape_strides == nullptr) { - throw std::runtime_error("Unable to allocate device memory"); - } + const py::ssize_t *shape_strides = shape_strides_owner.get(); sycl::event strided_fn_ev = strided_fn(exec_q, rhs_nelems, nd, shape_strides, rhs_data, rhs_offset, lhs_data, lhs_offset, depends, {copy_shape_ev}); // async free of shape_strides temporary - auto ctx = exec_q.get_context(); - - sycl::event tmp_cleanup_ev = exec_q.submit([&](sycl::handler &cgh) { - cgh.depends_on(strided_fn_ev); - using dpctl::tensor::alloc_utils::sycl_free_noexcept; - cgh.host_task( - [ctx, shape_strides]() { sycl_free_noexcept(shape_strides, ctx); }); - }); + sycl::event tmp_cleanup_ev = dpctl::tensor::alloc_utils::async_smart_free( + exec_q, {strided_fn_ev}, shape_strides_owner); host_tasks.push_back(tmp_cleanup_ev); From addd690eeca6c228fc3b69f11911af9b924821e4 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 10 Jan 2025 18:42:41 +0100 Subject: [PATCH 7/9] ReInstall dpnp on failure --- .github/workflows/cron-run-tests.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index 3ebe4485f5a8..e7bb7b83bc4c 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -78,10 +78,14 @@ jobs: activate-environment: ${{ env.TEST_ENV_NAME }} - name: Install dpnp + id: install_dpnp + run: | + mamba install ${{ env.PACKAGE_NAME }}=${{ steps.find_latest_tag.outputs.tag }} pytest ${{ env.CHANNELS }} + + - name: ReInstall dpnp + if: steps.install_dpnp.outcome == 'failure' run: | mamba install ${{ env.PACKAGE_NAME }}=${{ steps.find_latest_tag.outputs.tag }} pytest ${{ env.CHANNELS }} - env: - MAMBA_NO_LOW_SPEED_LIMIT: 1 - name: List installed packages run: mamba list From 41b88638a9bcbe040fbfd5a016d537da14e2feb3 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 10 Jan 2025 18:50:45 +0100 Subject: [PATCH 8/9] Resetup miniconda or reinstall conda-build on failure in Conda package GH action --- .github/workflows/conda-package.yml | 21 ++++++++++++++++++--- .github/workflows/cron-run-tests.yaml | 1 + 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 499de400c68b..07107b2574f6 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -56,6 +56,8 @@ jobs: fetch-depth: 0 - name: Setup miniconda + id: setup_miniconda + continue-on-error: true uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0 with: miniforge-version: latest @@ -65,9 +67,16 @@ jobs: python-version: ${{ env.CONDA_BUILD_INDEX_ENV_PY_VER}} activate-environment: 'build' - # Sometimes `mamba install ...` fails due to slow download speed rate, so disable the check in mamba - - name: Disable speed limit check in mamba - run: echo "MAMBA_NO_LOW_SPEED_LIMIT=1" >> $GITHUB_ENV + - name: ReSetup miniconda + if: steps.setup_miniconda.outcome == 'failure' + uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0 + with: + miniforge-version: latest + use-mamba: 'true' + channels: conda-forge + conda-remove-defaults: 'true' + python-version: ${{ env.CONDA_BUILD_INDEX_ENV_PY_VER}} + activate-environment: 'build' - name: Store conda paths as envs shell: bash -el {0} @@ -76,6 +85,12 @@ jobs: echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE${{ runner.os == 'Linux' && '/' || '\\' }}" >> $GITHUB_ENV - name: Install conda-build + id: install_conda_build + continue-on-error: true + run: mamba install conda-build=${{ env.CONDA_BUILD_VERSION}} + + - name: ReInstall conda-build + if: steps.install_conda_build.outcome == 'failure' run: mamba install conda-build=${{ env.CONDA_BUILD_VERSION}} - name: Build conda package diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index e7bb7b83bc4c..e0836b95111f 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -79,6 +79,7 @@ jobs: - name: Install dpnp id: install_dpnp + continue-on-error: true run: | mamba install ${{ env.PACKAGE_NAME }}=${{ steps.find_latest_tag.outputs.tag }} pytest ${{ env.CHANNELS }} From a571e0685e8416ba22d854a807debdf3fe700e6f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sat, 11 Jan 2025 11:35:18 +0100 Subject: [PATCH 9/9] Disable workflow with nightly tests in forks --- .github/workflows/cron-run-tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index e0836b95111f..d47572fdaa50 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -21,6 +21,9 @@ jobs: test: name: Test + # disable scheduled workflow to be run in forks + if: github.event.repository.fork == false + runs-on: ${{ matrix.runner }} defaults: