diff --git a/.github/workflows/build-sphinx.yml b/.github/workflows/build-sphinx.yml index 05379fee84ce..c34bf409aed9 100644 --- a/.github/workflows/build-sphinx.yml +++ b/.github/workflows/build-sphinx.yml @@ -36,11 +36,12 @@ jobs: pull-requests: write env: - python-ver: '3.12' - CHANNELS: '-c dppy/label/dev -c intel -c conda-forge --override-channels' - NO_INTEL_CHANNELS: '-c dppy/label/dev -c conda-forge --override-channels' - # Install the latest oneAPI compiler to work around an issue - INSTALL_ONE_API: 'yes' + environment-file: 'environments/environment.yml' + build-with-oneapi-env: 'environments/build_with_oneapi.yml' + building-docs-env: 'environments/building_docs.yml' + oneapi-pkgs-env: '' + # Enable env when it's required to use only conda packages without OneAPI installation + # oneapi-pkgs-env: '${{ github.workspace }}/environments/oneapi_pkgs.yml' steps: - name: Cancel Previous Runs @@ -60,6 +61,7 @@ jobs: docker-images: false - name: Add Intel repository + if: env.oneapi-pkgs-env == '' run: | wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB cat GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null @@ -75,6 +77,7 @@ jobs: sudo apt install --reinstall -y gcc-9 g++-9 libstdc++-9-dev - name: Install Intel OneAPI + if: env.oneapi-pkgs-env == '' run: | sudo apt install hwloc \ intel-oneapi-mkl \ @@ -101,6 +104,16 @@ jobs: with: fetch-depth: 0 + - name: Install conda-merge tool + uses: BSFishy/pip-action@8f2d471d809dc20b6ada98c91910b6ae6243f318 # v1 + with: + packages: conda-merge + + - name: Merge conda env files + run: | + conda-merge ${{ env.build-with-oneapi-env }} ${{ env.building-docs-env }} ${{ env.oneapi-pkgs-env }} > ${{ env.environment-file }} + cat ${{ env.environment-file }} + - name: Setup miniconda id: setup_miniconda continue-on-error: true @@ -108,9 +121,8 @@ jobs: with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ env.python-ver }} + environment-file: ${{ env.environment-file }} activate-environment: 'docs' - name: ReSetup miniconda @@ -119,35 +131,10 @@ jobs: with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ env.python-ver }} + environment-file: ${{ env.environment-file }} activate-environment: 'docs' - # Sometimes `mamba install ...` fails due to slow download speed rate, so disable the check in mamba - - name: Disable speed limit check in mamba - run: echo "MAMBA_NO_LOW_SPEED_LIMIT=1" >> $GITHUB_ENV - - - name: Install sphinx dependencies - run: | - mamba install sphinx sphinx_rtd_theme - pip install sphinxcontrib-googleanalytics==0.4 \ - pyenchant sphinxcontrib-spelling - - - name: Install dpnp dependencies - if: env.INSTALL_ONE_API == 'yes' - run: | - mamba install numpy dpctl">=0.18.0dev0" cmake cython pytest ninja scikit-build ${{ env.NO_INTEL_CHANNELS }} - - - name: Install dpnp dependencies - if: env.INSTALL_ONE_API != 'yes' - run: | - mamba install numpy dpctl">=0.18.0dev0" mkl-devel-dpcpp onedpl-devel tbb-devel dpcpp_linux-64 \ - cmake cython pytest ninja scikit-build ${{ env.CHANNELS }} - - - name: Install cuPy dependencies - run: mamba install cupy - - name: Conda info run: mamba info @@ -178,7 +165,6 @@ jobs: echo PROJECT_NUMBER=${PROJECT_NUMBER} echo "PROJECT_NUMBER=$PROJECT_NUMBER" >> $GITHUB_ENV - # https://github.com/marketplace/actions/doxygen-action - name: Build backend docs uses: mattnotmitt/doxygen-action@b84fe17600245bb5db3d6c247cc274ea98c15a3b # v1.12 with: @@ -187,7 +173,6 @@ jobs: - name: Copy backend docs run: cp -r dpnp/backend/doc/html ${{ env.PUBLISH_DIR }}/backend_doc - # https://github.com/marketplace/actions/github-pages-action # The step is only used to build docs while pushing a PR to "master" - name: Deploy docs if: env.GH_EVENT_PUSH_UPSTREAM == 'true' diff --git a/.github/workflows/check-mkl-interfaces.yaml b/.github/workflows/check-mkl-interfaces.yaml index 1e3fed6db5f9..7997a61a081d 100644 --- a/.github/workflows/check-mkl-interfaces.yaml +++ b/.github/workflows/check-mkl-interfaces.yaml @@ -9,76 +9,103 @@ on: permissions: read-all env: - CHANNELS: '-c dppy/label/dev -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels' - TEST_ENV_NAME: 'test_onemkl_interfaces' - RERUN_TESTS_ON_FAILURE: 'true' - RUN_TESTS_MAX_ATTEMPTS: 2 - BUILD_DEP_PKGS: >- - mkl-devel-dpcpp - tbb-devel - dpctl - onedpl-devel - setuptools - python - numpy - cython - cmake - ninja - scikit-build + environment-file: 'environments/environment.yml' + environment-file-name: 'environment.yml' + environment-file-loc: '${{ github.workspace }}/environments' + build-with-oneapi-env: 'environments/build_with_oneapi.yml' + oneapi-pkgs-env: 'environments/oneapi_pkgs.yml' + test-env-name: 'test_onemkl_interfaces' + rerun-tests-on-failure: 'true' + rerun-tests-max-attempts: 2 + rerun-tests-timeout: 20 jobs: + build_env_file: + name: Build conda env file + + permissions: + # Needed to cancel any previous runs that are not completed for a given workflow + actions: write + + runs-on: 'ubuntu-latest' + + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@85880fa0301c86cca9da44039ee3bb12d3bedbfa # 0.12.1 + with: + access_token: ${{ github.token }} + + - name: Checkout DPNP repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - name: Install conda-merge tool + uses: BSFishy/pip-action@8f2d471d809dc20b6ada98c91910b6ae6243f318 # v1 + with: + packages: conda-merge + + - name: Merge conda env files + run: | + conda-merge ${{ env.build-with-oneapi-env }} ${{ env.oneapi-pkgs-env }} > ${{ env.environment-file }} + + - name: Upload artifact + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: ${{ env.environment-file-name }} + path: ${{ env.environment-file }} + test_by_tag: name: Run tests with oneMKL tag + needs: build_env_file + strategy: + fail-fast: false matrix: # python 3.13 is blocked since BLAS requires "mkl<2025.0" (see https://github.com/conda-forge/blas-feedstock/pull/128 # which depends on resolving MKL issue https://github.com/conda-forge/intel_repack-feedstock/issues/83) python: ['3.12'] os: [ubuntu-22.04] # windows-2019 - no DFT support for Windows in oneMKL - permissions: - # Needed to cancel any previous runs that are not completed for a given workflow - actions: write - runs-on: ${{ matrix.os }} defaults: run: shell: ${{ matrix.os == 'windows-2019' && 'cmd /C CALL {0}' || 'bash -el {0}' }} - continue-on-error: false - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@85880fa0301c86cca9da44039ee3bb12d3bedbfa # 0.12.1 - with: - access_token: ${{ github.token }} - - name: Checkout DPNP repo uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 + - name: Download artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: ${{ env.environment-file-name }} + path: ${{ env.environment-file-loc }} + - name: Setup miniconda + id: setup_miniconda + continue-on-error: true uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ matrix.python }} - activate-environment: ${{ env.TEST_ENV_NAME }} - - # Sometimes `mamba install ...` fails due to slow download speed rate, so disable the check in mamba - - name: Disable speed limit check in mamba - run: echo "MAMBA_NO_LOW_SPEED_LIMIT=1" >> $GITHUB_ENV + environment-file: ${{ env.environment-file }} + activate-environment: ${{ env.test-env-name }} - - name: Install dpnp build dependencies - run: | - mamba install ${{ env.DPCPP_PKG }} ${{ env.BUILD_DEP_PKGS }} ${{ env.CHANNELS }} - env: - DPCPP_PKG: ${{ matrix.os == 'windows-2019' && 'dpcpp_win-64 vs_win-64=2017.9' || 'dpcpp_linux-64' }} + - name: ReSetup miniconda + if: steps.setup_miniconda.outcome == 'failure' + uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 + with: + miniforge-version: latest + use-mamba: 'true' + conda-remove-defaults: 'true' + environment-file: ${{ env.environment-file }} + activate-environment: ${{ env.test-env-name }} - name: Conda info run: | @@ -94,29 +121,25 @@ jobs: python -m dpctl -f python -c "import dpnp; print(dpnp.__version__)" - - name: Install pytest - run: | - mamba install pytest ${{ env.CHANNELS }} - - name: Run tests - if: env.RERUN_TESTS_ON_FAILURE != 'true' + if: env.rerun-tests-on-failure != 'true' run: | python -m pytest -ra --pyargs dpnp.tests env: SYCL_CACHE_PERSISTENT: 1 - name: ReRun tests on Linux - if: env.RERUN_TESTS_ON_FAILURE == 'true' + if: env.rerun-tests-on-failure == 'true' id: run_tests uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 with: - timeout_minutes: 12 - max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }} + timeout_minutes: ${{ env.rerun-tests-timeout }} + max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | . $CONDA/etc/profile.d/conda.sh . $CONDA/etc/profile.d/mamba.sh - mamba activate ${{ env.TEST_ENV_NAME }} + mamba activate ${{ env.test-env-name }} python -m pytest -ra --pyargs dpnp.tests env: @@ -125,39 +148,37 @@ jobs: test_by_branch: name: Run tests with oneMKL develop branch + needs: build_env_file + strategy: + fail-fast: false matrix: # python 3.13 is blocked since BLAS requires "mkl<2025.0" (see https://github.com/conda-forge/blas-feedstock/pull/128 # which depends on resolving MKL issue https://github.com/conda-forge/intel_repack-feedstock/issues/83) python: ['3.12'] os: [ubuntu-22.04] # windows-2019 - no DFT support for Windows in oneMKL - permissions: - # Needed to cancel any previous runs that are not completed for a given workflow - actions: write - runs-on: ${{ matrix.os }} defaults: run: shell: ${{ matrix.os == 'windows-2019' && 'cmd /C CALL {0}' || 'bash -el {0}' }} - continue-on-error: true - env: onemkl-source-dir: '${{ github.workspace }}/onemkl/' steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@85880fa0301c86cca9da44039ee3bb12d3bedbfa # 0.12.1 - with: - access_token: ${{ github.token }} - - name: Checkout DPNP repo uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 + - name: Download artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: ${{ env.environment-file-name }} + path: ${{ env.environment-file-loc }} + - name: Checkout oneMKL repo uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -171,24 +192,25 @@ jobs: ls -la ${{ env.onemkl-source-dir }} - name: Setup miniconda + id: setup_miniconda + continue-on-error: true uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ matrix.python }} - activate-environment: ${{ env.TEST_ENV_NAME }} + environment-file: ${{ env.environment-file }} + activate-environment: ${{ env.test-env-name }} - # Sometimes `mamba install ...` fails due to slow download speed rate, so disable the check in mamba - - name: Disable speed limit check in mamba - run: echo "MAMBA_NO_LOW_SPEED_LIMIT=1" >> $GITHUB_ENV - - - name: Install dpnp build dependencies - run: | - mamba install ${{ env.DPCPP_PKG }} ${{ env.BUILD_DEP_PKGS }} ${{ env.CHANNELS }} - env: - DPCPP_PKG: ${{ matrix.os == 'windows-2019' && 'dpcpp_win-64 vs_win-64=2017.9' || 'dpcpp_linux-64' }} + - name: ReSetup miniconda + if: steps.setup_miniconda.outcome == 'failure' + uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 + with: + miniforge-version: latest + use-mamba: 'true' + conda-remove-defaults: 'true' + environment-file: ${{ env.environment-file }} + activate-environment: ${{ env.test-env-name }} - name: Conda info run: | @@ -204,29 +226,25 @@ jobs: python -m dpctl -f python -c "import dpnp; print(dpnp.__version__)" - - name: Install pytest - run: | - mamba install pytest ${{ env.CHANNELS }} - - name: Run tests - if: env.RERUN_TESTS_ON_FAILURE != 'true' + if: env.rerun-tests-on-failure == 'true' run: | python -m pytest -ra --pyargs dpnp.tests env: SYCL_CACHE_PERSISTENT: 1 - name: ReRun tests on Linux - if: env.RERUN_TESTS_ON_FAILURE == 'true' + if: env.rerun-tests-on-failure == 'true' id: run_tests uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 with: - timeout_minutes: 15 - max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }} + timeout_minutes: ${{ env.rerun-tests-timeout }} + max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | . $CONDA/etc/profile.d/conda.sh . $CONDA/etc/profile.d/mamba.sh - mamba activate ${{ env.TEST_ENV_NAME }} + mamba activate ${{ env.test-env-name }} python -m pytest -ra --pyargs dpnp.tests env: diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 7c43666a670a..ca762d6b7de1 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -9,22 +9,20 @@ on: permissions: read-all env: - PACKAGE_NAME: dpnp + package-name: dpnp # Follow oneAPI installation instruction for conda, since intel channel is not longer available - # CHANNELS: '-c dppy/label/dev -c intel -c conda-forge --override-channels' - CHANNELS: '-c dppy/label/dev -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels' - CONDA_BUILD_INDEX_ENV_PY_VER: '3.12' # conda does not support python 3.13 - CONDA_BUILD_VERSION: '25.1.1' - CONDA_INDEX_VERSION: '0.5.0' + # channels-list: '-c dppy/label/dev -c intel -c conda-forge --override-channels' + channels-list: '-c dppy/label/dev -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels' + ver-script-part1: "import json; f = open('version.json', 'r'); j = json.load(f); f.close(); " + ver-script-part2: "d = j['dpnp'][0]; print('='.join((d[s] for s in ('version', 'build'))))" + fetch-depth: 1 # python 3.13 is blocked since BLAS requires "mkl<2025.0" (see https://github.com/conda-forge/blas-feedstock/pull/128 # which depends on resolving MKL issue https://github.com/conda-forge/intel_repack-feedstock/issues/83) - LATEST_PYTHON: '3.12' - RERUN_TESTS_ON_FAILURE: 'true' - RUN_TESTS_MAX_ATTEMPTS: 2 - TEST_ENV_NAME: 'test' - VER_JSON_NAME: 'version.json' - VER_SCRIPT1: "import json; f = open('version.json', 'r'); j = json.load(f); f.close(); " - VER_SCRIPT2: "d = j['dpnp'][0]; print('='.join((d[s] for s in ('version', 'build'))))" + python-ver-test-all-dtypes: '3.12' + test-env-name: 'test' + rerun-tests-on-failure: 'true' + rerun-tests-max-attempts: 2 + rerun-tests-timeout: 35 jobs: build: @@ -48,6 +46,10 @@ jobs: run: shell: ${{ matrix.os == 'windows-2019' && 'cmd /C CALL {0}' || 'bash -el {0}' }} + env: + build-conda-pkg-env: 'environments/build_conda_pkg.yml' + build-env-name: 'build' + steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@85880fa0301c86cca9da44039ee3bb12d3bedbfa # 0.12.1 @@ -66,10 +68,9 @@ jobs: with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ env.CONDA_BUILD_INDEX_ENV_PY_VER}} - activate-environment: 'build' + environment-file: ${{ env.build-conda-pkg-env }} + activate-environment: ${{ env.build-env-name }} - name: ReSetup miniconda if: steps.setup_miniconda.outcome == 'failure' @@ -77,10 +78,9 @@ jobs: with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ env.CONDA_BUILD_INDEX_ENV_PY_VER}} - activate-environment: 'build' + environment-file: ${{ env.build-conda-pkg-env }} + activate-environment: ${{ env.build-env-name }} - name: Store conda paths as envs shell: bash -el {0} @@ -88,31 +88,30 @@ jobs: echo "CONDA_BLD=$CONDA_PREFIX/conda-bld/${{ runner.os == 'Linux' && 'linux' || 'win' }}-64/" | tr "\\\\" '/' >> $GITHUB_ENV echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE${{ runner.os == 'Linux' && '/' || '\\' }}" >> $GITHUB_ENV - - name: Install conda-build - id: install_conda_build + - name: Build conda package + id: build_conda_pkg continue-on-error: true - run: mamba install conda-build=${{ env.CONDA_BUILD_VERSION}} - - - name: ReInstall conda-build - if: steps.install_conda_build.outcome == 'failure' - run: mamba install conda-build=${{ env.CONDA_BUILD_VERSION}} + run: conda build --no-test --python ${{ matrix.python }} --numpy 2.0 ${{ env.channels-list }} conda-recipe + env: + MAX_BUILD_CMPL_MKL_VERSION: '2025.1a0' - - name: Build conda package - run: conda build --no-test --python ${{ matrix.python }} --numpy 2.0 ${{ env.CHANNELS }} conda-recipe + - name: ReBuild conda package + if: steps.build_conda_pkg.outcome == 'failure' + run: conda build --no-test --python ${{ matrix.python }} --numpy 2.0 ${{ env.channels-list }} conda-recipe env: MAX_BUILD_CMPL_MKL_VERSION: '2025.1a0' - name: Upload artifact uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python }} - path: ${{ env.CONDA_BLD }}${{ env.PACKAGE_NAME }}-*.conda + name: ${{ env.package-name }} ${{ runner.os }} Python ${{ matrix.python }} + path: ${{ env.CONDA_BLD }}${{ env.package-name }}-*.conda - name: Upload wheels artifact uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl + name: ${{ env.package-name }} ${{ runner.os }} Wheels Python ${{ matrix.python }} + path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.package-name }}-*.whl test_linux: name: Test @@ -133,15 +132,23 @@ jobs: os: [ubuntu-latest] env: + dpnp-repo-path: '${{ github.workspace }}/source/' + create-conda-channel-env: 'source/environments/create_conda_channel.yml' channel-path: '${{ github.workspace }}/channel/' pkg-path-in-channel: '${{ github.workspace }}/channel/linux-64/' ver-json-path: '${{ github.workspace }}/version.json' steps: + - name: Checkout DPNP repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: ${{ env.fetch-depth }} + path: ${{ env.dpnp-repo-path }} + - name: Download artifact uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python }} + name: ${{ env.package-name }} ${{ runner.os }} Python ${{ matrix.python }} path: ${{ env.pkg-path-in-channel }} - name: Setup miniconda @@ -151,10 +158,9 @@ jobs: with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ env.CONDA_BUILD_INDEX_ENV_PY_VER}} - activate-environment: ${{ env.TEST_ENV_NAME }} + environment-file: ${{ env.create-conda-channel-env }} + activate-environment: ${{ env.test-env-name }} - name: ReSetup miniconda if: steps.setup_miniconda.outcome == 'failure' @@ -162,14 +168,9 @@ jobs: with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ env.CONDA_BUILD_INDEX_ENV_PY_VER}} - activate-environment: ${{ env.TEST_ENV_NAME }} - - - name: Install conda-index - run: | - mamba install conda-index=${{ env.CONDA_INDEX_VERSION }} + environment-file: ${{ env.create-conda-channel-env }} + activate-environment: ${{ env.test-env-name }} - name: Create conda channel run: | @@ -177,17 +178,17 @@ jobs: - name: Test conda channel run: | - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.channel-path }} --override-channels --info --json > ${{ env.ver-json-path }} + conda search ${{ env.package-name }} -c ${{ env.channel-path }} --override-channels --info --json > ${{ env.ver-json-path }} cat ${{ env.ver-json-path }} - name: Get package version run: | - export PACKAGE_VERSION=$(python -c "${{ env.VER_SCRIPT1 }} ${{ env.VER_SCRIPT2 }}") + export PACKAGE_VERSION=$(python -c "${{ env.ver-script-part1 }} ${{ env.ver-script-part2 }}") echo PACKAGE_VERSION=${PACKAGE_VERSION} echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> $GITHUB_ENV - # conda-index does not support python 3.13 + # conda-index does not support python 3.13, but we need to test DPNP package with python 3.13 - name: Remove conda-index run: mamba remove conda-index @@ -195,16 +196,16 @@ jobs: id: install_dpnp continue-on-error: true run: | - mamba install ${{ env.PACKAGE_NAME }}=${{ env.PACKAGE_VERSION }} pytest pytest-xdist python=${{ matrix.python }} ${{ env.TEST_CHANNELS }} + mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} pytest pytest-xdist python=${{ matrix.python }} ${{ env.TEST_CHANNELS }} env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.CHANNELS }}' + TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}' - name: ReInstall dpnp if: steps.install_dpnp.outcome == 'failure' run: | - mamba install ${{ env.PACKAGE_NAME }}=${{ env.PACKAGE_VERSION }} pytest python=${{ matrix.python }} ${{ env.TEST_CHANNELS }} + mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} pytest pytest-xdist python=${{ matrix.python }} ${{ env.TEST_CHANNELS }} env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.CHANNELS }}' + TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}' - name: List installed packages run: mamba list @@ -215,33 +216,33 @@ jobs: python -c "import dpnp; print(dpnp.__version__)" - name: Run tests - if: env.RERUN_TESTS_ON_FAILURE != 'true' + if: env.rerun-tests-on-failure != 'true' run: | - if [[ ${{ matrix.python }} == ${{ env.LATEST_PYTHON }} ]]; then + if [[ ${{ matrix.python }} == ${{ env.python-ver-test-all-dtypes }} ]]; then export DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests + python -m pytest -ra --pyargs ${{ env.package-name }}.tests else - python -m pytest -n auto -ra --pyargs ${{ env.PACKAGE_NAME }}.tests + python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests fi - name: Run tests - if: env.RERUN_TESTS_ON_FAILURE == 'true' + if: env.rerun-tests-on-failure == 'true' id: run_tests_linux uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 with: - timeout_minutes: 25 - max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }} + timeout_minutes: ${{ env.rerun-tests-timeout }} + max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | . $CONDA/etc/profile.d/conda.sh . $CONDA/etc/profile.d/mamba.sh - mamba activate ${{ env.TEST_ENV_NAME }} + mamba activate ${{ env.test-env-name }} - if [[ ${{ matrix.python }} == ${{ env.LATEST_PYTHON }} ]]; then + if [[ ${{ matrix.python }} == ${{ env.python-ver-test-all-dtypes }} ]]; then export DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests + python -m pytest -ra --pyargs ${{ env.package-name }}.tests else - python -m pytest -n auto -ra --pyargs ${{ env.PACKAGE_NAME }}.tests + python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests fi test_windows: @@ -263,16 +264,23 @@ jobs: os: [windows-2019] env: + dpnp-repo-path: '${{ github.workspace }}\source' + create-conda-channel-env: 'source\environments\create_conda_channel.yml' channel-path: '${{ github.workspace }}\channel\' pkg-path-in-channel: '${{ github.workspace }}\channel\win-64\' ver-json-path: '${{ github.workspace }}\version.json' - workdir: '${{ github.workspace }}' steps: + - name: Checkout DPNP repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: ${{ env.fetch-depth }} + path: ${{ env.dpnp-repo-path }} + - name: Download artifact uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python }} + name: ${{ env.package-name }} ${{ runner.os }} Python ${{ matrix.python }} path: ${{ env.pkg-path-in-channel }} - name: Store a path to package archive @@ -287,14 +295,25 @@ jobs: (echo FULL_PACKAGE_PATH=%FULL_PACKAGE_PATH%) >> %GITHUB_ENV% - name: Setup miniconda + id: setup_miniconda + continue-on-error: true uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ env.CONDA_BUILD_INDEX_ENV_PY_VER}} - activate-environment: ${{ env.TEST_ENV_NAME }} + environment-file: ${{ env.create-conda-channel-env }} + activate-environment: ${{ env.test-env-name }} + + - name: ReSetup miniconda + if: steps.setup_miniconda.outcome == 'failure' + uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 + with: + miniforge-version: latest + use-mamba: 'true' + conda-remove-defaults: 'true' + environment-file: ${{ env.create-conda-channel-env }} + activate-environment: ${{ env.test-env-name }} - name: Store conda paths as envs run: | @@ -314,7 +333,7 @@ jobs: - name: Test conda channel run: | @echo on - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.channel-path }} --override-channels --info --json > ${{ env.ver-json-path }} + conda search ${{ env.package-name }} -c ${{ env.channel-path }} --override-channels --info --json > ${{ env.ver-json-path }} - name: Dump version.json run: more ${{ env.ver-json-path }} @@ -322,23 +341,23 @@ jobs: - name: Get package version run: | @echo on - set "SCRIPT=${{ env.VER_SCRIPT1 }} ${{ env.VER_SCRIPT2 }}" + set "SCRIPT=${{ env.ver-script-part1 }} ${{ env.ver-script-part2 }}" FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( set PACKAGE_VERSION=%%F ) echo PACKAGE_VERSION: %PACKAGE_VERSION% (echo PACKAGE_VERSION=%PACKAGE_VERSION%) >> %GITHUB_ENV% - # conda-index does not support python 3.13 + # conda-index does not support python 3.13, but we need to test DPNP package with python 3.13 - name: Remove conda-index run: mamba remove conda-index - name: Install dpnp run: | @echo on - mamba install ${{ env.PACKAGE_NAME }}=${{ env.PACKAGE_VERSION }} pytest pytest-xdist python=${{ matrix.python }} ${{ env.TEST_CHANNELS }} + mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} pytest pytest-xdist python=${{ matrix.python }} ${{ env.TEST_CHANNELS }} env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.CHANNELS }}' + TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}' MAMBA_NO_LOW_SPEED_LIMIT: 1 - name: List installed packages @@ -363,29 +382,29 @@ jobs: python -c "import dpnp; print(dpnp.__version__)" - name: Run tests - if: env.RERUN_TESTS_ON_FAILURE != 'true' + if: env.rerun-tests-on-failure != 'true' run: | - if (${{ matrix.python }} -eq ${{ env.LATEST_PYTHON }}) { + if (${{ matrix.python }} -eq ${{ env.python-ver-test-all-dtypes }}) { set DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests + python -m pytest -ra --pyargs ${{ env.package-name }}.tests } else { - python -m pytest -n auto -ra --pyargs ${{ env.PACKAGE_NAME }}.tests + python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests } - name: Run tests - if: env.RERUN_TESTS_ON_FAILURE == 'true' + if: env.rerun-tests-on-failure == 'true' id: run_tests_win uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 with: - timeout_minutes: 35 - max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }} + timeout_minutes: ${{ env.rerun-tests-timeout }} + max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | - if ( ${{ matrix.python }} -eq ${{ env.LATEST_PYTHON }} ) { + if ( ${{ matrix.python }} -eq ${{ env.python-ver-test-all-dtypes }} ) { set DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests + python -m pytest -ra --pyargs ${{ env.package-name }}.tests } else { - python -m pytest -n auto -ra --pyargs ${{ env.PACKAGE_NAME }}.tests + python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests } upload: @@ -394,6 +413,7 @@ jobs: needs: [test_linux, test_windows] strategy: + fail-fast: false matrix: # python 3.13 is blocked due to MKL issue python: ['3.9', '3.10', '3.11', '3.12'] @@ -405,50 +425,65 @@ jobs: run: shell: ${{ matrix.os == 'windows-2019' && 'cmd /C CALL {0}' || 'bash -el {0}' }} - continue-on-error: true + env: + upload-conda-pkg-env: 'environments/upload_cleanup_conda_pkg.yml' + upload-env-name: 'upload' if: | (github.repository == 'IntelPython/dpnp') && (github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true) || github.event_name == 'push' && contains(github.ref, 'refs/tags/')) steps: + - name: Checkout DPNP repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: ${{ env.fetch-depth }} + - name: Download artifact uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python }} + name: ${{ env.package-name }} ${{ runner.os }} Python ${{ matrix.python }} - name: Download wheels artifact uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python }} + name: ${{ env.package-name }} ${{ runner.os }} Wheels Python ${{ matrix.python }} - name: Setup miniconda + id: setup_miniconda + continue-on-error: true uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ matrix.python }} - activate-environment: 'upload' + environment-file: ${{ env.upload-conda-pkg-env }} + activate-environment: ${{ env.upload-env-name }} - - name: Install anaconda-client - run: mamba install anaconda-client + - name: ReSetup miniconda + if: steps.setup_miniconda.outcome == 'failure' + uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 + with: + miniforge-version: latest + use-mamba: 'true' + conda-remove-defaults: 'true' + environment-file: ${{ env.upload-conda-pkg-env }} + activate-environment: ${{ env.upload-env-name }} - name: Package version shell: bash -el {0} run: | - echo "PACKAGE_VERSION=$(basename ${{ env.PACKAGE_NAME }}-*.conda | sed 's/^${{ env.PACKAGE_NAME }}-\([^-]*\).*/\1/')" >> $GITHUB_ENV + echo "PACKAGE_VERSION=$(basename ${{ env.package-name }}-*.conda | sed 's/^${{ env.package-name }}-\([^-]*\).*/\1/')" >> $GITHUB_ENV - name: Upload run: | - anaconda --token ${{ env.ANACONDA_TOKEN }} upload --user dppy --label dev ${{ env.PACKAGE_NAME }}-*.conda + anaconda --token ${{ env.ANACONDA_TOKEN }} upload --user dppy --label dev ${{ env.package-name }}-*.conda env: ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - name: Upload wheels run: | - anaconda --token ${{ env.ANACONDA_TOKEN }} upload --user dppy --label dev ${{ env.PACKAGE_NAME }}-*.whl --version ${{ env.PACKAGE_VERSION }} + anaconda --token ${{ env.ANACONDA_TOKEN }} upload --user dppy --label dev ${{ env.package-name }}-*.whl --version ${{ env.PACKAGE_VERSION }} env: ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} @@ -463,7 +498,6 @@ jobs: strategy: matrix: - python: ['3.12'] os: [ubuntu-22.04] runs-on: ${{ matrix.os }} @@ -477,17 +511,26 @@ jobs: env: array-api-tests-path: '${{ github.workspace }}/array-api-tests/' json-report-file: '${{ github.workspace }}/.report.json' - dpnp-repo-path: '${{ github.workspace }}/dpnp/' - array-api-skips-file: '${{ github.workspace }}/dpnp/.github/workflows/array-api-skips.txt' + dpnp-repo-path: '${{ github.workspace }}/source/' + array-api-skips-file: '${{ github.workspace }}/source/.github/workflows/array-api-skips.txt' + create-conda-channel-env: 'source/environments/create_conda_channel.yml' + python-ver: '3.12' # it has to be aligned with python in create_conda_channel.yml + conda-env-name: 'array-api-conformity' channel-path: '${{ github.workspace }}/channel/' pkg-path-in-channel: '${{ github.workspace }}/channel/linux-64/' ver-json-path: '${{ github.workspace }}/version.json' steps: + - name: Checkout DPNP repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: ${{ env.fetch-depth }} + path: ${{ env.dpnp-repo-path }} + - name: Download artifact uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python }} + name: ${{ env.package-name }} ${{ runner.os }} Python ${{ env.python-ver }} path: ${{ env.pkg-path-in-channel }} - name: Setup miniconda @@ -497,10 +540,9 @@ jobs: with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ matrix.python }} - activate-environment: 'array-api-conformity' + environment-file: ${{ env.create-conda-channel-env }} + activate-environment: ${{ env.conda-env-name }} - name: ReSetup miniconda if: steps.setup_miniconda.outcome == 'failure' @@ -508,21 +550,9 @@ jobs: with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ matrix.python }} - activate-environment: 'array-api-conformity' - - - name: Install conda-index - id: install_conda_index - continue-on-error: true - run: | - mamba install conda-index=${{ env.CONDA_INDEX_VERSION }} - - - name: ReInstall conda-index - if: steps.install_conda_index.outcome == 'failure' - run: | - mamba install conda-index=${{ env.CONDA_INDEX_VERSION }} + environment-file: ${{ env.create-conda-channel-env }} + activate-environment: ${{ env.conda-env-name }} - name: Create conda channel run: | @@ -530,12 +560,12 @@ jobs: - name: Test conda channel run: | - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.channel-path }} --override-channels --info --json > ${{ env.ver-json-path }} + conda search ${{ env.package-name }} -c ${{ env.channel-path }} --override-channels --info --json > ${{ env.ver-json-path }} cat ${{ env.ver-json-path }} - name: Get package version run: | - export PACKAGE_VERSION=$(python -c "${{ env.VER_SCRIPT1 }} ${{ env.VER_SCRIPT2 }}") + export PACKAGE_VERSION=$(python -c "${{ env.ver-script-part1 }} ${{ env.ver-script-part2 }}") echo PACKAGE_VERSION=${PACKAGE_VERSION} echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> $GITHUB_ENV @@ -544,16 +574,16 @@ jobs: id: install_dpnp continue-on-error: true run: | - mamba install ${{ env.PACKAGE_NAME }}=${{ env.PACKAGE_VERSION }} pytest python=${{ matrix.python }} ${{ env.TEST_CHANNELS }} + mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} pytest python=${{ env.python-ver }} ${{ env.TEST_CHANNELS }} env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.CHANNELS }}' + TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}' - name: ReInstall dpnp if: steps.install_dpnp.outcome == 'failure' run: | - mamba install ${{ env.PACKAGE_NAME }}=${{ env.PACKAGE_VERSION }} pytest python=${{ matrix.python }} ${{ env.TEST_CHANNELS }} + mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} pytest python=${{ env.python-ver }} ${{ env.TEST_CHANNELS }} env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.CHANNELS }}' + TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}' - name: List installed packages run: mamba list @@ -568,7 +598,7 @@ jobs: with: repository: 'data-apis/array-api-tests' path: ${{ env.array-api-tests-path }} - fetch-depth: 0 + fetch-depth: ${{ env.fetch-depth }} submodules: 'recursive' - name: Install array API test dependencies @@ -588,12 +618,6 @@ jobs: python -c "import dpctl; dpctl.lsplatform()" python -c "import dpnp; print(dpnp.__version__)" - # need to fetch array-api-skips.txt - - name: Checkout DPNP repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - path: ${{ env.dpnp-repo-path }} - - name: Run array API conformance tests run: | python -m pytest --json-report --json-report-file=${{ env.json-report-file }} --disable-deadline --skips-file ${{ env.array-api-skips-file }} array_api_tests || true @@ -602,7 +626,7 @@ jobs: SYCL_CACHE_PERSISTENT: 1 working-directory: ${{ env.array-api-tests-path }} - - name: Set Github environment variables + - name: Set GitHub environment variables run: | FILE=${{ env.json-report-file }} if test -f "$FILE"; then @@ -643,28 +667,45 @@ jobs: run: shell: bash -el {0} + env: + cleanup-conda-pkg-env: 'environments/upload_cleanup_conda_pkg.yml' + cleanup-env-name: 'cleanup' + steps: - - uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 + - name: Checkout DPNP repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: Setup miniconda + id: setup_miniconda + continue-on-error: true + uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - run-post: 'false' - python-version: '3.13' - activate-environment: 'cleanup' + environment-file: ${{ env.upload-conda-pkg-env }} + activate-environment: ${{ env.upload-env-name }} - - name: Install anaconda-client - run: mamba install anaconda-client + - name: ReSetup miniconda + if: steps.setup_miniconda.outcome == 'failure' + uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 + with: + miniforge-version: latest + use-mamba: 'true' + conda-remove-defaults: 'true' + environment-file: ${{ env.cleanup-conda-pkg-env }} + activate-environment: ${{ env.cleanup-env-name }} - name: Checkout repo uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: IntelPython/devops-tools - fetch-depth: 0 + fetch-depth: ${{ env.fetch-depth }} - name: Cleanup old packages run: | python scripts/cleanup-old-packages.py \ --verbose --force --token ${{ secrets.ANACONDA_TOKEN }} \ - --package dppy/${{ env.PACKAGE_NAME }} --label dev + --package dppy/${{ env.package-name }} --label dev diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index 4714c8f0f6dd..83a1d9befaaa 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -12,10 +12,11 @@ on: permissions: read-all env: - PACKAGE_NAME: dpnp - CHANNELS: '-c dppy/label/dev -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels' - TEST_ENV_NAME: test - RUN_TESTS_MAX_ATTEMPTS: 2 + package-name: dpnp + channels-list: '-c dppy/label/dev -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels' + test-env-name: test + rerun-tests-max-attempts: 2 + rerun-tests-timeout: 20 jobs: test: @@ -69,7 +70,7 @@ jobs: channels: conda-forge conda-remove-defaults: 'true' python-version: ${{ matrix.python }} - activate-environment: ${{ env.TEST_ENV_NAME }} + activate-environment: ${{ env.test-env-name }} - name: ReSetup miniconda if: steps.setup_miniconda.outcome == 'failure' @@ -80,18 +81,18 @@ jobs: channels: conda-forge conda-remove-defaults: 'true' python-version: ${{ matrix.python }} - activate-environment: ${{ env.TEST_ENV_NAME }} + activate-environment: ${{ env.test-env-name }} - name: Install dpnp id: install_dpnp continue-on-error: true run: | - mamba install ${{ env.PACKAGE_NAME }}=${{ steps.find_latest_tag.outputs.tag }} pytest ${{ env.CHANNELS }} + mamba install ${{ env.package-name }}=${{ steps.find_latest_tag.outputs.tag }} pytest ${{ env.channels-list }} - name: ReInstall dpnp if: steps.install_dpnp.outcome == 'failure' run: | - mamba install ${{ env.PACKAGE_NAME }}=${{ steps.find_latest_tag.outputs.tag }} pytest ${{ env.CHANNELS }} + mamba install ${{ env.package-name }}=${{ steps.find_latest_tag.outputs.tag }} pytest ${{ env.channels-list }} - name: List installed packages run: mamba list @@ -119,7 +120,7 @@ jobs: id: run_tests continue-on-error: true run: | - python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests + python -m pytest -ra --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 @@ -128,15 +129,15 @@ jobs: id: run_tests_linux uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 with: - timeout_minutes: 15 - max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }} + timeout_minutes: ${{ env.rerun-tests-timeout }} + max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | . $CONDA/etc/profile.d/conda.sh . $CONDA/etc/profile.d/mamba.sh - mamba activate ${{ env.TEST_ENV_NAME }} + mamba activate ${{ env.test-env-name }} - python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests + python -m pytest -ra --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 @@ -145,10 +146,10 @@ jobs: id: run_tests_win uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 with: - timeout_minutes: 17 - max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }} + timeout_minutes: ${{ env.rerun-tests-timeout }} + max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | - python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests + python -m pytest -ra --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 diff --git a/.github/workflows/generate_coverage.yaml b/.github/workflows/generate_coverage.yaml index 7061ea63436f..9a27e505e81d 100644 --- a/.github/workflows/generate_coverage.yaml +++ b/.github/workflows/generate_coverage.yaml @@ -20,11 +20,12 @@ jobs: shell: bash -el {0} env: - python-ver: '3.12' - CHANNELS: '-c dppy/label/dev -c intel -c conda-forge --override-channels' - NO_INTEL_CHANNELS: '-c dppy/label/dev -c conda-forge --override-channels' - # Install the latest oneAPI compiler to work around an issue - INSTALL_ONE_API: 'yes' + environment-file: 'environments/environment.yml' + build-with-oneapi-env: 'environments/build_with_oneapi.yml' + coverage-env: 'environments/coverage.yml' + oneapi-pkgs-env: '' + # Enable env when it's required to use only conda packages without OneAPI installation + # oneapi-pkgs-env: '${{ github.workspace }}/environments/oneapi_pkgs.yml' steps: - name: Cancel Previous Runs @@ -32,13 +33,8 @@ jobs: with: access_token: ${{ github.token }} - - name: Checkout repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Add Intel repository - if: env.INSTALL_ONE_API == 'yes' + if: env.oneapi-pkgs-env == '' run: | wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB cat GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null @@ -47,7 +43,7 @@ jobs: sudo apt update - name: Install latest Intel OneAPI - if: env.INSTALL_ONE_API == 'yes' + if: env.oneapi-pkgs-env == '' run: | sudo apt install hwloc \ intel-oneapi-mkl \ @@ -60,6 +56,22 @@ jobs: - name: Install Lcov run: | sudo apt-get install lcov + sudo gem install coveralls-lcov + + - name: Checkout repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - name: Install conda-merge tool + uses: BSFishy/pip-action@8f2d471d809dc20b6ada98c91910b6ae6243f318 # v1 + with: + packages: conda-merge + + - name: Merge conda env files + run: | + conda-merge ${{ env.build-with-oneapi-env }} ${{ env.coverage-env }} ${{ env.oneapi-pkgs-env }} > ${{ env.environment-file }} + cat ${{ env.environment-file }} - name: Setup miniconda id: setup_miniconda @@ -68,9 +80,8 @@ jobs: with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ env.python-ver }} + environment-file: ${{ env.environment-file }} activate-environment: 'coverage' - name: ReSetup miniconda @@ -79,27 +90,10 @@ jobs: with: miniforge-version: latest use-mamba: 'true' - channels: conda-forge conda-remove-defaults: 'true' - python-version: ${{ env.python-ver }} + environment-file: ${{ env.environment-file }} activate-environment: 'coverage' - # Sometimes `mamba install ...` fails due to slow download speed rate, so disable the check in mamba - - name: Disable speed limit check in mamba - run: echo "MAMBA_NO_LOW_SPEED_LIMIT=1" >> $GITHUB_ENV - - - name: Install dpnp dependencies - if: env.INSTALL_ONE_API == 'yes' - run: | - mamba install cython llvm cmake">=3.21" scikit-build ninja pytest pytest-cov coverage[toml] \ - dpctl">=0.18.0dev0" ${{ env.NO_INTEL_CHANNELS }} - - - name: Install dpnp dependencies - if: env.INSTALL_ONE_API != 'yes' - run: | - mamba install cython llvm cmake">=3.21" scikit-build ninja pytest pytest-cov coverage[toml] \ - dpctl">=0.18.0dev0" dpcpp_linux-64 mkl-devel-dpcpp tbb-devel onedpl-devel ${{ env.CHANNELS }} - - name: Conda info run: | mamba info @@ -122,12 +116,7 @@ jobs: - name: Total number of coverage attempts run: | - echo "Total number of coverage attempts made: ${{ steps.build_coverage.outputs.total_attempts }}" - - - name: Install coverall dependencies - run: | - sudo gem install coveralls-lcov - pip install coveralls==4.0.1 + echo "Total number of coverage attempts: ${{ steps.build_coverage.outputs.total_attempts }}" - name: Upload coverage data to coveralls.io run: | @@ -144,9 +133,13 @@ jobs: coveralls: name: Indicate completion to coveralls.io + needs: generate-coverage + runs-on: ubuntu-latest + container: python:3-slim + steps: - name: Finished run: | diff --git a/environments/base_build_docs.txt b/environments/base_build_docs.txt new file mode 100644 index 000000000000..edab09ee45ab --- /dev/null +++ b/environments/base_build_docs.txt @@ -0,0 +1,3 @@ +pyenchant==3.2.2 +sphinxcontrib-googleanalytics==0.4 +sphinxcontrib-spelling==8.0.1 diff --git a/environments/build_conda_pkg.yml b/environments/build_conda_pkg.yml new file mode 100644 index 000000000000..3dad51fb48e3 --- /dev/null +++ b/environments/build_conda_pkg.yml @@ -0,0 +1,6 @@ +name: Build DPNP conda package +channels: + - conda-forge +dependencies: + - python=3.12 # conda-build does not support python 3.13 + - conda-build=25.1.1 diff --git a/environments/build_with_oneapi.yml b/environments/build_with_oneapi.yml new file mode 100644 index 000000000000..715023ee48db --- /dev/null +++ b/environments/build_with_oneapi.yml @@ -0,0 +1,12 @@ +name: Packages to build DPNP with OneAPI env activated +channels: + - dppy/label/dev + - conda-forge +dependencies: + - cmake + - cython + - dpctl>=0.19.0dev0 + - ninja + - numpy + - pytest + - scikit-build diff --git a/environments/building_docs.yml b/environments/building_docs.yml new file mode 100644 index 000000000000..2d452ddf3e48 --- /dev/null +++ b/environments/building_docs.yml @@ -0,0 +1,10 @@ +name: Building docs specific packages +channels: + - conda-forge +dependencies: + - python=3.12 + - cupy + - sphinx + - sphinx_rtd_theme + - pip: + - -r base_build_docs.txt diff --git a/environments/coverage.txt b/environments/coverage.txt new file mode 100644 index 000000000000..12f4cc07e1f2 --- /dev/null +++ b/environments/coverage.txt @@ -0,0 +1 @@ +coveralls==4.0.1 diff --git a/environments/coverage.yml b/environments/coverage.yml new file mode 100644 index 000000000000..0fcc8d82129d --- /dev/null +++ b/environments/coverage.yml @@ -0,0 +1,10 @@ +name: Coverage specific packages +channels: + - conda-forge +dependencies: + - python=3.12 + - coverage[toml] + - llvm + - pytest-cov + - pip: + - -r coverage.txt diff --git a/environments/create_conda_channel.yml b/environments/create_conda_channel.yml new file mode 100644 index 000000000000..3735d4a9a558 --- /dev/null +++ b/environments/create_conda_channel.yml @@ -0,0 +1,6 @@ +name: Create conda channel with DPNP package +channels: + - conda-forge +dependencies: + - python=3.12 # conda does not support python 3.13 + - conda-index=0.5.0 diff --git a/environments/oneapi_pkgs.yml b/environments/oneapi_pkgs.yml new file mode 100644 index 000000000000..8882c82fdc71 --- /dev/null +++ b/environments/oneapi_pkgs.yml @@ -0,0 +1,8 @@ +name: OneAPI packages to build DPNP without OneAPI env activated +channels: + - https://software.repos.intel.com/python/conda/ +dependencies: + - dpcpp_linux-64 + - mkl-devel-dpcpp + - onedpl-devel + - tbb-devel diff --git a/environments/upload_cleanup_conda_pkg.yml b/environments/upload_cleanup_conda_pkg.yml new file mode 100644 index 000000000000..700ad2b5787b --- /dev/null +++ b/environments/upload_cleanup_conda_pkg.yml @@ -0,0 +1,6 @@ +name: Upload or clean up a conda package +channels: + - conda-forge +dependencies: + - python=3.13 + - anaconda-client=1.12.3