Merge pull request #1409 from IntelPython/feature/cache-conda-package-wf

Author: ZzEeKkAa

.flake8

@@ -19,6 +19,5 @@ exclude =
     .git,
     __pycache__,
     _version.py,
-    versioneer.py,
     lowerer.py,
     parfor.py

.github/actions/setup-miniconda/action.yml

@@ -0,0 +1,96 @@
+name: "Setup Miniconda"
+description: "Wrapper around conda-incubator/setup-miniconda to cache lock files."
+inputs:
+  python-version:
+    description: "Refer conda-incubator/setup-miniconda."
+    required: false
+    default: ""
+  activate-environment:
+    description: "Refer conda-incubator/setup-miniconda."
+    required: false
+    default: "test"
+  environment-file:
+    description: "Refer conda-incubator/setup-miniconda."
+    required: false
+    default: ""
+  channels:
+    description: "Refer conda-incubator/setup-miniconda."
+    required: false
+    default: ""
+  channel-priority:
+    description: "Refer conda-incubator/setup-miniconda, default changed to 'disabled'."
+    required: false
+    default: "disabled"
+  run-post:
+    description: "Refer conda-incubator/setup-miniconda, default changed to 'false'."
+    required: false
+    default: "false"
+  cache-number:
+    description: "Increase this value to reset cache if env file has not changed."
+    required: false
+    default: "-3"
+  dependencies:
+    description: "List of packages separated by ; to use in environment file."
+    required: false
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Get Date
+      id: get-date
+      shell: bash
+      run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
+
+    - name: Generate channels hash
+      id: channels-hash
+      shell: bash
+      run: echo "hash=$(echo '${{inputs.channels}}' | sha1sum | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
+
+    - name: Write dependencies
+      if: inputs.dependencies != ''
+      shell: bash
+      run: |
+        printf "dependencies:;${{inputs.dependencies}}" | sed 's/;/\n  - /g' > \
+        ${{inputs.environment-file}}
+
+    - name: Cache Conda env
+      uses: actions/cache@v4
+      with:
+        # TODO: use channels as part of the key
+        path: ${{inputs.environment-file}}.lock
+        key: setup-miniconda-${{ runner.os }}-${{ runner.arch }}-${{
+          inputs.python-version }}-${{
+          steps.get-date.outputs.today }}-${{
+          steps.channels-hash.hash }}-${{
+          hashFiles(inputs.environment-file) }}-${{
+          inputs.cache-number }}
+      id: cache
+
+    - name: Setup miniconda
+      if: steps.cache.outputs.cache-hit != 'true'
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        python-version: ${{ inputs.python-version }}
+        activate-environment: ${{inputs.activate-environment}}
+        channels: ${{ inputs.channels }}
+        channel-priority: ${{ inputs.channel-priority }}
+        environment-file: ${{ inputs.environment-file }}
+        run-post: ${{ inputs.run-post }}
+
+    - name: Save environment resolution
+      if: steps.cache.outputs.cache-hit != 'true'
+      shell: bash -l {0}
+      run: conda list --explicit > ${{inputs.activate-environment}}.lock
+
+    - name: Setup conda
+      if: steps.cache.outputs.cache-hit == 'true'
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        activate-environment: ${{inputs.activate-environment}}
+        channels: ${{ inputs.channels }}
+        environment-file: ${{inputs.activate-environment}}.lock
+        run-post: ${{ inputs.run-post }}
+
+    - name: Disable defaults
+      shell: bash -l {0}
+      run: conda config --remove channels defaults

.github/workflows/conda-package.yml

@@ -49,34 +49,27 @@ jobs:
       - name: Checkout ${{ env.PACKAGE_NAME }} repo
         uses: actions/checkout@v4
         with:
+          # We need tags to build proper version
           fetch-depth: 0
 
       - name: Setup miniconda
-        uses: conda-incubator/setup-miniconda@v3
+        uses: ./.github/actions/setup-miniconda
         with:
-          python-version: ${{ inputs.python-version }}
-          miniforge-variant: Mambaforge
-          miniforge-version: latest
+          python-version: ${{ matrix.python }}
           activate-environment: "build"
           channels: ${{ env.CHANNELS }}
           channel-priority: "disabled"
-          run-post: false
-
-      - name: Disable defautls
-        run: conda config --remove channels defaults
+          dependencies: "conda-build"
+          environment-file: environment/conda-package-build.yml
 
       - name: Store conda paths as envs
         run: |
           echo "CONDA_BLD=$CONDA_PREFIX/conda-bld/${{ runner.os == 'Linux' && 'linux' || 'win' }}-64/" | tr "\\" '/' >> $GITHUB_ENV
           echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE${{ runner.os == 'Linux' && '/' || '\\' }}" >> $GITHUB_ENV
           echo "WHEELS_NAME=$PACKAGE_NAME" | tr "-" "_" >> $GITHUB_ENV
 
-      # boa is an extention to conda so we can use mamba resolver in conda build
-      - name: Install conda-build
-        run: mamba install boa
-
       - name: Build conda package
-        run: conda mambabuild --no-test --python ${{ matrix.python }} conda-recipe
+        run: conda build --no-test --python ${{ matrix.python }} conda-recipe
 
       - name: Upload artifact
         uses: actions/upload-artifact@v4
@@ -107,28 +100,32 @@ jobs:
         python: ['3.9', '3.10', '3.11']
         os: [ubuntu-20.04, ubuntu-latest, windows-latest]
         experimental: [false]
-        use_mlir: [false]
         scope: ['tests', 'examples', 'gdb']
         exclude:
           - os: windows-latest
             scope: 'gdb'
 
-    continue-on-error: ${{ matrix.experimental || matrix.use_mlir }}
+    continue-on-error: ${{ matrix.experimental }}
 
     steps:
+      - name: Checkout setup config
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github/actions
+            environment
+            pyproject.toml
+          sparse-checkout-cone-mode: false
+
       - name: Setup miniconda
-        uses: conda-incubator/setup-miniconda@v3
+        uses: ./.github/actions/setup-miniconda
         with:
           python-version: ${{ matrix.python }}
-          miniforge-variant: Mambaforge
-          miniforge-version: latest
           activate-environment: "test"
           channels: ${{ env.CHANNELS }}
           channel-priority: "disabled"
-          run-post: false
-
-      - name: Disable defautls
-        run: conda config --remove channels defaults
+          dependencies: "conda-index;pytest-cov;conda-tree"
+          environment-file: environment/conda-package-test.yml
 
       - name: Store conda paths as envs
         shell: bash -l {0}
@@ -153,10 +150,6 @@ jobs:
           mkdir -p ${EXTRACTED_PKG_PATH}
           tar -xvf ${PKG_PATH_IN_CHANNEL}${PACKAGE_NAME}-*.tar.bz2 -C ${EXTRACTED_PKG_PATH} --force-local
 
-      - name: Install conda build
-        run: |
-          mamba install conda-build conda-index
-
       - name: Create conda channel
         # Note conda index (instead of python -m conda_index) may find legacy
         # conda-build index.
@@ -165,7 +158,7 @@ jobs:
 
       - name: Test conda channel
         run: |
-          mamba search ${{ env.PACKAGE_NAME }} -c ${{ env.CHANNEL_PATH }} --override-channels --info --json > ${{ env.VER_JSON_PATH }}
+          conda search ${{ env.PACKAGE_NAME }} -c ${{ env.CHANNEL_PATH }} --override-channels --info --json > ${{ env.VER_JSON_PATH }}
           cat ${{ env.VER_JSON_PATH }}
 
       - name: Collect dependencies
@@ -176,14 +169,8 @@ jobs:
 
       # We want to make sure that all dependecies install automatically.
       # intel::intel-opencl-rt is needed for set-intel-ocl-icd-registry.ps1
-      # TODO: remove pytest dependency directly, once resolved:
-      # https://github.com/pytest-dev/pytest/issues/12036
-      - name: Install builded package
-        run: mamba install ${{ env.PACKAGE_NAME }}=${{ env.PACKAGE_VERSION }} intel::intel-opencl-rt pytest-cov pytest=8.0.1 conda-tree -c ${{ env.CHANNEL_PATH }}
-
-      - name: Install numba-mlir
-        if: matrix.use_mlir
-        run: mamba install numba-mlir -c dppy/label/dev -c conda-forge -c intel
+      - name: Install built package
+        run: conda install ${{ env.PACKAGE_NAME }}=${{ env.PACKAGE_VERSION }} intel::intel-opencl-rt -c ${{ env.CHANNEL_PATH }}
 
       - name: Setup OpenCL CPU device
         if: runner.os == 'Windows'
@@ -192,13 +179,6 @@ jobs:
           $script_path="$env:CONDA_PREFIX\Scripts\set-intel-ocl-icd-registry.ps1"
           &$script_path
 
-      - name: Checkout setup config
-        uses: actions/checkout@v4
-        with:
-          sparse-checkout: |
-            pyproject.toml
-          sparse-checkout-cone-mode: false
-
       - name: List installed packages
         run: conda list
 
@@ -210,9 +190,7 @@ jobs:
         run: conda-tree depends -t numba-dpex
 
       - name: Smoke test
-        env:
-          NUMBA_DPEX_USE_MLIR: ${{ matrix.use_mlir && '1' || '0'  }}
-        run: python -c "import dpnp, dpctl, numba_dpex; dpctl.lsplatform()"
+        run: python -c "import dpnp, dpctl, numba_dpex; dpctl.lsplatform(verbosity=2)"
 
       - name: Smoke test - test import with no default device
         env:
@@ -221,8 +199,6 @@ jobs:
 
       - name: Run tests
         if: ${{ matrix.scope == 'tests' }}
-        env:
-          NUMBA_DPEX_USE_MLIR: ${{ matrix.use_mlir && '1' || '0'  }}
         run: |
           pytest -q -ra --disable-warnings --pyargs ${{ env.MODULE_NAME }} -vv -k "not test_1d_strided_dpnp_array_in_kernel[2]"
 
@@ -232,7 +208,6 @@ jobs:
         # and are possibly due to some driver/opencl compiler bug.
         if: ${{ matrix.scope == 'tests' }}
         env:
-          NUMBA_DPEX_USE_MLIR: ${{ matrix.use_mlir && '1' || '0'  }}
           # Disabling device driver optimization to prevent catching bugs
           # from driver compiler.
           ONEAPI_DEVICE_SELECTOR: "opencl:cpu"
@@ -267,10 +242,10 @@ jobs:
           export ONEAPI_ROOT=/tmp/gdb
           ./${{ env.GDB_INSTALLER }} -a -s --eula accept --install-dir $ONEAPI_ROOT
           source $ONEAPI_ROOT/debugger/latest/env/vars.sh
-          # We match only major and minor version because latest gdb is not
-          # often available.
-          gdb_version=$(echo "$GDB_INSTALLER" | grep -oP '\d+\.\d+' | head -n 1)
-          icpx_version=$(conda list dpcpp-cpp-rt | tail -n 1 | awk '{print $2}' | grep -oP '\d+\.\d+')
+          # We match only major version because latest gdb is not often
+          # available.
+          gdb_version=$(echo "$GDB_INSTALLER" | grep -oP '\d+' | head -n 1)
+          icpx_version=$(conda list dpcpp-cpp-rt | tail -n 1 | awk '{print $2}' | grep -oP '\d+' | head -n 1)
           if [ "$gdb_version" != "$icpx_version" ]; then
             echo "Error: GDB version ($gdb_version) does not match icpx version ($icpx_version)"
             exit 1
@@ -297,6 +272,13 @@ jobs:
     continue-on-error: false
 
     steps:
+      - name: Checkout setup config
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github/actions
+            environment
+
       - name: Download artifact
         uses: actions/download-artifact@v4
         with:
@@ -308,18 +290,13 @@ jobs:
           name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python }}
 
       - name: Setup miniconda
-        uses: conda-incubator/setup-miniconda@v3
+        uses: ./.github/actions/setup-miniconda
         with:
           python-version: ${{ matrix.python }}
-          miniforge-variant: Mambaforge
-          miniforge-version: latest
           activate-environment: "anaconda"
-          channels: ${{ env.CHANNELS }}
           channel-priority: "disabled"
-          run-post: false
-
-      - name: Install anaconda-client
-        run: mamba install anaconda-client
+          dependencies: "anaconda-client"
+          environment-file: environment/conda-package-anaconda.yml
 
       - name: Upload
         run: anaconda --token ${{ secrets.ANACONDA_TOKEN }} upload --user dppy --label dev ${{ env.PACKAGE_NAME }}-*.tar.bz2
@@ -342,21 +319,26 @@ jobs:
       run:
         shell: bash -el {0}
     steps:
-      - uses: conda-incubator/setup-miniconda@v3
+      - name: Checkout setup config
+        uses: actions/checkout@v4
         with:
-          run-post: false
-          channel-priority: "disabled"
-          channels: conda-forge
-          python-version: '3.11'
+          sparse-checkout: |
+            .github/actions
+            environment
 
-      - name: Install anaconda-client
-        run: conda install anaconda-client
+      - name: Setup miniconda
+        uses: ./.github/actions/setup-miniconda
+        with:
+          python-version: '3.11'
+          activate-environment: "anaconda"
+          channel-priority: "disabled"
+          dependencies: "anaconda-client"
+          environment-file: environment/conda-package-anaconda.yml
 
       - name: Checkout repo
         uses: actions/checkout@v4
         with:
           repository: IntelPython/devops-tools
-          fetch-depth: 0
 
       - name: Cleanup old packages
         run: |