Merge branch 'main' into gold/2021

Author: mingjie-intel

.github/workflows/black.yml

@@ -14,7 +14,7 @@ jobs:
   # This workflow contains a single job called "black"
   black:
     # The type of runner that the job will run on
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -27,3 +27,4 @@ jobs:
       - uses: psf/black@stable
         with:
           args: ". --check"
+          version: "23.1.0"

.github/workflows/conda-package.yml

@@ -16,11 +16,11 @@ env:
 
 jobs:
   build_linux:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     strategy:
       matrix:
-        python: ["3.8", "3.9"]
+        python: ["3.8", "3.9", "3.10"]
 
     steps:
       - uses: actions/checkout@v3
@@ -46,18 +46,12 @@ jobs:
 
     strategy:
       matrix:
-        python: ["3.8", "3.9"]
-        integration_channels: [""]
+        python: ["3.8", "3.9", "3.10"]
         experimental: [false]
         artifact_name: [""]
-        include:
-          - python: "3.8"
-            integration_channels: -c dppy/label/dev
-            artifact_name: -c dppy_label_dev
-            experimental: true
     continue-on-error: ${{ matrix.experimental }}
     env:
-      CHANNELS: ${{ matrix.integration_channels }} -c intel -c defaults -c numba -c numba/label/dev -c dppy/label/dev --override-channels
+      CHANNELS: -c dppy/label/dev -c intel -c defaults -c numba -c numba/label/dev --override-channels
       conda-bld: C:\Miniconda\conda-bld\win-64\
 
     steps:
@@ -92,11 +86,11 @@ jobs:
 
   test_linux:
     needs: build_linux
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     strategy:
       matrix:
-        python: ["3.8", "3.9"]
+        python: ["3.8", "3.9", "3.10"]
         numba: ["0.56"]
         dpnp: ["0.11"]
 
@@ -143,28 +137,34 @@ jobs:
           # echo "libintelocl.so" | tee /etc/OpenCL/vendors/intel-cpu.icd
           export OCL_ICD_FILENAMES=libintelocl.so
           python -m pytest -q -ra --disable-warnings --pyargs $MODULE_NAME -vv
+      - name: Run examples
+        run: |
+          ls
+          pwd
+          cd ./numba-dpex/numba_dpex/examples/
+          source $CONDA/etc/profile.d/conda.sh
+          conda activate numba_dpex_env
+          # echo "libintelocl.so" | tee /etc/OpenCL/vendors/intel-cpu.icd
+          export OCL_ICD_FILENAMES=libintelocl.so
+          for script in $(find . \( -not -name "_*" -not "vector_sum2D.py" -not -name "vectorize.py" -not -name "scan.py" -and -name "*.py" \))
+          do
+            echo "Executing ${script}"
+            python ${script} || exit 1
+          done
 
   test_windows:
     needs: build_windows
     runs-on: windows-latest
 
     strategy:
       matrix:
-        python: ["3.8", "3.9"]
+        python: ["3.8", "3.9", "3.10"]
         integration_channels: [""]
-        experimental: [true]  # packages are not available on -c intel yet
-        artifact_name: [""]
-        dependencies: [""]
-        include:
-          - python: "3.8"
-            integration_channels: -c dppy/label/dev
-            artifact_name: -c dppy_label_dev
-            experimental: true  # current stable
-            dependencies: ""
+        experimental: [false]  # packages are not available on -c intel yet
     continue-on-error: ${{ matrix.experimental }}
     env:
       # conda-forge: llvm-spirv 11 not on intel channel yet
-      CHANNELS: ${{ matrix.integration_channels }} -c dppy/label/dev -c intel -c defaults -c numba -c numba/label/dev -c conda-forge --override-channels
+      CHANNELS: -c dppy/label/dev -c intel -c defaults -c numba -c numba/label/dev --override-channels
 
     steps:
       - name: Create dir for numba-dpex repo
@@ -175,15 +175,16 @@ jobs:
           path: dpex-repo
           fetch-depth: 0
       - name: Download artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
-          name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python }} ${{ matrix.artifact_name }}
+          name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python }}
       - uses: conda-incubator/setup-miniconda@v2
         with:
-          auto-activate-base: true
-          activate-environment: ""
-      - name: Install conda-build
-        run: conda install conda-build
+          use-only-tar-bz2: true
+          auto-update-conda: true
+          conda-build-version: '*'
+          miniconda-version: 'latest'
+          activate-environment: "numba_dpex_wintest"
       - name: Create conda channel
         run: |
           mkdir -p $env:GITHUB_WORKSPACE/channel/win-64
@@ -196,7 +197,7 @@ jobs:
       - name: Cache conda packages
         uses: actions/cache@v2
         env:
-          CACHE_NUMBER: 0  # Increase to reset cache
+          CACHE_NUMBER: 1  # Increase to reset cache
         with:
           path: /home/runner/conda_pkgs_dir
           key:
@@ -256,10 +257,10 @@ jobs:
   upload_linux:
     needs: test_linux
     if: ${{github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/heads/release') == true)}}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python: ["3.8", "3.9"]
+        python: ["3.8", "3.9", "3.10"]
     steps:
       - name: Download artifact
         uses: actions/download-artifact@v2
@@ -287,7 +288,7 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        python: ["3.8", "3.9"]
+        python: ["3.8", "3.9", "3.10"]
     steps:
       - name: Download artifact
         uses: actions/download-artifact@v2

.github/workflows/license.yml

@@ -21,4 +21,4 @@ jobs:
     - name: Check license
       run: |
         export PATH=${PATH}:`go env GOPATH`/bin
-        addlicense -check -l apache -c "Intel Corporation" numba_dpex/**/*.py numba_dpex/*.py setup.py
+        addlicense -check -l apache -c "Intel Corporation" -ignore numba_dpex/_version.py numba_dpex/**/*.py numba_dpex/*.py setup.py

.github/workflows/numba.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     env:
       ID: ${{ github.event.inputs.numba_pr }}

.github/workflows/pre-commit.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   pre-commit:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2

.gitignore

@@ -18,7 +18,7 @@ docs/_build/
 docs/gh-pages/
 dist/
 .idea/
-.vscode/settings.json
+.vscode/
 .mypy_cache/
 .ipynb_checkpoints/
 __pycache__/

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
 -   repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 23.1.0
     hooks:
     -   id: black
         exclude: "versioneer.py|numba_dpex/_version.py"
@@ -23,7 +23,7 @@ repos:
     -   id: blacken-docs
         additional_dependencies: [black==22.10]
 -   repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
     -   id: isort
         name: isort (python)

.vscode/extensions.json

@@ -19,13 +19,13 @@ requirements:
         - setuptools
         - cython
         - numba 0.56*
-        - dpctl >=0.13.0
+        - dpctl >=0.14*
         - dpnp >=0.11*
         - wheel
     run:
         - python
         - numba >=0.56*
-        - dpctl >=0.13.0
+        - dpctl >=0.14*
         - spirv-tools
         - llvm-spirv 11.*
         - dpnp >=0.11*

conda-recipe/meta.yaml

@@ -0,0 +1,24 @@
+.. _caching:
+
+Caching Mechanism in Numba-dpex
+================================
+
+Caching is done by saving the compiled kernel code, the ELF object of the executable code. By using the kernel code, cached kernels have minimal overhead because no compilation is needed.
+
+Unlike Numba, we do not perform file-based caching, instead we use an Least Recently Used (LRU) caching mechanism. However when a kernel needs to be evicted, we utilize numba's file-based caching mechanism described `here <https://numba.pydata.org/numba-doc/latest/developer/caching.html>`_.
+
+Algorithm
+==========
+
+The caching mechanism for Numba-dpex works as follows: The cache is an LRU cache backed by an ordered dictionary mapped onto a doubly linked list. The tail of the list contains the most recently used (MRU) kernel and the head of the list contains the least recently used (LRU) kernel. The list  has a fixed size. If a new kernel arrives to be cached and if the size is already on the maximum limit, the algorithm evicts the LRU kernel to make room for the MRU kernel. The evicted item will be serialized and pickled into a file using Numba's caching mechanism.
+
+Everytime whenever a kernel needs to be retrieved from the cache, the mechanism will look for the kernel in the cache and will be loaded if it's already present. However, if the program is seeking for a kernel that has been evicted, the algorithm will load it from the file and enqueue in the cache.
+
+Settings
+========
+
+Therefore, we employ similar environment variables as used in Numba, i.e. ``NUMBA_CACHE_DIR`` etc. However we add three more environment variables to control the caching mechanism.
+
+- In order to specify cache capacity, one can use ``NUMBA_DPEX_CACHE_SIZE``. By default, it's set to 10.
+- ``NUMBA_DPEX_ENABLE_CACHE`` can be used to enable/disable the caching mechanism. By default it's enabled, i.e. set to 1.
+- In order to enable the debugging messages related to caching, one can set ``NUMBA_DPEX_DEBUG_CACHE`` to 1. All environment variables are defined in :file:`numba_dpex/config.py`.