Merge pull request #26 from IntelPython/merge/gold_into_main

Merge gold into main

Author: ZzEeKkAa

.github/actions/setup-conda-build/action.yml

@@ -16,108 +16,51 @@ env:
   MODULE_NAME: dpcpp_llvm_spirv
 
 jobs:
-  build_linux:
-    runs-on: ubuntu-latest
-
+  build:
     strategy:
       matrix:
         python: ["3.9", "3.10", "3.11"]
-
+        os: ["ubuntu-latest", "windows-latest"]
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: ${{ matrix.os == 'windows-latest' && 'cmd /C CALL {0}' || 'bash -l {0}' }}
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Setup conda-build
-        uses: ./.github/actions/setup-conda-build
-
-      - name: Build conda package
-        env:
-          CHANNELS: -c intel -c main --override-channels
-        run: conda build --python ${{ matrix.python }} ${{ env.CHANNELS }} conda-recipe
-
-      - name: Upload artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python }}
-          path: /usr/share/miniconda/conda-bld/linux-64/${{ env.PACKAGE_NAME }}-*.tar.bz2
-
-  build_windows:
-    runs-on: windows-latest
-
-    strategy:
-      matrix:
-        python: ["3.9", "3.10", "3.11"]
-
-    env:
-      CHANNELS: -c intel -c main --override-channels
-      conda-bld: C:\Miniconda\conda-bld\win-64\
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
           auto-activate-base: true
           activate-environment: ""
 
-      - name: Cache conda packages
-        uses: actions/cache@v3
-        env:
-          CACHE_NUMBER: 0  # Increase to reset cache
-        with:
-          path: /home/runner/conda_pkgs_dir
-          key:
-            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-${{hashFiles('**/meta.yaml') }}
-          restore-keys: |
-            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-
-            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-
       - name: Install conda-build
         run: conda install conda-build
+
       - name: Build conda package
+        env:
+          CHANNELS: -c intel -c conda-forge --override-channels
         run: conda build --python ${{ matrix.python }} ${{ env.CHANNELS }} conda-recipe
+
       - name: Upload artifact
         uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python }} ${{ matrix.artifact_name }}
-          path: ${{ env.conda-bld }}${{ env.PACKAGE_NAME }}-*.tar.bz2
-
-  upload_linux:
-    needs: build_linux
-    if: ${{github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/heads/release') == true)}}
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python: ["3.9", "3.10", "3.11"]
-    steps:
-      - name: Download artifact
-        uses: actions/download-artifact@v3
         with:
           name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python }}
+          path: /usr/share/miniconda/conda-bld/linux-64/${{ env.PACKAGE_NAME }}-*.tar.bz2
 
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          auto-activate-base: true
-          activate-environment: ""
-      - name: Install anaconda-client
-        run: conda install anaconda-client
-
-      - name: Upload
-        env:
-          ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
-        run: |
-          source /usr/share/miniconda/etc/profile.d/conda.sh
-          conda activate
-          anaconda --token $ANACONDA_TOKEN upload --user dppy --label dev ${PACKAGE_NAME}-*.tar.bz2
-
-  upload_windows:
-    needs: build_windows
+  upload:
+    needs: build
     if: ${{github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/heads/release') == true)}}
-    runs-on: windows-latest
     strategy:
       matrix:
         python: ["3.9", "3.10", "3.11"]
+        os: ["ubuntu-latest", "windows-latest"]
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: ${{ matrix.os == 'windows-latest' && 'cmd /C CALL {0}' || 'bash -l {0}' }}
     steps:
       - name: Download artifact
         uses: actions/download-artifact@v3
@@ -132,7 +75,4 @@ jobs:
         run: conda install anaconda-client
 
       - name: Upload
-        env:
-          ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
-        run: |
-          anaconda --token ${{ env.ANACONDA_TOKEN }} upload --user dppy --label dev ${{ env.PACKAGE_NAME }}-*.tar.bz2
+        run: anaconda --token ${{ secrets.ANACONDA_TOKEN }} upload --user dppy --label dev ${{ env.PACKAGE_NAME }}-*.tar.bz2

.github/workflows/conda-package.yml

@@ -1,6 +1,6 @@
-{% set version = "2023.0.0" %}
-{% set intel_build_number = "25370" %}   # [linux]
-{% set intel_build_number = "25922" %}   # [win]
+{% set version = "2024.0.0" %}
+{% set intel_build_number = "49819" %}   # [linux]
+{% set intel_build_number = "49840" %}   # [win]
 {% set target_platform = "linux-64" %}  # [linux64]
 {% set target_platform = "win-64" %}    # [win64]
 
@@ -15,10 +15,6 @@ package:
 source:
   - path: ../pkg
     folder: package
-  - url: https://anaconda.org/intel/dpcpp_impl_{{ target_platform }}/{{ version }}/download/{{ target_platform }}/dpcpp_impl_{{ target_platform }}-{{ version }}-intel_{{ intel_build_number }}.tar.bz2  # [linux64 or win64]
-    md5: 0acd44eea158cf828f7f1058f6fc85da  # [win64]
-    md5: 5217aa49f13b313a0e357aef6a182264  # [linux64]
-    folder: compiler
   - path: ..
 
 build:
@@ -31,10 +27,19 @@ outputs:
   - name: dpcpp-llvm-spirv
     script: repack.sh   # [linux]
     script: repack.bat  # [win]
+    build:
+      include_recipe: False
+      script_env:
+        - WHEELS_OUTPUT_FOLDER
     requirements:
+      build:
+        - dpcpp_linux-64 =={{ version }}=intel_{{ intel_build_number }}  #[linux]
+        - dpcpp_win-64 =={{ version }}=intel_{{ intel_build_number }}  #[win]
       host:
         - python
-        - setuptools  >=63
+        - setuptools
+        - patchelf # [linux]
+        - wheel
       run:
         - python
     test:
@@ -59,5 +64,5 @@ outputs:
         disclaimers or license terms for third party or open source software
         included in or with the software.</strong>
         <br/><br/>
-        EULA: <a href="https://software.intel.com/content/dam/develop/external/us/en/documents/pdf/intel-developer-tools-eula-09-03-19.pdf" target="_blank">LicenseRef-Proprietary-Intel-End-User-License-Agreement-for-Developer-Tools</a>
+        EULA: <a href="https://www.intel.com/content/dam/develop/external/us/en/documents/pdf/intel-developer-tools-eula-09-03-19.pdf" target="_blank">LicenseRef-Proprietary-Intel-End-User-License-Agreement-for-Developer-Tools</a>
         <br/><br/>

conda-recipe/meta.yaml

@@ -1,16 +1,31 @@
 
-set /P DPCPP_LLVM_SPIRV_VERSION < %PYTHON% get_icpx_version.py
+set "DPCPP_LLVM_SPIRV_VERSION=%PKG_VERSION%"
 echo "Inferred DPCPP_LLVM_SPIRV_VERSION=%DPCPP_LLVM_SPIRV_VERSION%"
 
+set "BUILD_ARGS=--single-version-externally-managed --record=llvm_spirv_record.txt"
+
+if not exist %BUILD_PREFIX%\Library\bin\compiler\llvm-spirv.exe (exit 1)
+
 pushd %SRC_DIR%\package
-%PYTHON% setup.py install --single-version-externally-managed --record=llvm_spirv_record.txt
-type llvm_spirv_record.txt
-popd
+if not exist %SRC_DIR%\package\dpcpp_llvm_spirv\bin mkdir %SRC_DIR%\package\dpcpp_llvm_spirv\bin
+copy %BUILD_PREFIX%\Library\bin\compiler\llvm-spirv.exe %SRC_DIR%\package\dpcpp_llvm_spirv\bin\
+
+for /f %%a in (
+  "%BUILD_PREFIX%\Library\bin\onnxruntime.*"
+) do copy %%~fa %SRC_DIR%\package\dpcpp_llvm_spirv\bin\
+
+rem Workaround to remove spaces from the env value
+set WHEELS_OUTPUT_FOLDER=%WHEELS_OUTPUT_FOLDER: =%
+if NOT "%WHEELS_OUTPUT_FOLDER%"=="" (
+
+  %PYTHON% setup.py install %BUILD_ARGS% bdist_wheel -p win_amd64 --python-tag py%PY_VER:.=%
+  if errorlevel 1 exit 1
+  copy dist\dpcpp_llvm_spirv*.whl %WHEELS_OUTPUT_FOLDER%
+  if errorlevel 1 exit 1
+) ELSE (
+  %PYTHON% setup.py install %BUILD_ARGS%
+  if errorlevel 1 exit 1
+  type llvm_spirv_record.txt
+)
 
-pushd %SRC_DIR%\compiler
-%PYTHON% -c "import dpcpp_llvm_spirv as p; print(p.get_llvm_spirv_path())" > Output
-set /p DIRSTR= < Output
-if not exist Library\bin-llvm\llvm-spirv.exe (exit 1)
-copy Library\bin-llvm\llvm-spirv.exe %DIRSTR%
-del Output
 popd

conda-recipe/repack.bat

@@ -1,22 +1,36 @@
 #!/bin/bash
+
+set -ex
+
 echo -e "Start building dpcpp-llvm-spirv package \n"
 src="${SRC_DIR}"
 
 echo "Python: ${PYTHON}"
 [ -z "${PYTHON}" ] && exit 1
 
-export DPCPP_LLVM_SPIRV_VERSION=$(${PYTHON} get_icpx_version.py)
+export DPCPP_LLVM_SPIRV_VERSION="${PKG_VERSION}"
 echo -e "Inferred DPCPP_LLVM_SPIRV_VERSION=${DPCPP_LLVM_SPIRV_VERSION}"
 
+BUILD_ARGS="--single-version-externally-managed --record=llvm_spirv_record.txt"
+WHEELS_BUILD_ARGS="-p manylinux2014_x86_64 --python-tag py${PY_VER//./}"
+
 pushd $src/package
-${PYTHON} setup.py install --single-version-externally-managed --record=llvm_spirv_record.txt
-cat llvm_spirv_record.txt
-popd
+echo -e "Start vendoring of llvm-spirv executable \n"
+mkdir -p $src/package/dpcpp_llvm_spirv/bin
+mkdir -p $src/package/dpcpp_llvm_spirv/lib
+cp ${BUILD_PREFIX}/bin/compiler/llvm-spirv $src/package/dpcpp_llvm_spirv/bin/
+cp ${BUILD_PREFIX}/lib/libonnxruntime.* $src/package/dpcpp_llvm_spirv/lib/
+patchelf --force-rpath --set-rpath '$ORIGIN/../lib' $src/package/dpcpp_llvm_spirv/bin/llvm-spirv
 
-echo -e "Done building the Python package. Start vendoring of llvm-spirv executable \n"
+if [ -n "${WHEELS_OUTPUT_FOLDER}" ]; then
+  # Build wheel package
+  $PYTHON setup.py install ${BUILD_ARGS} bdist_wheel ${WHEELS_BUILD_ARGS}
+  cp dist/dpcpp_llvm_spirv*.whl ${WHEELS_OUTPUT_FOLDER}
+else
+  pushd $src/package
+  ${PYTHON} setup.py install ${BUILD_ARGS}
+  cat llvm_spirv_record.txt
+fi
+echo "done. \n"
 
-pushd ${src}/compiler
-cp bin-llvm/llvm-spirv $(${PYTHON} -c "import dpcpp_llvm_spirv as p; print(p.get_llvm_spirv_path())")
-echo "copy llvm-spirv to: $(${PYTHON} -c "import dpcpp_llvm_spirv as p; print(p.get_llvm_spirv_path())")"
 popd
-echo "done. \n"

conda-recipe/repack.sh

@@ -242,7 +242,7 @@ to items referenced therein, at any time without notice, but is not obligated to
 support, update or provide training for the Materials under the terms of this
 Agreement. Intel offers free community and paid priority support options. More
 information on these support options can be found at:
-https://software.intel.com/content/www/us/en/develop/support/priority-support.html.
+https://www.intel.com/content/www/us/en/developer/get-help/priority-support.html.
 
 7.	LIMITATION OF LIABILITY.
 

license.txt

@@ -11,7 +11,7 @@ def get_llvm_spirv_path():
     vendored in this package.
     """
 
-    result = os.path.dirname(__file__)
+    result = os.path.join(os.path.dirname(__file__), "bin")
 
     if platform.system() is "Windows":
         result += "\llvm-spirv.exe"

pkg/dpcpp_llvm_spirv/_helper.py

@@ -6,16 +6,16 @@
 import os.path
 
 from setuptools import setup
+from setuptools.command.install import install
 
 pkg_version = os.getenv("DPCPP_LLVM_SPIRV_VERSION", "0.0.0+dev")
 
 with open(os.path.join("dpcpp_llvm_spirv", "_version.py"), "w") as fh:
     fh.write(f"__version__ = '{pkg_version}'")
     fh.write("\n")
 
-
 setup(
-    name="dpcpp_llvm_spirv",
+    name="dpcpp-llvm-spirv",
     packages=[
         "dpcpp_llvm_spirv",
     ],
@@ -32,4 +32,11 @@
         "Programming Language :: Python :: 3.11",
     ],
     license="Intel End User License Agreement for Developer Tools",
+    package_data={
+        "dpcpp_llvm_spirv": [
+            "bin/llvm-spirv*",
+            "lib/libonnxruntime.*",  # linux
+            "bin/onnxruntime.*",  # windows
+        ]
+    },
 )