Gaudi sw-stack update to ver. 1.21.0 (#16)

Signed-off-by: Pawel Szewc <[email protected]>

Author: pszewc-habana

.github/workflows/_gaudi_hpu_benchmark.yml

@@ -38,6 +38,10 @@ on:
           type: string
           default: '3.10'
           description: 'The python version to use'
+      rel_ver:
+        required: true
+        type: string
+        description: 'The release version of torch_hpu to use'
 
 # Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
 # declared as "shell: bash -el {0}" on steps that need to be properly activated.
@@ -100,7 +104,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: HabanaAI/gaudi-pytorch-bridge
-          ref: v1.20.0
+          ref: v${{ inputs.rel_ver }}
           path: torch_hpu
 
       - name: Install pip dependencies

.github/workflows/_gaudi_hpu_build_torch_hpu.yml

@@ -24,6 +24,10 @@ on:
         type: string
         default: '3.10'
         description: 'The python version to use'
+      rel_ver:
+        required: true
+        type: string
+        description: 'The release version of torch_hpu to use'
     outputs:
       torch-hpu-artifact:
         description: 'The distribution artifact name of torch_hpu'
@@ -48,6 +52,8 @@ jobs:
       env:
         HABANA_VISIBLE_DEVICES: all
         OMPI_MCA_btl_vader_single_copy_mechanism: none
+        REL_VERSION: ${{inputs.rel_ver}}
+        BASE_INC: "/usr/include/habanalabs"
       options: >-
         --ipc host
         --cap-add sys_nice
@@ -92,14 +98,14 @@ jobs:
         run: |
           mkdir torch_hpu && cd torch_hpu
           # apt update -y && apt install -y curl gnupg pciutils wget
-          # wget 'https://vault.habana.ai/artifactory/gaudi-installer/latest/habanalabs-installer.sh'
+          wget "https://vault.habana.ai/artifactory/gaudi-installer/$REL_VERSION/habanalabs-installer.sh"
           export HABANA_SOFTWARE_STACK="$(pwd)"
           export PYTORCH_MODULES_ROOT_PATH="$HABANA_SOFTWARE_STACK/gaudi-pytorch-bridge"
           git clone https://github.com/HabanaAI/gaudi-pytorch-bridge $PYTORCH_MODULES_ROOT_PATH
 
-          # IFS=- read -r VERSION BUILD <<EOF
-          # $(bash habanalabs-installer.sh -v)
-          # EOF
+          IFS=- read -r VERSION BUILD <<EOF
+          $(bash habanalabs-installer.sh -v)
+          EOF
           # "${PYTORCH_MODULES_ROOT_PATH}"/scripts/install_torch_fork.sh "$VERSION" "$BUILD"
           
           mkdir 3rd-parties
@@ -115,13 +121,10 @@ jobs:
           sed -i 's/namespace nlohmann/namespace nlohmannV340/; s/nlohmann::/nlohmannV340::/g' json/single_include/nlohmann/json.hpp
 
           popd
-          git clone --depth 1 --branch 1.20.0-543 https://github.com/HabanaAI/HCL.git
-          git clone --depth 1 --branch main https://github.com/HabanaAI/Intel_Gaudi3_Software.git
+          pip install -r "$PYTORCH_MODULES_ROOT_PATH"/requirements.txt
+          pip install habana-media-loader==$VERSION.$BUILD
 
-          ln -s /usr/include/habanalabs/ /usr/include/habanalabs/include
-          ln -s /usr/lib/habanalabs/libaeon.so.1 /usr/lib/habanalabs/libaeon.so
-          patch -p1 <$PYTORCH_MODULES_ROOT_PATH/.devops/patches/Intel_Gaudi3_Software.patch
-          cp $PYTORCH_MODULES_ROOT_PATH/.devops/patches/media_pytorch_proxy.h /usr/include/habanalabs/media_pytorch_proxy.h
+          chmod +w /usr/lib/habanalabs/
       
       - name: Build torch_hpu
         working-directory: torch_hpu
@@ -130,20 +133,21 @@ jobs:
           export THIRD_PARTIES_ROOT="$HABANA_SOFTWARE_STACK/3rd-parties"
 
           export HCL_ROOT="$HABANA_SOFTWARE_STACK/HCL/hcl/"
-          export HL_LOGGER_INCLUDE_DIRS="$HABANA_SOFTWARE_STACK/HCL/dependencies/swtools_sdk/hl_logger/include;$THIRD_PARTIES_ROOT"
-          export MEDIA_ROOT=/usr/include/habanalabs/
-          export SPECS_EXT_ROOT="$HABANA_SOFTWARE_STACK/Intel_Gaudi3_Software/specs_external/"
-          export SYNAPSE_ROOT=/usr/include/habanalabs/
-          export SYNAPSE_UTILS_ROOT=/usr/include/habanalabs/
+          export HCL_INCLUDE_DIR=$BASE_INC/
+          export HL_LOGGER_INCLUDE_DIRS=$BASE_INC/hl_logger
+          export MEDIA_ROOT=$(python -c "import habana_frameworks.mediapipe, os;print(os.path.dirname(habana_frameworks.mediapipe.__file__))")
+          export SPECS_EXT_ROOT=$BASE_INC
+          export SYNAPSE_ROOT=$BASE_INC
+          export SYNAPSE_INCLUDE_DIR=$BASE_INC
+          export SYNAPSE_UTILS_ROOT=$BASE_INC
+          export SYNAPSE_UTILS_INCLUDE_DIR=$BASE_INC
 
           export BUILD_ROOT="$HABANA_SOFTWARE_STACK/builds"
           export BUILD_ROOT_LATEST=/usr/lib/habanalabs/
           export PYTORCH_MODULES_RELEASE_BUILD="$BUILD_ROOT/pytorch_modules_release"  # the release build artifact directory
           export PYTORCH_MODULES_DEBUG_BUILD="$BUILD_ROOT/pytorch_modules_debug"  # the debug build artifact directory
           export PYTORCH_MODULES_ROOT_PATH="$HABANA_SOFTWARE_STACK/gaudi-pytorch-bridge"
 
-          pip install -r "$PYTORCH_MODULES_ROOT_PATH"/requirements.txt
-
           "$PYTORCH_MODULES_ROOT_PATH"/.devops/build.py -cir
 
       - name: List distribution package
@@ -195,4 +199,4 @@ jobs:
 
       - name: Cleanup workspace
         if: always()
-        run: rm -rf ${{ github.workspace }}/*
+        run: rm -rf ${{ github.workspace }}/*

.github/workflows/_gaudi_hpu_ut.yml

@@ -38,6 +38,10 @@ on:
           type: string
           default: '3.10'
           description: 'The python version to use'
+      rel_ver:
+        required: true
+        type: string
+        description: 'The release version of torch_hpu to use'
 
 # Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
 # declared as "shell: bash -el {0}" on steps that need to be properly activated.
@@ -100,7 +104,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: HabanaAI/gaudi-pytorch-bridge
-          ref: v1.20.0
+          ref: v${{ inputs.rel_ver }}
           path: torch_hpu
 
       - name: Install pip dependencies

.github/workflows/gaudi_hpu_test.yml

@@ -49,9 +49,10 @@ on:
         required: true
         type: choice
         options:
+          - vault.habana.ai/gaudi-docker/1.21.0/ubuntu24.04/habanalabs/pytorch-installer-2.6.0:latest
           - vault.habana.ai/gaudi-docker/1.20.0/ubuntu24.04/habanalabs/pytorch-installer-2.6.0:latest
           - vault.habana.ai/gaudi-docker/1.19.2/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest
-        default: 'vault.habana.ai/gaudi-docker/1.20.0/ubuntu24.04/habanalabs/pytorch-installer-2.6.0:latest'
+        default: 'vault.habana.ai/gaudi-docker/1.21.0/ubuntu24.04/habanalabs/pytorch-installer-2.6.0:latest'
         description: 'The docker image which will be loaded'
       device:
         required: true
@@ -77,10 +78,13 @@ jobs:
   prepare:
     name: Prepare
     runs-on: ubuntu-latest
+    env:
+      REL_VERSION: "1.21.0"
     outputs:
       runner: ${{ steps.set-env.outputs.runner }}
       image: ${{ steps.set-env.outputs.image }}
       device: ${{ steps.set-env.outputs.device }}
+      rel_ver: ${{ steps.set-env.outputs.rel_ver }}
       ref: ${{ steps.list-ref.outputs.ref }}
       pr-number: ${{ steps.list-ref.outputs.pr_number }}
     steps:
@@ -89,7 +93,8 @@ jobs:
         run: |
           echo "runner=${{ github.event.inputs.runner || 'linux-gaudi3' }}" >> $GITHUB_OUTPUT
           echo "device=${{ github.event.inputs.device || '0' }}" >> $GITHUB_OUTPUT
-          echo "image=${{ github.event.inputs.image || 'vault.habana.ai/gaudi-docker/1.20.0/ubuntu24.04/habanalabs/pytorch-installer-2.6.0:latest' }}" >> $GITHUB_OUTPUT
+          echo "image=${{ github.event.inputs.image || 'vault.habana.ai/gaudi-docker/1.21.0/ubuntu24.04/habanalabs/pytorch-installer-2.6.0:latest' }}" >> $GITHUB_OUTPUT
+          echo "rel_ver=$REL_VERSION" >> $GITHUB_OUTPUT
 
       - name: List ref to the PyTorch branch
         id: list-ref
@@ -128,6 +133,7 @@ jobs:
       image: ${{ needs.prepare.outputs.image }}
       device: ${{ needs.prepare.outputs.device }}
       torch-artifact: ${{ needs.build-torch.outputs.torch-artifact }}
+      rel_ver: ${{ needs.prepare.outputs.rel_ver }}
 
   test:
     name: Test torch_hpu
@@ -147,6 +153,7 @@ jobs:
       torch-artifact: ${{ needs.build-torch.outputs.torch-artifact }}
       torch-hpu-artifact: ${{ needs.build.outputs.torch-hpu-artifact }}
       torch-hpu-dl-artifact: ${{ needs.build.outputs.torch-hpu-dl-artifact }}
+      rel_ver: ${{ needs.prepare.outputs.rel_ver }}
 
   benchmark:
     name: Run benchmarks
@@ -167,3 +174,4 @@ jobs:
       torch-artifact: ${{ needs.build-torch.outputs.torch-artifact }}
       torch-hpu-artifact: ${{ needs.build.outputs.torch-hpu-artifact }}
       torch-hpu-dl-artifact: ${{ needs.build.outputs.torch-hpu-dl-artifact }}
+      rel_ver: ${{ needs.prepare.outputs.rel_ver }}