diff --git a/.github/workflows/e2e-reusable.yml b/.github/workflows/e2e-reusable.yml
index 71446621de..acfb5cc965 100644
--- a/.github/workflows/e2e-reusable.yml
+++ b/.github/workflows/e2e-reusable.yml
@@ -137,7 +137,7 @@ jobs:
 
       - name: Install python test dependencies
         run: |
-          pip install pyyaml pandas scipy 'numpy==1.26.4' psutil pyre_extensions torchrec
+          pip install pyyaml pandas scipy psutil
 
       - name: Install transformers package
         if: ${{ inputs.suite == 'huggingface' }}
@@ -186,11 +186,19 @@ jobs:
         if: ${{ inputs.suite == 'torchbench' }}
         run: |
           cd benchmark
+          sed -i 's/^ *pynvml.*//' requirements.txt
+          pip install -r requirements.txt
+          # for dlrm
+          pip install pyre-extensions
+          curl -fsSL https://raw.githubusercontent.com/facebookresearch/dlrm/refs/heads/torchrec-dlrm/requirements.txt |xargs pip install
+          # for soft_actor_critic, temp fix
+          pip install git+https://github.com/nocoding03/gym@fix-np
+
           if [[ "${{ inputs.only_one_model }}" ]]; then
             python install.py "${{ inputs.only_one_model }}"
           else
             # install all models
-            python install.py
+            python install.py --continue_on_fail
           fi
           pip install -e .
 
@@ -200,6 +208,9 @@ jobs:
         run: |
           cd pytorch
 
+          # Some models are still not compatible with numpy>2.0
+          pip install 'numpy==1.26.4'
+
           export WORKSPACE=$GITHUB_WORKSPACE
 
           if [[ "${{ inputs.TORCH_COMPILE_DEBUG }}" = "1" ]] ; then