Fix test-cpu-eval-sanity-check-* CI jobs broken due to "evaluate" module conflicts (#1215)

* Fix eval sanity check CI

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

* Fixate psutil to 6.0.0

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

* Pre-import evaluate

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

* Force install evaluate

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

* Revert test-cpu-eval-sanity-check-float32 changes since it breaks AOTI

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

* Remove debug log

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

* Cleanup

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

* Remove redundant import

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

* Ok can't remove import lm_eval. Move it inside eval condition

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

* Add file hash to cache key

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

* Update cache key for runner-et

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

Author: larryliu0820

.github/workflows/pull.yml

@@ -904,14 +904,12 @@ jobs:
           echo "et-git-hash=$(cat ${TORCHCHAT_ROOT}/install/.pins/et-pin.txt)" >> "$GITHUB_ENV"
       - name: Load or install ET
         id: install-et
-        uses: actions/cache@v3
-        env:
-          cache-key: et-build-${{runner.os}}-${{runner.arch}}-${{env.et-git-hash}}
+        uses: actions/cache@v4
         with:
-          path: ./et-build
-          key: ${{env.cache-key}}
-          restore-keys: |
-            ${{env.cache-key}}
+          path: |
+            ./et-build
+            ./torchchat/utils/scripts
+          key: et-build-${{runner.os}}-${{runner.arch}}-${{env.et-git-hash}}-${{ hashFiles('**/install_et.sh') }}
       - if: ${{ steps.install-et.outputs.cache-hit != 'true' }}
         continue-on-error: true
         run: |

install/install_requirements.sh

@@ -103,5 +103,5 @@ fi
 
 (
   set -x
-  $PIP_EXECUTABLE install lm-eval=="0.4.2"
+  $PIP_EXECUTABLE install evaluate=="0.4.3" lm-eval=="0.4.2" psutil=="6.0.0"
 )

torchchat/model.py

@@ -30,7 +30,9 @@
     SequenceParallel,
 )
 from torch.nn import functional as F
-
+# TODO: remove this after we figure out where in torchtune an `evaluate` module
+# is being imported, which is being confused with huggingface's `evaluate``.
+import lm_eval  # noqa 
 from torchtune.models.clip import clip_vision_encoder
 from torchtune.models.llama3_1._component_builders import llama3_1 as llama3_1_builder
 from torchtune.models.llama3_2_vision._component_builders import (

torchchat/utils/scripts/install_et.sh

@@ -20,4 +20,9 @@ find_cmake_prefix_path
 clone_executorch
 install_executorch_libs $ENABLE_ET_PYBIND
 install_executorch_python_libs $ENABLE_ET_PYBIND
+# TODO: figure out the root cause of 'AttributeError: module 'evaluate'
+# has no attribute 'utils'' error from evaluate CI jobs and remove
+# `import lm_eval` from torchchat.py since it requires a specific version
+# of numpy.
+pip install numpy=='1.26.4'
 popd