Ok can't remove import lm_eval. Move it inside eval condition

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

Author: larryliu0820

install/install_requirements.sh

@@ -103,5 +103,5 @@ fi
 
 (
   set -x
-  $PIP_EXECUTABLE install evaluate=="0.4.3" lm-eval=="0.4.2" "numpy>=1.17,<2.0" psutil=="6.0.0"
+  $PIP_EXECUTABLE install evaluate=="0.4.3" lm-eval=="0.4.2" psutil=="6.0.0"
 )

torchchat/model.py

@@ -32,6 +32,7 @@
 from torch.nn import functional as F
 
 from torchtune.models.clip import clip_vision_encoder
+import lm_eval  # noqa
 from torchtune.models.llama3_1._component_builders import llama3_1 as llama3_1_builder
 from torchtune.models.llama3_2_vision._component_builders import (
     llama3_2_vision_decoder,

torchchat/usages/eval.py

@@ -10,8 +10,6 @@
 import torch._dynamo.config
 import torch._inductor.config
 
-import lm_eval  # noqa
-
 from torchchat.cli.builder import (
     _initialize_model,
     _initialize_tokenizer,
@@ -30,6 +28,8 @@
 torch._inductor.config.triton.cudagraphs = True
 torch._dynamo.config.cache_size_limit = 100000
 
+import lm_eval
+
 from lm_eval.evaluator import evaluate
 from lm_eval.models.huggingface import HFLM as eval_wrapper
 from lm_eval.tasks import get_task_dict

torchchat/utils/scripts/install_et.sh

@@ -20,4 +20,9 @@ find_cmake_prefix_path
 clone_executorch
 install_executorch_libs $ENABLE_ET_PYBIND
 install_executorch_python_libs $ENABLE_ET_PYBIND
+# TODO: figure out the root cause of 'AttributeError: module 'evaluate'
+# has no attribute 'utils'' error from evaluate CI jobs and remove
+# `import lm_eval` from torchchat.py since it requires a specific version
+# of numpy.
+pip install numpy=='1.26.4'
 popd