Just Try Catch

Author: Jack-Khuu

install/install_requirements.sh

@@ -52,9 +52,6 @@ PYTORCH_NIGHTLY_VERSION=dev20240901
 # Nightly version for torchvision
 VISION_NIGHTLY_VERSION=dev20240901
 
-# Nightly version for torchtune
-TUNE_NIGHTLY_VERSION=dev20240928
-
 # Uninstall triton, as nightly will depend on pytorch-triton, which is one and the same
 (
   set -x
@@ -75,7 +72,6 @@ fi
 REQUIREMENTS_TO_INSTALL=(
   torch=="2.5.0.${PYTORCH_NIGHTLY_VERSION}"
   torchvision=="0.20.0.${VISION_NIGHTLY_VERSION}"
-  torchtune=="0.3.0.${TUNE_NIGHTLY_VERSION}"
 )
 
 # Install the requirements. --extra-index-url tells pip to look for package
@@ -91,6 +87,12 @@ REQUIREMENTS_TO_INSTALL=(
   $PIP_EXECUTABLE install torchao=="0.5.0"
 )
 
+# Rely on the latest tochtune for flamingo support
+(
+  set -x
+  $PIP_EXECUTABLE install -I git+https://github.com/pytorch/torchtune.git@d002d45e3ec700fa770d9dcc61b02c59e2507bf6
+)
+
 if [[ -x "$(command -v nvidia-smi)" ]]; then
   (
     set -x

torchchat/model.py

@@ -30,6 +30,15 @@
     SequenceParallel,
 )
 from torch.nn import functional as F
+
+try:
+    # TODO: remove this after we figure out where in torchtune an `evaluate` module
+    # is being imported, which is being confused with huggingface's `evaluate``.
+    import lm_eval  # noqa 
+except Exception:
+    pass
+
+from torchtune.models.clip import clip_vision_encoder
 from torchtune.models.llama3_1._component_builders import llama3_1 as llama3_1_builder
 from torchtune.models.llama3_2_vision._component_builders import (
     llama3_2_vision_decoder,
@@ -223,9 +232,6 @@ def _flamingo(cls):
 
     @classmethod
     def _llava(cls):
-        # TODO: Import locally until we find where in torchtune an `evaluate` module
-        # is being imported, which is being confused with huggingface's `evaluate``.
-        from torchtune.models.clip import clip_vision_encoder
         return cls(
             model_type=ModelType.Llava,
             modules={

torchchat/usages/eval.py

@@ -3,10 +3,9 @@
 
 # This source code is licensed under the license found in the
 # LICENSE file in the root directory of this source tree.
+import argparse
 from typing import Callable, Optional
 
-import lm_eval
-
 import torch
 import torch._dynamo.config
 import torch._inductor.config
@@ -17,6 +16,7 @@
     BuilderArgs,
     TokenizerArgs,
 )
+from torchchat.cli.cli import add_arguments_for_verb, arg_init
 
 from torchchat.model import Model
 from torchchat.utils.build_utils import set_precision
@@ -28,6 +28,8 @@
 torch._inductor.config.triton.cudagraphs = True
 torch._dynamo.config.cache_size_limit = 100000
 
+import lm_eval
+
 from lm_eval.evaluator import evaluate
 from lm_eval.models.huggingface import HFLM as eval_wrapper
 from lm_eval.tasks import get_task_dict

torchchat/utils/scripts/install_utils.sh

@@ -94,6 +94,12 @@ install_executorch_python_libs() {
       bash ./install_requirements.sh --pybind xnnpack
   fi
 
+  # TODO: figure out the root cause of 'AttributeError: module 'evaluate'
+  # has no attribute 'utils'' error from evaluate CI jobs and remove
+  # `import lm_eval` from torchchat.py since it requires a specific version
+  # of numpy.
+  pip install numpy=='1.26.4'
+
   pip3 list
   popd
 }