Add export-lora and nice print

Author: Lorenzo Toniazzi

tests/test_lora_conversion_and_inference.sh

@@ -1,10 +1,12 @@
 #!/bin/bash
 set -e
 
+MODELS_REPO=https://huggingface.co/ltoniazzi/reduce-llms-for-testing
+
 # Step 1: Clone the Hugging Face repository if the directory does not exist
 if [ ! -d "reduce-llms-for-testing" ]; then
     echo "Cloning the Hugging Face repository..."
-    git clone https://huggingface.co/ltoniazzi/reduce-llms-for-testing
+    git clone $MODELS_REPO
 else
     echo "Repository already exists. Skipping clone."
 fi
@@ -16,22 +18,51 @@ run_conversion_and_inference_lora() {
     local model_size_mb=$3
 
     echo "Running convert_hf_to_gguf.py for $model_name with size $size_matrix..."
-    python convert_hf_to_gguf.py reduce-llms-for-testing/$model_name/size=$size_matrix/base --outtype f32
+    python convert_hf_to_gguf.py reduce-llms-for-testing/$model_name/size=$size_matrix/base \
+    --outtype f32
 
     echo "Running convert_lora_to_gguf.py for $model_name with size $size_matrix..."
-    python3 convert_lora_to_gguf.py reduce-llms-for-testing/$model_name/size=$size_matrix/lora --base reduce-llms-for-testing/$model_name/size=$size_matrix/base --outtype f32
+    python3 convert_lora_to_gguf.py reduce-llms-for-testing/$model_name/size=$size_matrix/lora \
+    --base reduce-llms-for-testing/$model_name/size=$size_matrix/base \
+    --outtype f32
 
     echo "Running llama-cli without lora for $model_name with size $size_matrix and model size $model_size_mb..."
-    llama-cli -m reduce-llms-for-testing/$model_name/size=$size_matrix/base/Base-$model_size_mb-F32.gguf -p "<bos>When forty winters shall besiege" -n 50
+    OUTPUT_BASE=$(llama-cli -m reduce-llms-for-testing/$model_name/size=$size_matrix/base/Base-$model_size_mb-F32.gguf \
+    -p "<bos>When forty winters shall besiege" -n 50 --seed 42)
 
     echo "Running llama-cli with lora for $model_name with size $size_matrix and model size $model_size_mb..."
-    llama-cli -m reduce-llms-for-testing/$model_name/size=$size_matrix/base/Base-$model_size_mb-F32.gguf --lora reduce-llms-for-testing/$model_name/size=$size_matrix/lora/Lora-F32-LoRA.gguf -p "<bos>I see a little silhouetto" -n 50
+    OUTPUT_LORA_HOT=$(llama-cli -m reduce-llms-for-testing/$model_name/size=$size_matrix/base/Base-$model_size_mb-F32.gguf \
+    --lora reduce-llms-for-testing/$model_name/size=$size_matrix/lora/Lora-F32-LoRA.gguf \
+    -p "<bos>I see a little silhouetto" -n 50 --seed 42)
 
     # TODO add merge lora with lora-export and check
+    echo "Running llama-export-lora with lora for $model_name with size $size_matrix and model size $model_size_mb..."
+    llama-export-lora \
+    -m reduce-llms-for-testing/$model_name/size=$size_matrix/base/Base-$model_size_mb-F32.gguf \
+    -o reduce-llms-for-testing/$model_name/size=$size_matrix/base/Base-$model_size_mb-F32-lora-merged.gguf \
+    --lora reduce-llms-for-testing/$model_name/size=$size_matrix/lora/Lora-F32-LoRA.gguf \
+
+    echo "Running llama-cli with exported lora for $model_name with size $size_matrix and model size $model_size_mb..."
+    OUTPUT_LORA_MERGED=$(llama-cli -m reduce-llms-for-testing/$model_name/size=$size_matrix/base/Base-$model_size_mb-F32-lora-merged.gguf \
+    -p "<bos>I see a little silhouetto" -n 50 --seed 42)
+
+
+    # Echo the outputs with bullet points and spacing
+    echo -e "\n\n\n\033[1mResults:\033[0m"
+    echo -e "\n  • \033[32mBase:\n $OUTPUT_BASE"  # Green color for "BASE"
+    echo -e "\n  • \033[34mLora hot:\n $OUTPUT_LORA_HOT"  # Blue color for "Lora hot"
+    echo -e "\n  • \033[36mLora merged:\n $OUTPUT_LORA_MERGED"  # Cyan color for "Lora merged"
+    echo -e "\n\n\n \033[0m"
+
 
     echo "All steps completed for $model_name with size $size_matrix and model size $model_size_mb!"
 }
 
-# Example usage:
-run_conversion_and_inference_lora "Gemma2ForCausalLM" "64" "19M"
+declare -a params=(
+    "Gemma2ForCausalLM 64 19M"
+    # "AnotherModel 128 25M"
+)
 
+for param in "${params[@]}"; do
+    run_conversion_and_inference_lora $param
+done