Merge pull request #167 from georgian-io/fix-llm-qa

Fix LLM QA

Author: benjaminye

test_utils/test_config.py

@@ -6,6 +6,7 @@
     InferenceConfig,
     LoraConfig,
     ModelConfig,
+    QaConfig,
     SftArgs,
     TrainingArgs,
     TrainingConfig,
@@ -70,4 +71,16 @@ def get_sample_config():
             test_size=0.1,
             train_test_split_seed=42,
         ),
+        qa=QaConfig(
+            llm_tests=[
+                "jaccard_similarity",
+                "dot_product",
+                "rouge_score",
+                "word_overlap",
+                "verb_percent",
+                "adjective_percent",
+                "noun_percent",
+                "summary_length",
+            ]
+        ),
     )

tests/qa/test_generics.py

@@ -12,9 +12,9 @@ def mock_rich_ui(mocker):
 @pytest.fixture
 def example_data():
     data = {
-        "prompt": ["What is 2+2?", "What is the capital of France?"],
-        "ground_truth": ["4", "Paris"],
-        "model_prediction": ["3", "Paris"],
+        "Prompt": ["What is 2+2?", "What is the capital of France?"],
+        "Ground Truth": ["4", "Paris"],
+        "Predicted": ["3", "Paris"],
     }
     return DataFrame(data)
 
@@ -65,9 +65,7 @@ def test_save_test_results(mock_csv, mock_tests, mocker):
 
 def test_print_test_results(capfd, example_data):
     tests = [MockQaTest()]
-    test_suite = LLMTestSuite(
-        tests, example_data["prompt"], example_data["ground_truth"], example_data["model_prediction"]
-    )
+    test_suite = LLMTestSuite(tests, example_data["Prompt"], example_data["Ground Truth"], example_data["Predicted"])
     test_suite.print_test_results()
     out, err = capfd.readouterr()