Merge pull request #67 from ansible/chatbot_eval

PR to update e2e tests for Ansible chatbot service including model evaluation n response.

Author: justjais

scripts/evaluation/driver.py

@@ -1,6 +1,8 @@
+# ruff: noqa: I001
 """Driver for evaluation."""
 
 import argparse
+from os import path
 import sys
 
 from httpx import Client
@@ -96,8 +98,11 @@ def main():
     client = Client(base_url=args.eval_api_url, verify=False)  # noqa: S501
 
     if "localhost" not in args.eval_api_url:
-        with open(args.eval_api_token_file, mode="r", encoding="utf-8") as t_f:
-            token = t_f.read().rstrip()
+        if path.isfile(args.eval_api_token_file):
+            with open(args.eval_api_token_file, mode="r", encoding="utf-8") as t_f:
+                token = t_f.read().rstrip()
+        else:
+            token = args.eval_api_token_file
         client.headers.update({"Authorization": f"Bearer {token}"})
 
     resp_eval = ResponseEvaluation(args, client)

scripts/evaluation/olsconfig.yaml

@@ -17,12 +17,18 @@ llm_providers:
     models:
       - name: "mistral"
       - name: 'llama3.2:latest'
-  - name: my_rhoai
+  - name: my_rhoai_g3
     type: openai
-    url: "https://granite3-8b-wisdom-model-staging.apps.stage2-west.v2dz.p1.openshiftapps.com/v1"
+    url: "http://localhost:8000/v1"
     credentials_path: ols_api_key.txt
     models:
       - name: granite3-8b
+  - name: my_rhoai_g31
+    type: openai
+    url: "http://localhost:8000/v1"
+    credentials_path: rhoai_api_key.txt
+    models:
+      - name: granite3-1-8b
 ols_config:
   # max_workers: 1
   reference_content:
@@ -37,8 +43,10 @@ ols_config:
     app_log_level: info
     lib_log_level: warning
     uvicorn_log_level: info
-  default_provider: ollama
-  default_model: 'llama3.2:latest'
+  # default_provider: my_rhoai_g31
+  # default_model: 'granite3-1-8b'
+  default_provider: my_rhoai_g3
+  default_model: 'granite3-8b'
   query_validation_method: llm
   user_data_collection:
     feedback_disabled: false

scripts/evaluation/utils/constants.py

@@ -11,8 +11,8 @@
     "azure_openai+gpt-4o": ("azure_openai", "gpt-4o"),
     "ollama+llama3.1:latest": ("ollama", "llama3.1:latest"),
     "ollama+mistral": ("ollama", "mistral"),
-    "my_rhoai+granite3-8b": ("my_rhoai", "granite3-8b"),
-    "my_rhoai3+granite3-1-8b": ("my_rhoai3", "granite3-1-8b"),
+    "my_rhoai_g31+granite3-1-8b": ("my_rhoai_g31", "granite3-1-8b"),
+    "my_rhoai_g3+granite3-8b": ("my_rhoai_g3", "granite3-8b"),
 }
 
 NON_LLM_EVALS = {
@@ -44,7 +44,7 @@
 DEFAULT_RESULT_DIR = "eval_result"
 
 # Retry settings for LLM calls used when model does not respond reliably in 100% cases
-MAX_RETRY_ATTEMPTS = 10
+MAX_RETRY_ATTEMPTS = 5
 REST_API_TIMEOUT = 120
 TIME_TO_BREATH = 10
 

scripts/evaluation/utils/prompts.py

@@ -16,11 +16,12 @@
 
 Use below json format for your response. Do not add any additional text apart from json output.
 {{
-    Question: [
+    "Question": [
         QUESTION 1,
         QUESTION 2,
+        QUESTION 3
     ],
-    Valid: 0 or 1
+    "Valid": 0 or 1
 }}
 
 ```

scripts/evaluation/utils/response.py

@@ -17,7 +17,7 @@ def get_model_response(query, provider, model, mode, api_client=None):
     """Get response depending upon the mode."""
     if mode == "ols":
         response = api_client.post(
-            "/v1/query",
+            "/api/v1/ai/chat/",
             json={
                 "query": query,
                 "provider": provider,