Merge pull request #115 from Azure-Samples/installable

Don't misuse the model configuration to store extra keys

Author: pamelafox

src/evaltools/eval/evaluate.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import os
 import time
 from pathlib import Path
 
@@ -94,6 +95,8 @@ def run_evaluation(
     num_questions=None,
     target_response_answer_jmespath=None,
     target_response_context_jmespath=None,
+    model=None,
+    azure_credential=None,
 ):
     logger.info("Running evaluation using data from %s", testdata_path)
     testdata = load_jsonl(testdata_path)
@@ -123,8 +126,8 @@ def run_evaluation(
         return False
 
     logger.info("Sending a test chat completion to the GPT deployment to ensure it is running...")
-    gpt_response = service_setup.get_openai_client(openai_config).chat.completions.create(
-        model=openai_config["model"],
+    gpt_response = service_setup.get_openai_client(openai_config, azure_credential).chat.completions.create(
+        model=model,
         messages=[{"role": "user", "content": "Hello!"}],
         n=1,
     )
@@ -190,7 +193,7 @@ def evaluate_row(row):
 
     with open(results_dir / "evaluate_parameters.json", "w", encoding="utf-8") as parameters_file:
         parameters = {
-            "evaluation_gpt_model": openai_config["model"],
+            "evaluation_gpt_model": model,
             "evaluation_timestamp": int(time.time()),
             "testdata_path": str(testdata_path),
             "target_url": target_url,
@@ -221,7 +224,14 @@ def process_config(obj: dict):
 
 
 def run_evaluate_from_config(
-    working_dir, config_path, num_questions=None, target_url=None, results_dir=None, openai_config=None
+    working_dir,
+    config_path,
+    num_questions=None,
+    target_url=None,
+    results_dir=None,
+    openai_config=None,
+    model=None,
+    azure_credential=None,
 ):
     config_path = working_dir / Path(config_path)
     logger.info("Running evaluation from config %s", config_path)
@@ -245,6 +255,8 @@ def run_evaluate_from_config(
         ),
         target_response_answer_jmespath=config.get("target_response_answer_jmespath", "message.content"),
         target_response_context_jmespath=config.get("target_response_context_jmespath", "context.data_points.text"),
+        model=model or os.environ["OPENAI_GPT_MODEL"],
+        azure_credential=azure_credential,
     )
 
     if evaluation_run_complete:

src/evaltools/review/diff_markdown.py

@@ -1,8 +1,15 @@
 from pathlib import Path
+from typing import Any
 
 from .utils import diff_directories
 
 
+def _round_metric(value: Any) -> Any:
+    if isinstance(value, float):
+        return round(value, 1)
+    return value
+
+
 def main(directories: list[Path], changed: str | None = None):
     data_dicts = diff_directories(directories, changed)
 
@@ -29,14 +36,14 @@ def main(directories: list[Path], changed: str | None = None):
             if isinstance(value, int | float):
                 metrics[column] = []
         for metric_name in metrics.keys():
+            first_value = _round_metric(data_dicts[0][question].get(metric_name))
             for ind, data_dict in enumerate(data_dicts):
-                value = data_dict[question].get(metric_name)
-                value_str = str(round(value, 1) if isinstance(value, float) else value)
+                value = _round_metric(data_dict[question].get(metric_name))
                 # Insert arrow emoji based on the difference between metric value and the first data_dict
                 value_emoji = ""
-                if value is not None and ind > 0 and value != data_dicts[0][question][metric_name]:
+                if value is not None and ind > 0 and value != first_value:
                     value_emoji = "⬆️" if value > data_dicts[0][question][metric_name] else "⬇️"
-                metrics[metric_name].append(f"{value_str} {value_emoji}")
+                metrics[metric_name].append(f"{value} {value_emoji}")
         # make a row for each metric
         for metric_name, metric_values in metrics.items():
             markdown_str += (

src/evaltools/service_setup.py

@@ -38,7 +38,6 @@ def get_openai_config() -> dict:
             }
             # azure-ai-evaluate will call DefaultAzureCredential behind the scenes,
             # so we must be logged in to Azure CLI with the correct tenant
-        openai_config["model"] = os.environ["OPENAI_GPT_MODEL"]
     else:
         logger.info("Using OpenAI Service with API Key from OPENAICOM_KEY")
         openai_config: OpenAIModelConfiguration = {
@@ -97,14 +96,13 @@ def get_search_client():
     )
 
 
-def get_openai_client(oai_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration]):
+def get_openai_client(
+    oai_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], azure_credential=None
+):
     if "azure_deployment" in oai_config:
         azure_token_provider = None
-        azure_credential = None
-        if "credential" in oai_config:
-            logger.info("Using Azure OpenAI Service with provided credential")
-            azure_credential = oai_config["credential"]
-        elif not os.environ.get("AZURE_OPENAI_KEY"):
+
+        if azure_credential is None and not os.environ.get("AZURE_OPENAI_KEY"):
             logger.info("Using Azure OpenAI Service with Azure Developer CLI Credential")
             azure_credential = get_azd_credential(os.environ.get("AZURE_OPENAI_TENANT_ID"))
         if azure_credential is not None: