Merge pull request #11 from jpodivin/ld_auth

Authentication improvement

Author: jpodivin

.gitignore

@@ -0,0 +1,2 @@
+# Ignore potential API keys in file form
+.*API*

README.md

@@ -51,13 +51,14 @@ Evaluation of Log Detective performance can be performed automatically using
 the `validation.py`script. Dependencies for the tool are defined in the
 `requirements.txt` file and should be installed in a virtual environment.
 
-Before running the script, the API key for the LLM judge must be set
-in an environment variable `OPENAI_API_KEY`.
+Keys for OpenAI compatible LLM inference provider and Log Detective itself,
+must be stored in files. These will be used at runtime.
+This prevents logging of secrets in history.
 
 Example:
 
 ```
-./validation.py <DATA_PATH> <LOG_DETECTIVE_URL> <LLM_URL> <LLM_NAME>
+./validation.py <PATH_TO_OPEN_AI_API_KEY> <DATA_PATH> <LOG_DETECTIVE_URL> <LLM_URL> <LLM_NAME>
 ```
 Script sends each of the the stored log files for evaluation by Log Detective,
 then submits both results of final analysis from Log Detective and actual issue

validation.py

@@ -10,15 +10,20 @@
 import argparse
 from pydantic import BaseModel, Field, ValidationError
 
-# --- Configuration ---
-# Set your OpenAI API key as an environment variable named OPENAI_API_KEY
-# You can get a key from https://beta.openai.com/account/api-keys
-API_KEY = os.getenv("OPENAI_API_KEY")
 LOG_REPO_BASE_URL = (
     "https://raw.githubusercontent.com/fedora-copr/logdetective-sample/main/data/"
 )
 
 
+def get_api_key_from_file(path: str):
+    """Attempt to read API key from a file.
+    This is safer than typing it in CLI."""
+
+    with open(path) as key_file:
+
+        return key_file.read().strip()
+
+
 class SimilarityScore(BaseModel):
     """Defines the structure for the similarity score response from the LLM."""
 
@@ -94,6 +99,7 @@ def evaluate_samples(
     llm_model: str,
     llm_token: str,
     log_detective_api_timeout: int,
+    log_detective_api_key: str = "",
 ) -> None:
     """
     Traverses a directory to find and evaluate log analysis samples.
@@ -106,6 +112,10 @@ def evaluate_samples(
 
     full_api_url = f"{server_address}{api_endpoint}"
 
+    log_detective_request_headers = {}
+    if log_detective_api_key:
+        log_detective_request_headers["Authorization"] = f"Bearer {log_detective_api_key}"
+
     client = openai.OpenAI(base_url=llm_url, api_key=llm_token)
     scores = []
     elapsed_times = []
@@ -146,8 +156,8 @@ def evaluate_samples(
                     )
                     start_time = time.time()
                     api_response = requests.post(
-                        full_api_url, json=payload, timeout=log_detective_api_timeout
-                    )
+                        full_api_url, json=payload, timeout=log_detective_api_timeout,
+                        headers=log_detective_request_headers)
                     api_response.raise_for_status()
                     actual_response_data = api_response.json()
                     time_elapsed = time.time() - start_time
@@ -206,6 +216,12 @@ def main():
     """
     parser = argparse.ArgumentParser(
         description="Evaluate AI system performance by comparing expected and actual responses.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument(
+        "open_ai_api_key",
+        help="Path to file with API key to OpenAI compatible inference provider",
+        type=str,
     )
     parser.add_argument(
         "data_directory", help="Path to the directory containing the sample data."
@@ -217,28 +233,37 @@ def main():
     parser.add_argument("llm_url", help="URL of LLM API to use as judge")
     parser.add_argument("llm_model", help="Name of LLM model to use a judge")
     parser.add_argument(
-        "log_detective_api_timeout",
+        "--log-detective-api-timeout",
         help="Request timeout for Log Detective API",
         type=int,
         default=60,
     )
+    parser.add_argument(
+        "--log-detective-api-key",
+        help="Path to file with Log Detective API key, if one is necessary",
+        type=str,
+        default=""
+    )
     args = parser.parse_args()
 
-    if not API_KEY:
-        print("Error: OPENAI_API_KEY environment variable not set.", file=sys.stderr)
-        sys.exit(1)
+    open_ai_api_key = get_api_key_from_file(args.open_ai_api_key)
 
     if not os.path.isdir(args.data_directory):
         print(f"Error: Directory not found at '{args.data_directory}'", file=sys.stderr)
         sys.exit(1)
 
+    log_detective_api_key = ""
+    if args.log_detective_api_key:
+        log_detective_api_key = get_api_key_from_file(args.log_detective_api_key)
+
     evaluate_samples(
-        args.data_directory,
-        args.logdetective_url,
-        args.llm_url,
-        args.llm_model,
-        API_KEY,
-        args.log_detective_api_timeout,
+        directory=args.data_directory,
+        server_address=args.logdetective_url,
+        llm_url=args.llm_url,
+        llm_model=args.llm_model,
+        llm_token=open_ai_api_key,
+        log_detective_api_timeout=args.log_detective_api_timeout,
+        log_detective_api_key=log_detective_api_key
     )