Add early LLM credential validation to prevent wasted work (#150)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: mawad-amd <112003944+mawad-amd@users.noreply.github.com>

Author: Copilot

src/intelliperf/__main__.py

@@ -213,6 +213,14 @@ def main():
 	else:
 		logging.basicConfig(level=logging.WARNING, format="[INTELLIPERF] %(levelname)s: %(message)s")
 
+	# Validate LLM credentials early before doing any work (only for non-diagnoseOnly formulas)
+	if args.formula != "diagnoseOnly":
+		from intelliperf.core.llm import validate_llm_credentials
+		from intelliperf.utils.env import get_llm_api_key
+
+		llm_key = get_llm_api_key()
+		validate_llm_credentials(api_key=llm_key, model=args.model, provider=args.provider)
+
 	# Create an optimizer based on available IntelliPerf formulas.
 	if args.formula == "bankConflict":
 		from intelliperf.formulas.bank_conflict import bank_conflict

src/intelliperf/core/llm.py

@@ -23,16 +23,92 @@
 ################################################################################
 
 
+import logging
 import sys
 from typing import Optional
 
 import dspy
 import requests
-
 from intelliperf.core.logger import Logger
 
 
+def validate_llm_credentials(api_key: str, model: str, provider: str) -> bool:
+	"""Validate LLM credentials before doing any work.
+
+	Args:
+	    api_key: The API key to validate
+	    model: The model name to validate
+	    provider: The provider URL to validate
+
+	Returns:
+	    bool: True if credentials are valid, False otherwise.
+
+	Raises:
+	    SystemExit: If credentials are invalid, exits with error message.
+	"""
+	# Create a temporary LLM instance to validate credentials
+	temp_llm = LLM(api_key=api_key, system_prompt="test", model=model, provider=provider)
+	return temp_llm._validate_credentials()
+
+
 class LLM:
+	def _validate_credentials(self) -> bool:
+		"""Validate that the LLM credentials, model, and provider are correct.
+
+		Returns:
+		    bool: True if credentials are valid, False otherwise.
+
+		Raises:
+		    SystemExit: If credentials are invalid, exits with error message.
+		"""
+		try:
+			if self.use_amd:
+				# Test AMD/Azure credentials with a minimal request
+				body = {
+					"messages": [
+						{"role": "system", "content": "You are a helpful assistant."},
+						{"role": "user", "content": "Hi"},
+					],
+					"max_Tokens": 10,
+					"max_Completion_Tokens": 10,
+				}
+				url = f"{self.provider}/engines/{self.model}/chat/completions"
+				resp = requests.post(url, json=body, headers=self.header, timeout=10)
+				resp.raise_for_status()
+				logging.info("LLM credentials validated successfully.")
+				return True
+			else:
+				# Test DSPy/OpenAI credentials with a minimal request
+				try:
+					# Make a simple test call through dspy
+					test_signature = "input: str -> output: str"
+					chain = dspy.ChainOfThought(test_signature)
+					chain(input="test")
+					logging.info("LLM credentials validated successfully.")
+					return True
+				except Exception as e:
+					raise e
+		except requests.exceptions.HTTPError as e:
+			status_code = e.response.status_code if hasattr(e, "response") else None
+			error_msg = "LLM credential validation failed.\n"
+			error_msg += f"Provider: {self.provider}\n"
+			error_msg += f"Model: {self.model}\n"
+			if status_code == 401:
+				error_msg += "Error: Authentication failed. Please check your API key.\n"
+			elif status_code == 404:
+				error_msg += "Error: Model or endpoint not found. Please check the model name and provider URL.\n"
+			else:
+				error_msg += f"Error: HTTP {status_code} - {str(e)}\n"
+			logging.error(error_msg)
+			sys.exit(1)
+		except Exception as e:
+			error_msg = "LLM credential validation failed.\n"
+			error_msg += f"Provider: {self.provider}\n"
+			error_msg += f"Model: {self.model}\n"
+			error_msg += f"Error: {str(e)}\n"
+			logging.error(error_msg)
+			sys.exit(1)
+
 	def _get_model_context_length(self) -> Optional[int]:
 		"""Query the model's max context length from the API"""
 		import logging

tests/test_llm_validation.py

@@ -0,0 +1,76 @@
+from unittest.mock import Mock, patch
+
+import pytest
+from intelliperf.core.llm import validate_llm_credentials
+
+
+def test_validate_credentials_amd_success():
+	"""Test successful credential validation for AMD provider."""
+	with patch("requests.post") as mock_post, patch("dspy.LM"), patch("dspy.configure"):
+		mock_response = Mock()
+		mock_response.status_code = 200
+		mock_response.json.return_value = {"choices": [{"message": {"content": "test"}}]}
+		mock_post.return_value = mock_response
+
+		# Should not raise an exception
+		result = validate_llm_credentials(
+			api_key="test-key", model="test-model", provider="https://llm-api.amd.com/azure"
+		)
+		assert result is True
+
+
+def test_validate_credentials_amd_auth_failure():
+	"""Test credential validation fails with 401 for AMD provider."""
+	with patch("requests.post") as mock_post, patch("dspy.LM"), patch("dspy.configure"):
+		mock_response = Mock()
+		mock_response.status_code = 401
+		mock_http_error = Exception("401 Unauthorized")
+		mock_http_error.response = mock_response
+		mock_response.raise_for_status.side_effect = mock_http_error
+		mock_post.return_value = mock_response
+
+		with pytest.raises(SystemExit):
+			validate_llm_credentials(api_key="bad-key", model="test-model", provider="https://llm-api.amd.com/azure")
+
+
+def test_validate_credentials_amd_model_not_found():
+	"""Test credential validation fails with 404 for AMD provider."""
+	with patch("requests.post") as mock_post, patch("dspy.LM"), patch("dspy.configure"):
+		mock_response = Mock()
+		mock_response.status_code = 404
+		mock_http_error = Exception("404 Not Found")
+		mock_http_error.response = mock_response
+		mock_response.raise_for_status.side_effect = mock_http_error
+		mock_post.return_value = mock_response
+
+		with pytest.raises(SystemExit):
+			validate_llm_credentials(
+				api_key="test-key",
+				model="nonexistent-model",
+				provider="https://llm-api.amd.com/azure",
+			)
+
+
+def test_validate_credentials_openai_success():
+	"""Test successful credential validation for OpenAI provider."""
+	with patch("dspy.LM"), patch("dspy.configure"), patch("dspy.ChainOfThought") as mock_chain:
+		# Mock the chain of thought call
+		mock_chain_instance = Mock()
+		mock_chain_instance.return_value = Mock(output="test")
+		mock_chain.return_value = mock_chain_instance
+
+		# Should not raise an exception
+		result = validate_llm_credentials(api_key="test-key", model="gpt-4", provider="openai")
+		assert result is True
+
+
+def test_validate_credentials_openai_failure():
+	"""Test credential validation fails for OpenAI provider."""
+	with patch("dspy.LM"), patch("dspy.configure"), patch("dspy.ChainOfThought") as mock_chain:
+		# Mock the chain of thought to raise an exception
+		mock_chain_instance = Mock()
+		mock_chain_instance.side_effect = Exception("Authentication failed")
+		mock_chain.return_value = mock_chain_instance
+
+		with pytest.raises(SystemExit):
+			validate_llm_credentials(api_key="bad-key", model="gpt-4", provider="openai")