Skip to content

Fix AzureOpenAI keyless auth by deferring credential validation to SDK #2464

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ppon1086 wants to merge 1 commit into
base: main
Choose a base branch
from
+176 −17
Open

Fix AzureOpenAI keyless auth by deferring credential validation to SDK #2464

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 36 additions & 17 deletions deepeval/models/llms/azure_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,33 +442,52 @@ def _client_kwargs(self) -> Dict:
return kwargs

def _build_client(self, cls):
# Only require the API key / Azure ad token if no token provider is supplied
azure_ad_token = None
api_key = None

# Defer authentication validation to the OpenAI SDK.
# Only fail fast if the user explicitly provided an empty credential.

api_key_value = None
if self.api_key is not None:
try:
api_key_value = self.api_key.get_secret_value()
except Exception:
api_key_value = str(self.api_key)

azure_ad_token_value = None
if self.azure_ad_token is not None:
try:
azure_ad_token_value = self.azure_ad_token.get_secret_value()
except Exception:
azure_ad_token_value = str(self.azure_ad_token)

if self.azure_ad_token_provider is None:
Copy link
Collaborator

@A-Vamshi A-Vamshi Feb 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In this block we're following a precedence, we're giving priority to azure_ad_token_provider first, if that doesn't exist we give priority to azure_ad_token over api_key. This allows us to handle the case where a user provides all 3 of these fields (azure_ad_token_provider > azure_ad_token > api_key), since we're already populating api_key_value and azure_ad_token_value which are all being passed to the client below, I'm worried about which one is used by the client in this case? Do you know what happens in this case and if it's safe to do this?

if self.azure_ad_token is not None:
azure_ad_token = require_secret_api_key(
self.azure_ad_token,
provider_label="AzureOpenAI",
env_var_name="AZURE_OPENAI_AD_TOKEN",
param_hint="`azure_ad_token` to AzureOpenAIModel(...)",
if (
azure_ad_token_value is not None
and isinstance(azure_ad_token_value, str)
and not azure_ad_token_value.strip()
):
raise DeepEvalError(
"azure_ad_token was provided but is empty. Omit it to defer auth to the OpenAI SDK."
)
else:
api_key = require_secret_api_key(
self.api_key,
provider_label="AzureOpenAI",
env_var_name="AZURE_OPENAI_API_KEY",
param_hint="`api_key` to AzureOpenAIModel(...)",

if (
api_key_value is not None
and isinstance(api_key_value, str)
and not api_key_value.strip()
):
raise DeepEvalError(
"api_key was provided but is empty. Omit it to defer auth to the OpenAI SDK."
)
# else: neither key nor token nor provider set -> defer to SDK


kw = dict(
api_key=api_key,
api_key=api_key_value,
api_version=self.api_version,
azure_endpoint=self.base_url,
azure_deployment=self.deployment_name,
azure_ad_token_provider=self.azure_ad_token_provider,
azure_ad_token=azure_ad_token,
azure_ad_token=self.azure_ad_token_value,
Copy link
Collaborator

@A-Vamshi A-Vamshi Feb 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason we're using self.azure_ad_token_value here and not azure_ad_token_value you created above?

**self._client_kwargs(),
)
try:
Expand Down
140 changes: 140 additions & 0 deletions tests/test_core/test_models/test_azure_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,146 @@ def test_none_generation_kwargs(self, settings):
# Test Secret Management #
##########################

def test_azure_openai_model_defers_auth_when_no_key_token_or_provider(monkeypatch, settings):
"""
Keyless / Managed Identity scenarios may have key-based auth disabled.
DeepEval should NOT fail fast when api_key / azure_ad_token / provider are all unset.
It should defer auth validation to the OpenAI SDK.
"""
# Ensure Settings has the non-auth Azure config required for client construction
with settings.edit(persist=False):
settings.AZURE_OPENAI_API_KEY = None # critical: no key
settings.AZURE_OPENAI_AD_TOKEN = None # critical: no token (if present in settings)
settings.AZURE_OPENAI_ENDPOINT = "https://azure.example.com"
settings.AZURE_DEPLOYMENT_NAME = "settings-deployment"
settings.AZURE_MODEL_NAME = "gpt-4.1"
settings.OPENAI_API_VERSION = "2024-02-15-preview"
settings.OPENAI_COST_PER_INPUT_TOKEN = 1e-6
settings.OPENAI_COST_PER_OUTPUT_TOKEN = 1e-6

reset_settings(reload_dotenv=False)

# Stub SDK clients so no real network calls happen
monkeypatch.setattr(azure_mod, "AzureOpenAI", _RecordingClient, raising=True)
monkeypatch.setattr(azure_mod, "AsyncAzureOpenAI", _RecordingClient, raising=True)

# This should NOT raise DeepEvalError anymore (it should defer to SDK)
model = AzureOpenAIModel()

client = model.model
kw = client.kwargs

# We expect credentials to be None (SDK can attempt keyless auth internally)
assert kw.get("api_key") is None
assert kw.get("azure_ad_token") is None
assert kw.get("azure_ad_token_provider") is None


@pytest.mark.parametrize("bad_key", ["", " ", "\n\t"])
def test_azure_openai_model_raises_on_explicit_empty_api_key(monkeypatch, settings, bad_key):
"""
If the user explicitly provides api_key but it's empty/whitespace,
DeepEval should fail fast with a helpful error message.
"""
with settings.edit(persist=False):
settings.AZURE_OPENAI_API_KEY = None
settings.AZURE_OPENAI_ENDPOINT = "https://azure.example.com"
settings.AZURE_DEPLOYMENT_NAME = "settings-deployment"
settings.AZURE_MODEL_NAME = "gpt-4.1"
settings.OPENAI_API_VERSION = "2024-02-15-preview"
settings.OPENAI_COST_PER_INPUT_TOKEN = 1e-6
settings.OPENAI_COST_PER_OUTPUT_TOKEN = 1e-6

reset_settings(reload_dotenv=False)

monkeypatch.setattr(azure_mod, "AzureOpenAI", _RecordingClient, raising=True)
monkeypatch.setattr(azure_mod, "AsyncAzureOpenAI", _RecordingClient, raising=True)

with pytest.raises(Exception) as e:
_ = AzureOpenAIModel(api_key=bad_key)

# match your new error text (keeps test stable & intentional)
assert "api_key was provided but is empty" in str(e.value)


def test_azure_openai_model_raises_on_explicit_empty_api_key_secretstr(monkeypatch, settings):
"""
Same as above, but ensures SecretStr is unwrapped via get_secret_value().
This directly validates the new SecretStr handling logic.
"""
with settings.edit(persist=False):
settings.AZURE_OPENAI_API_KEY = None
settings.AZURE_OPENAI_ENDPOINT = "https://azure.example.com"
settings.AZURE_DEPLOYMENT_NAME = "settings-deployment"
settings.AZURE_MODEL_NAME = "gpt-4.1"
settings.OPENAI_API_VERSION = "2024-02-15-preview"
settings.OPENAI_COST_PER_INPUT_TOKEN = 1e-6
settings.OPENAI_COST_PER_OUTPUT_TOKEN = 1e-6

reset_settings(reload_dotenv=False)

monkeypatch.setattr(azure_mod, "AzureOpenAI", _RecordingClient, raising=True)
monkeypatch.setattr(azure_mod, "AsyncAzureOpenAI", _RecordingClient, raising=True)

with pytest.raises(Exception) as e:
_ = AzureOpenAIModel(api_key=SecretStr(" "))

assert "api_key was provided but is empty" in str(e.value)


@pytest.mark.parametrize("bad_token", ["", " ", "\n\t"])
def test_azure_openai_model_raises_on_explicit_empty_ad_token(monkeypatch, settings, bad_token):
"""
If the user explicitly provides azure_ad_token but it's empty/whitespace,
DeepEval should fail fast with a helpful error message.
"""
with settings.edit(persist=False):
settings.AZURE_OPENAI_API_KEY = None
settings.AZURE_OPENAI_ENDPOINT = "https://azure.example.com"
settings.AZURE_DEPLOYMENT_NAME = "settings-deployment"
settings.AZURE_MODEL_NAME = "gpt-4.1"
settings.OPENAI_API_VERSION = "2024-02-15-preview"
settings.OPENAI_COST_PER_INPUT_TOKEN = 1e-6
settings.OPENAI_COST_PER_OUTPUT_TOKEN = 1e-6

reset_settings(reload_dotenv=False)

monkeypatch.setattr(azure_mod, "AzureOpenAI", _RecordingClient, raising=True)
monkeypatch.setattr(azure_mod, "AsyncAzureOpenAI", _RecordingClient, raising=True)

with pytest.raises(Exception) as e:
_ = AzureOpenAIModel(azure_ad_token=bad_token)

assert "azure_ad_token was provided but is empty" in str(e.value)


def test_azure_openai_model_does_not_fail_fast_when_token_provider_present(monkeypatch, settings):
"""
If a token provider is supplied, we should not block early on missing key/token.
Provider controls auth.
"""
with settings.edit(persist=False):
settings.AZURE_OPENAI_API_KEY = None
settings.AZURE_OPENAI_ENDPOINT = "https://azure.example.com"
settings.AZURE_DEPLOYMENT_NAME = "settings-deployment"
settings.AZURE_MODEL_NAME = "gpt-4.1"
settings.OPENAI_API_VERSION = "2024-02-15-preview"
settings.OPENAI_COST_PER_INPUT_TOKEN = 1e-6
settings.OPENAI_COST_PER_OUTPUT_TOKEN = 1e-6

reset_settings(reload_dotenv=False)

monkeypatch.setattr(azure_mod, "AzureOpenAI", _RecordingClient, raising=True)
monkeypatch.setattr(azure_mod, "AsyncAzureOpenAI", _RecordingClient, raising=True)

def provider():
return "token"

model = AzureOpenAIModel(azure_ad_token_provider=provider)

client = model.model
kw = client.kwargs
assert kw.get("azure_ad_token_provider") is provider

def test_azure_openai_model_uses_explicit_key_over_settings_and_strips_secret(
monkeypatch, settings
Expand Down
Loading