Skip to content

Commit 62e47e6

Browse files
authored
Groundedness - Correct prompty in runtime (fixes using evaluate function) (#43357)
* check for correct prompty in kwargs * dummy commit * Revert "dummy commit" This reverts commit 372c448.
1 parent ef96495 commit 62e47e6

File tree

1 file changed

+18
-9
lines changed
  • sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness

1 file changed

+18
-9
lines changed

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -202,18 +202,24 @@ def __call__( # pylint: disable=docstring-missing-param
202202
"""
203203

204204
if kwargs.get("query", None):
205-
current_dir = os.path.dirname(__file__)
206-
prompty_path = os.path.join(current_dir, self._PROMPTY_FILE_WITH_QUERY)
207-
self._prompty_file = prompty_path
208-
prompty_model_config = construct_prompty_model_config(
209-
validate_model_config(self._model_config),
210-
self._DEFAULT_OPEN_API_VERSION,
211-
UserAgentSingleton().value,
212-
)
213-
self._flow = AsyncPrompty.load(source=self._prompty_file, model=prompty_model_config)
205+
self._ensure_query_prompty_loaded()
214206

215207
return super().__call__(*args, **kwargs)
216208

209+
def _ensure_query_prompty_loaded(self):
210+
"""Switch to the query prompty file if not already loaded."""
211+
212+
current_dir = os.path.dirname(__file__)
213+
prompty_path = os.path.join(current_dir, self._PROMPTY_FILE_WITH_QUERY)
214+
215+
self._prompty_file = prompty_path
216+
prompty_model_config = construct_prompty_model_config(
217+
validate_model_config(self._model_config),
218+
self._DEFAULT_OPEN_API_VERSION,
219+
UserAgentSingleton().value,
220+
)
221+
self._flow = AsyncPrompty.load(source=self._prompty_file, model=prompty_model_config)
222+
217223
def _has_context(self, eval_input: dict) -> bool:
218224
"""
219225
Return True if eval_input contains a non-empty 'context' field.
@@ -279,6 +285,9 @@ def _convert_kwargs_to_eval_input(self, **kwargs):
279285
response = kwargs.get("response")
280286
tool_definitions = kwargs.get("tool_definitions")
281287

288+
if query and self._prompty_file != self._PROMPTY_FILE_WITH_QUERY:
289+
self._ensure_query_prompty_loaded()
290+
282291
if (not query) or (not response): # or not tool_definitions:
283292
msg = f"{type(self).__name__}: Either 'conversation' or individual inputs must be provided. For Agent groundedness 'query' and 'response' are required."
284293
raise EvaluationException(

0 commit comments

Comments
 (0)