Skip to content

Commit 75502e2

Browse files
nagkumar91Nagkumar Arkalgud
andauthored
readme code updates and minor bugfixes (Azure#37505)
* readme code updates * Change __file__ to os.cwd() * Fix typo * Adding prompty to setup and access as a package resource * Add experimental message * Remove additonal expermiental call --------- Co-authored-by: Nagkumar Arkalgud <[email protected]>
1 parent b24fae7 commit 75502e2

File tree

6 files changed

+187
-14
lines changed

6 files changed

+187
-14
lines changed

sdk/evaluation/azure-ai-evaluation/README.md

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,9 @@ if __name__ == "__main__":
5757

5858
# Initialize Project Scope
5959
azure_ai_project = {
60-
"subscription_id": "e0fd569c-e34a-4249-8c24-e8d723c7f054",
61-
"resource_group_name": "rg-test",
62-
"project_name": "project-test",
60+
"subscription_id": <subscription_id>,
61+
"resource_group_name": <resource_group_name>,
62+
"project_name": <project_name>
6363
}
6464

6565
violence_eval = ViolenceEvaluator(azure_ai_project)
@@ -128,16 +128,15 @@ Application code:
128128
import json
129129
import asyncio
130130
from typing import Any, Dict, List, Optional
131-
from azure.ai.evaluation.synthetic import Simulator
131+
from azure.ai.evaluation.simulator import Simulator
132132
from promptflow.client import load_flow
133133
from azure.identity import DefaultAzureCredential
134134
import os
135135

136136
azure_ai_project = {
137137
"subscription_id": os.environ.get("AZURE_SUBSCRIPTION_ID"),
138138
"resource_group_name": os.environ.get("RESOURCE_GROUP"),
139-
"project_name": os.environ.get("PROJECT_NAME"),
140-
"credential": DefaultAzureCredential(),
139+
"project_name": os.environ.get("PROJECT_NAME")
141140
}
142141

143142
import wikipedia
@@ -283,6 +282,8 @@ async def callback(
283282
}
284283

285284
```
285+
## Adversarial Simulator
286+
286287
### Adversarial QA:
287288
```python
288289
scenario = AdversarialScenario.ADVERSARIAL_QA
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
from ._simulator_data_classes import ConversationHistory, Turn
22
from ._language_suffix_mapping import SUPPORTED_LANGUAGES_MAPPING
3+
from ._experimental import experimental
34

4-
__all__ = ["ConversationHistory", "Turn", "SUPPORTED_LANGUAGES_MAPPING"]
5+
__all__ = ["ConversationHistory", "Turn", "SUPPORTED_LANGUAGES_MAPPING", "experimental"]
Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
# ---------------------------------------------------------
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# ---------------------------------------------------------
4+
5+
import functools
6+
import inspect
7+
import logging
8+
import sys
9+
from typing import Callable, Type, TypeVar, Union
10+
11+
from typing_extensions import ParamSpec
12+
13+
DOCSTRING_TEMPLATE = ".. note:: {0} {1}\n\n"
14+
DOCSTRING_DEFAULT_INDENTATION = 8
15+
EXPERIMENTAL_CLASS_MESSAGE = "This is an experimental class,"
16+
EXPERIMENTAL_METHOD_MESSAGE = "This is an experimental method,"
17+
EXPERIMENTAL_FIELD_MESSAGE = "This is an experimental field,"
18+
EXPERIMENTAL_LINK_MESSAGE = (
19+
"and may change at any time. Please see https://aka.ms/azuremlexperimental for more information."
20+
)
21+
22+
_warning_cache = set()
23+
module_logger = logging.getLogger(__name__)
24+
25+
TExperimental = TypeVar("TExperimental", bound=Union[Type, Callable])
26+
P = ParamSpec("P")
27+
T = TypeVar("T")
28+
29+
30+
def experimental(wrapped: TExperimental) -> TExperimental:
31+
"""Add experimental tag to a class or a method.
32+
33+
:param wrapped: Either a Class or Function to mark as experimental
34+
:type wrapped: TExperimental
35+
:return: The wrapped class or method
36+
:rtype: TExperimental
37+
"""
38+
if inspect.isclass(wrapped):
39+
return _add_class_docstring(wrapped)
40+
if inspect.isfunction(wrapped):
41+
return _add_method_docstring(wrapped)
42+
return wrapped
43+
44+
45+
def _add_class_docstring(cls: Type[T]) -> Type[T]:
46+
"""Add experimental tag to the class doc string.
47+
48+
:return: The updated class
49+
:rtype: Type[T]
50+
"""
51+
52+
P2 = ParamSpec("P2")
53+
54+
def _add_class_warning(func: Callable[P2, None]) -> Callable[P2, None]:
55+
"""Add warning message for class __init__.
56+
57+
:param func: The original __init__ function
58+
:type func: Callable[P2, None]
59+
:return: Updated __init__
60+
:rtype: Callable[P2, None]
61+
"""
62+
63+
@functools.wraps(func)
64+
def wrapped(*args, **kwargs):
65+
message = "Class {0}: {1} {2}".format(cls.__name__, EXPERIMENTAL_CLASS_MESSAGE, EXPERIMENTAL_LINK_MESSAGE)
66+
if not _should_skip_warning() and not _is_warning_cached(message):
67+
module_logger.warning(message)
68+
return func(*args, **kwargs)
69+
70+
return wrapped
71+
72+
doc_string = DOCSTRING_TEMPLATE.format(EXPERIMENTAL_CLASS_MESSAGE, EXPERIMENTAL_LINK_MESSAGE)
73+
if cls.__doc__:
74+
cls.__doc__ = _add_note_to_docstring(cls.__doc__, doc_string)
75+
else:
76+
cls.__doc__ = doc_string + ">"
77+
cls.__init__ = _add_class_warning(cls.__init__)
78+
return cls
79+
80+
81+
def _add_method_docstring(func: Callable[P, T] = None) -> Callable[P, T]:
82+
"""Add experimental tag to the method doc string.
83+
84+
:param func: The function to update
85+
:type func: Callable[P, T]
86+
:return: A wrapped method marked as experimental
87+
:rtype: Callable[P,T]
88+
"""
89+
doc_string = DOCSTRING_TEMPLATE.format(EXPERIMENTAL_METHOD_MESSAGE, EXPERIMENTAL_LINK_MESSAGE)
90+
if func.__doc__:
91+
func.__doc__ = _add_note_to_docstring(func.__doc__, doc_string)
92+
else:
93+
# '>' is required. Otherwise the note section can't be generated
94+
func.__doc__ = doc_string + ">"
95+
96+
@functools.wraps(func)
97+
def wrapped(*args: P.args, **kwargs: P.kwargs) -> T:
98+
message = "Method {0}: {1} {2}".format(func.__name__, EXPERIMENTAL_METHOD_MESSAGE, EXPERIMENTAL_LINK_MESSAGE)
99+
if not _should_skip_warning() and not _is_warning_cached(message):
100+
module_logger.warning(message)
101+
return func(*args, **kwargs)
102+
103+
return wrapped
104+
105+
106+
def _add_note_to_docstring(doc_string: str, note: str) -> str:
107+
"""Adds experimental note to docstring at the top and correctly indents original docstring.
108+
109+
:param doc_string: The docstring
110+
:type doc_string: str
111+
:param note: The note to add to the docstring
112+
:type note: str
113+
:return: Updated docstring
114+
:rtype: str
115+
"""
116+
indent = _get_indentation_size(doc_string)
117+
doc_string = doc_string.rjust(len(doc_string) + indent)
118+
return note + doc_string
119+
120+
121+
def _get_indentation_size(doc_string: str) -> int:
122+
"""Finds the minimum indentation of all non-blank lines after the first line.
123+
124+
:param doc_string: The docstring
125+
:type doc_string: str
126+
:return: Minimum number of indentation of the docstring
127+
:rtype: int
128+
"""
129+
lines = doc_string.expandtabs().splitlines()
130+
indent = sys.maxsize
131+
for line in lines[1:]:
132+
stripped = line.lstrip()
133+
if stripped:
134+
indent = min(indent, len(line) - len(stripped))
135+
return indent if indent < sys.maxsize else DOCSTRING_DEFAULT_INDENTATION
136+
137+
138+
def _should_skip_warning():
139+
skip_warning_msg = False
140+
141+
# Cases where we want to suppress the warning:
142+
# 1. When converting from REST object to SDK object
143+
for frame in inspect.stack():
144+
if frame.function == "_from_rest_object":
145+
skip_warning_msg = True
146+
break
147+
148+
return skip_warning_msg
149+
150+
151+
def _is_warning_cached(warning_msg):
152+
# use cache to make sure we only print same warning message once under same session
153+
# this prevents duplicated warnings got printed when user does a loop call on a method or a class
154+
if warning_msg in _warning_cache:
155+
return True
156+
_warning_cache.add(warning_msg)
157+
return False

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_prompty/__init__.py

Whitespace-only changes.

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/simulator.py

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,17 @@
1414

1515
from promptflow.client import load_flow
1616
from promptflow.core import AzureOpenAIModelConfiguration
17+
import importlib.resources as pkg_resources
18+
from pathlib import Path
1719

1820
from .._user_agent import USER_AGENT
1921
from ._conversation.constants import ConversationRole
20-
from ._helpers import ConversationHistory, Turn
22+
from ._helpers import ConversationHistory, Turn, experimental
2123
# from ._tracing import monitor_task_simulator
2224
from ._utils import JsonLineChatProtocol
2325

2426

27+
@experimental
2528
class Simulator:
2629
"""
2730
Simulator for generating synthetic conversations.
@@ -287,9 +290,14 @@ def _load_user_simulation_flow(
287290
:return: The loaded flow for simulating user interactions.
288291
"""
289292
if not user_simulator_prompty:
290-
current_dir = os.path.dirname(__file__)
291-
prompty_path = os.path.join(current_dir, "_prompty", "task_simulate.prompty")
292-
return load_flow(source=prompty_path, model=prompty_model_config)
293+
package = 'azure.ai.evaluation.simulator._prompty'
294+
resource_name = 'task_simulate.prompty'
295+
try:
296+
# Access the resource as a file path
297+
with pkg_resources.path(package, resource_name) as prompty_path:
298+
return load_flow(source=str(prompty_path), model=prompty_model_config)
299+
except FileNotFoundError:
300+
raise f"Flow path for {resource_name} does not exist in package {package}."
293301
return load_flow(
294302
source=user_simulator_prompty,
295303
model=prompty_model_config,
@@ -383,9 +391,14 @@ def _load_query_generation_flow(
383391
:return: The loaded flow for generating query responses.
384392
"""
385393
if not query_response_generating_prompty:
386-
current_dir = os.path.dirname(__file__)
387-
prompty_path = os.path.join(current_dir, "_prompty", "task_query_response.prompty")
388-
return load_flow(source=prompty_path, model=prompty_model_config)
394+
package = 'azure.ai.evaluation.simulator._prompty'
395+
resource_name = 'task_query_response.prompty'
396+
try:
397+
# Access the resource as a file path
398+
with pkg_resources.path(package, resource_name) as prompty_path:
399+
return load_flow(source=str(prompty_path), model=prompty_model_config)
400+
except FileNotFoundError:
401+
raise f"Flow path for {resource_name} does not exist in package {package}."
389402
return load_flow(
390403
source=query_response_generating_prompty,
391404
model=prompty_model_config,

sdk/evaluation/azure-ai-evaluation/setup.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,5 +86,6 @@
8686
},
8787
package_data={
8888
"pytyped": ["py.typed"],
89+
"azure.ai.evaluation.simulator._prompty": ["*.prompty"],
8990
},
9091
)

0 commit comments

Comments
 (0)