Skip to content

Commit c707cea

Browse files
w-javedCopilot
w-javed
and
Copilot
authored
update doc string (Azure#40962)
* update doc string * adding more * adding more * adding more * Update sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_intent_resolution/_intent_resolution.py Co-authored-by: Copilot <[email protected]> * Update sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_fluency/_fluency.py Co-authored-by: Copilot <[email protected]> * added sample url * updating readme --------- Co-authored-by: Copilot <[email protected]>
1 parent e453b20 commit c707cea

File tree

35 files changed

+860
-40
lines changed

35 files changed

+860
-40
lines changed

‎sdk/evaluation/azure-ai-evaluation/README.md

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,13 +76,23 @@ result = relevance_evaluator(
7676
response="The capital of Japan is Tokyo."
7777
)
7878

79-
# AI assisted safety evaluator
79+
# There are two ways to provide Azure AI Project.
80+
# Option #1 : Using Azure AI Project Details
8081
azure_ai_project = {
8182
"subscription_id": "<subscription_id>",
8283
"resource_group_name": "<resource_group_name>",
8384
"project_name": "<project_name>",
8485
}
8586

87+
violence_evaluator = ViolenceEvaluator(azure_ai_project)
88+
result = violence_evaluator(
89+
query="What is the capital of France?",
90+
response="Paris."
91+
)
92+
93+
# Option # 2 : Using Azure AI Project Url
94+
azure_ai_project = "https://{resource_name}.services.ai.azure.com/api/projects/{project_name}"
95+
8696
violence_evaluator = ViolenceEvaluator(azure_ai_project)
8797
result = violence_evaluator(
8898
query="What is the capital of France?",
@@ -233,11 +243,18 @@ with open("simulator_output.jsonl", "w") as f:
233243
```python
234244
from azure.ai.evaluation.simulator import AdversarialSimulator, AdversarialScenario
235245
from azure.identity import DefaultAzureCredential
246+
247+
# There are two ways to provide Azure AI Project.
248+
# Option #1 : Using Azure AI Project
236249
azure_ai_project = {
237250
"subscription_id": <subscription_id>,
238251
"resource_group_name": <resource_group_name>,
239252
"project_name": <project_name>
240253
}
254+
255+
# Option #2 : Using Azure AI Project Url
256+
azure_ai_project = "https://{resource_name}.services.ai.azure.com/api/projects/{project_name}"
257+
241258
scenario = AdversarialScenario.ADVERSARIAL_QA
242259
simulator = AdversarialSimulator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())
243260

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/rai_service.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -629,8 +629,9 @@ async def evaluate_with_rai_service(
629629
:type data: dict
630630
:param metric_name: The evaluation metric to use.
631631
:type metric_name: str
632-
:param project_scope: The Azure AI project scope details.
633-
:type project_scope: Dict
632+
:param project_scope: The Azure AI project, which can either be a string representing the project endpoint
633+
or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
634+
:type project_scope: Union[str, AzureAIProject]
634635
:param credential: The Azure authentication credential.
635636
:type credential: ~azure.core.credentials.TokenCredential
636637
:param annotation_task: The annotation task to use.
@@ -777,11 +778,11 @@ async def evaluate_with_rai_service_multimodal(
777778
:type messages: str
778779
:param metric_name: The evaluation metric to use.
779780
:type metric_name: str
780-
:param project_scope: The Azure AI project scope details.
781-
:type project_scope: Dict
781+
:param project_scope: The Azure AI project, which can either be a string representing the project endpoint
782+
or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
783+
:type project_scope: Union[str, AzureAIProject]
782784
:param credential: The Azure authentication credential.
783-
:type credential:
784-
~azure.core.credentials.TokenCredential
785+
:type credential: ~azure.core.credentials.TokenCredential
785786
:return: The parsed annotation result.
786787
:rtype: List[List[Dict]]
787788
"""

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -739,7 +739,17 @@ def evaluate(
739739
:end-before: [END evaluate_method]
740740
:language: python
741741
:dedent: 8
742-
:caption: Run an evaluation on local data with Coherence and Relevance evaluators.
742+
:caption: Run an evaluation on local data with one or more evaluators using azure.ai.evaluation.AzureAIProject
743+
744+
.. admonition:: Example using Azure AI Project URL:
745+
746+
.. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
747+
:start-after: [START evaluate_method]
748+
:end-before: [END evaluate_method]
749+
:language: python
750+
:dedent: 8
751+
:caption: Run an evaluation on local data with one or more evaluators using Azure AI Project URL in following format
752+
https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
743753
"""
744754
try:
745755
return _evaluate(

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_bleu/_bleu.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,17 @@ class BleuScoreEvaluator(EvaluatorBase):
3333
:end-before: [END bleu_score_evaluator]
3434
:language: python
3535
:dedent: 8
36-
:caption: Initialize and call an BleuScoreEvaluator.
36+
:caption: Initialize and call an BleuScoreEvaluator using azure.ai.evaluation.AzureAIProject
37+
38+
.. admonition:: Example using Azure AI Project URL:
39+
40+
.. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
41+
:start-after: [START bleu_score_evaluator]
42+
:end-before: [END bleu_score_evaluator]
43+
:language: python
44+
:dedent: 8
45+
:caption: Initialize and call an BleuScoreEvaluator using Azure AI Project URL in following format
46+
https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
3747
3848
.. admonition:: Example with Threshold:
3949
.. literalinclude:: ../samples/evaluation_samples_threshold.py

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_code_vulnerability/_code_vulnerability.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,15 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
6262
:end-before: [END code_vulnerability_evaluator]
6363
:language: python
6464
:dedent: 8
65-
:caption: Initialize and call a CodeVulnerabilityEvaluator with a query and response.
65+
:caption: Initialize and call CodeVulnerabilityEvaluator with a query and response using azure.ai.evaluation.AzureAIProject.
66+
67+
.. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
68+
:start-after: [START code_vulnerability_evaluator]
69+
:end-before: [END code_vulnerability_evaluator]
70+
:language: python
71+
:dedent: 8
72+
:caption: Initialize and call CodeVulnerabilityEvaluator using Azure AI Project URL in following format
73+
https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
6674
6775
.. note::
6876

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_coherence/_coherence.py

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,17 @@ class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
3131
:end-before: [END coherence_evaluator]
3232
:language: python
3333
:dedent: 8
34-
:caption: Initialize and call a CoherenceEvaluator with a query and response.
34+
:caption: Initialize and call CoherenceEvaluator using azure.ai.evaluation.AzureAIProject
35+
36+
.. admonition:: Example using Azure AI Project URL:
37+
38+
.. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
39+
:start-after: [START coherence_evaluator]
40+
:end-before: [END coherence_evaluator]
41+
:language: python
42+
:dedent: 8
43+
:caption: Initialize and call CoherenceEvaluator using Azure AI Project URL in following format
44+
https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
3545
3646
.. admonition:: Example with Threshold:
3747
@@ -40,7 +50,7 @@ class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
4050
:end-before: [END threshold_coherence_evaluator]
4151
:language: python
4252
:dedent: 8
43-
:caption: Initialize with threshold and and call a CoherenceEvaluator with a query and response.
53+
:caption: Initialize with threshold and call a CoherenceEvaluator with a query and response.
4454
4555
.. note::
4656

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,17 @@ class ContentSafetyEvaluator(MultiEvaluatorBase[Union[str, float]]):
4444
:end-before: [END content_safety_evaluator]
4545
:language: python
4646
:dedent: 8
47-
:caption: Initialize and call a ContentSafetyEvaluator.
47+
:caption: Initialize and call ContentSafetyEvaluator using azure.ai.evaluation.AzureAIProject.
48+
49+
.. admonition:: Example using Azure AI Project URL:
50+
51+
.. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
52+
:start-after: [START content_safety_evaluator]
53+
:end-before: [END content_safety_evaluator]
54+
:language: python
55+
:dedent: 8
56+
:caption: Initialize and call ContentSafetyEvaluator using Azure AI Project URL in the following format
57+
https://{resource_name}.services.ai.azure.com/api/projects/{project_name}.
4858
4959
.. admonition:: Example with Threshold:
5060
@@ -53,7 +63,7 @@ class ContentSafetyEvaluator(MultiEvaluatorBase[Union[str, float]]):
5363
:end-before: [END threshold_content_safety_evaluator]
5464
:language: python
5565
:dedent: 8
56-
:caption: Initialize with threshold and call a ContentSafetyEvaluator.
66+
:caption: Initialize with threshold and call a ContentSafetyEvaluator with a query and response.
5767
"""
5868

5969
id = "content_safety"

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,16 +58,26 @@ class HateUnfairnessEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
5858
:end-before: [END hate_unfairness_evaluator]
5959
:language: python
6060
:dedent: 8
61-
:caption: Initialize and call a HateUnfairnessEvaluator.
62-
63-
.. admonition:: Example with Threshold:
61+
:caption: Initialize and call HateUnfairnessEvaluator using azure.ai.evaluation.AzureAIProject.
62+
63+
.. admonition:: Example using Azure AI Project URL:
64+
65+
.. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
66+
:start-after: [START hate_unfairness_evaluator]
67+
:end-before: [END hate_unfairness_evaluator]
68+
:language: python
69+
:dedent: 8
70+
:caption: Initialize and call HateUnfairnessEvaluator using Azure AI Project URL in the following format
71+
https://{resource_name}.services.ai.azure.com/api/projects/{project_name}.
6472
73+
.. admonition:: Example with Threshold:
74+
6575
.. literalinclude:: ../samples/evaluation_samples_threshold.py
6676
:start-after: [START threshold_hate_unfairness_evaluator]
6777
:end-before: [END threshold_hate_unfairness_evaluator]
6878
:language: python
6979
:dedent: 8
70-
:caption: Initialize with threshold and call a HateUnfairnessEvaluator.
80+
:caption: Initialize with threshold and call a HateUnfairnessEvaluator with a query and response.
7181
"""
7282

7383
id = "azureml://registries/azureml/models/Hate-and-Unfairness-Evaluator/versions/4"

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,16 +52,17 @@ class SelfHarmEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
5252
:end-before: [END self_harm_evaluator]
5353
:language: python
5454
:dedent: 8
55-
:caption: Initialize and call a SelfHarmEvaluator.
56-
57-
.. admonition:: Example:
58-
59-
.. literalinclude:: ../samples/evaluation_samples_threshold.py
60-
:start-after: [START threshold_self_harm_evaluator]
61-
:end-before: [END threshold_self_harm_evaluator]
55+
:caption: Initialize and call SelfHarmEvaluator using azure.ai.evaluation.AzureAIProject.
56+
57+
.. admonition:: Example using Azure AI Project URL:
58+
59+
.. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
60+
:start-after: [START self_harm_evaluator]
61+
:end-before: [END self_harm_evaluator]
6262
:language: python
6363
:dedent: 8
64-
:caption: Initialize with threshold and call a SelfHarmEvaluator.
64+
:caption: Initialize and call SelfHarmEvaluator using Azure AI Project URL in the following format
65+
https://{resource_name}.services.ai.azure.com/api/projects/{project_name}.
6566
"""
6667

6768
id = "azureml://registries/azureml/models/Self-Harm-Related-Content-Evaluator/versions/3"

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,16 @@ class SexualEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
5656
:dedent: 8
5757
:caption: Initialize and call a SexualEvaluator.
5858
59+
.. admonition:: Example using Azure AI Project URL:
60+
61+
.. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
62+
:start-after: [START sexual_evaluator]
63+
:end-before: [END sexual_evaluator]
64+
:language: python
65+
:dedent: 8
66+
:caption: Initialize and call SexualEvaluator using Azure AI Project URL in following format
67+
https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
68+
5969
.. admonition:: Example with Threshold:
6070
6171
.. literalinclude:: ../samples/evaluation_samples_threshold.py

0 commit comments

Comments
 (0)