Fix MyPy, PyRight, and Black issues in Evaluations Samples (#43981)

* Fix mypy issues

* Fix pyright issues: wrong argument name

* Run Black

* Add missing required in coherence and fluency

* Remove ignore type for DataSourceConfigCustom

* Fix DataSourceConfigCustom type ignores

* black

Author: m7md7sien

sdk/ai/azure-ai-projects/pyproject.toml

@@ -63,9 +63,3 @@ pytyped = ["py.typed"]
 
 [tool.azure-sdk-build]
 verifytypes = false
-
-[tool.mypy]
-exclude = [
-    "samples/evaluations/.*"
-]
-

sdk/ai/azure-ai-projects/pyrightconfig.json

@@ -3,7 +3,6 @@
   "reportMissingImports": false,
   "pythonVersion": "3.11",
   "exclude": [
-    "**/samples/evaluations/**"
   ],
   "extraPaths": [
     "./../../core/azure-core",

sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_coherence.py

@@ -36,6 +36,7 @@
     SourceFileContent,
     SourceFileContentContent,
 )
+from openai.types.eval_create_params import DataSourceConfigCustom
 
 
 load_dotenv()
@@ -53,15 +54,17 @@ def main() -> None:
 
             client = project_client.get_openai_client()
 
-            data_source_config = {
-                "type": "custom",
-                "item_schema": {
-                    "type": "object",
-                    "properties": {"query": {"type": "string"}, "response": {"type": "string"}},
-                    "required": [],
-                },
-                "include_sample_schema": True,
-            }
+            data_source_config = DataSourceConfigCustom(
+                {
+                    "type": "custom",
+                    "item_schema": {
+                        "type": "object",
+                        "properties": {"query": {"type": "string"}, "response": {"type": "string"}},
+                        "required": ["query", "response"],
+                    },
+                    "include_sample_schema": True,
+                }
+            )
 
             testing_criteria = [
                 {
@@ -76,8 +79,8 @@ def main() -> None:
             print("Creating Eval Group")
             eval_object = client.evals.create(
                 name="Test Coherence Evaluator with inline data",
-                data_source_config=data_source_config, # type: ignore
-                testing_criteria=testing_criteria, # type: ignore
+                data_source_config=data_source_config,
+                testing_criteria=testing_criteria,  # type: ignore
             )
             print(f"Eval Group created")
 

sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_fluency.py

@@ -36,6 +36,7 @@
     SourceFileContent,
     SourceFileContentContent,
 )
+from openai.types.eval_create_params import DataSourceConfigCustom
 
 
 load_dotenv()
@@ -53,15 +54,17 @@ def main() -> None:
 
             client = project_client.get_openai_client()
 
-            data_source_config = {
-                "type": "custom",
-                "item_schema": {
-                    "type": "object",
-                    "properties": {"query": {"type": "string"}, "response": {"type": "string"}},
-                    "required": [],
-                },
-                "include_sample_schema": True,
-            }
+            data_source_config = DataSourceConfigCustom(
+                {
+                    "type": "custom",
+                    "item_schema": {
+                        "type": "object",
+                        "properties": {"query": {"type": "string"}, "response": {"type": "string"}},
+                        "required": ["response"],
+                    },
+                    "include_sample_schema": True,
+                }
+            )
 
             testing_criteria = [
                 {
@@ -76,8 +79,8 @@ def main() -> None:
             print("Creating Eval Group")
             eval_object = client.evals.create(
                 name="Test Fluency Evaluator with inline data",
-                data_source_config=data_source_config, # type: ignore
-                testing_criteria=testing_criteria, # type: ignore
+                data_source_config=data_source_config,
+                testing_criteria=testing_criteria,  # type: ignore
             )
             print(f"Eval Group created")
 

sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_generic_agentic_evaluator/agent_utils.py

@@ -5,11 +5,9 @@
 # ------------------------------------
 
 from dotenv import load_dotenv
-import json
 import os
 import time
 from pprint import pprint
-from samples.evaluation.sample_agentic_evaluators.sample_generic_agentic_evaluator.agent_utils import pprint
 
 from azure.identity import DefaultAzureCredential
 from azure.ai.projects import AIProjectClient
@@ -18,6 +16,7 @@
     SourceFileContent,
     SourceFileContentContent,
 )
+from openai.types.eval_create_params import DataSourceConfigCustom
 
 
 load_dotenv()
@@ -26,7 +25,7 @@
 def run_evaluator(
     evaluator_name: str,
     evaluation_contents: list[SourceFileContentContent],
-    data_source_config: dict,
+    data_source_config: DataSourceConfigCustom,
     initialization_parameters: dict[str, str],
     data_mapping: dict[str, str],
 ) -> None:
@@ -53,8 +52,8 @@ def run_evaluator(
             print("Creating Eval Group")
             eval_object = client.evals.create(
                 name=f"Test {evaluator_name} Evaluator with inline data",
-                data_source_config=data_source_config, # type: ignore
-                testing_criteria=testing_criteria, # type: ignore
+                data_source_config=data_source_config,
+                testing_criteria=testing_criteria,  # type: ignore
             )
             print(f"Eval Group created")
 

sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_generic_agentic_evaluator/sample_generic_agentic_evaluator.py

@@ -25,7 +25,7 @@
 
 from dotenv import load_dotenv
 import os
-from samples.evaluation.sample_agentic_evaluators.sample_generic_agentic_evaluator.agent_utils import run_evaluator
+from agent_utils import run_evaluator
 from schema_mappings import evaluator_to_data_source_config, evaluator_to_data_mapping
 from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent