From 40ca32c2c199e4d3eb08d2a8fdc0a8cc93b0cc33 Mon Sep 17 00:00:00 2001 From: Sydney Date: Thu, 26 Sep 2024 16:56:52 -0700 Subject: [PATCH 01/10] update promptflow-eval dependencies to azure-ai-evaluation --- scenarios/evaluate-app-endpoint/data.jsonl | 6 +- .../evaluate-target.ipynb | 49 +- .../evaluate-model-endpoints/app_target.py | 18 +- .../evaluate-models-target.ipynb | 469 ++++++++++++++++-- .../simulate_and_evaluate_ask_wiki.ipynb | 79 ++- ...imulate_and_evaluate_online_endpoint.ipynb | 53 +- 6 files changed, 521 insertions(+), 153 deletions(-) diff --git a/scenarios/evaluate-app-endpoint/data.jsonl b/scenarios/evaluate-app-endpoint/data.jsonl index 47d5a9bf..b6ae357f 100644 --- a/scenarios/evaluate-app-endpoint/data.jsonl +++ b/scenarios/evaluate-app-endpoint/data.jsonl @@ -1,3 +1,3 @@ -{"question":"When was United Stated found ?", "ground_truth":"1776"} -{"question":"What is the capital of France?", "ground_truth":"Paris"} -{"question":"Who is the best tennis player of all time ?", "ground_truth":"Roger Federer"} \ No newline at end of file +{"query":"When was United Stated found ?", "response":"1776", "question":"When was United Stated found ?"} +{"query":"What is the capital of France?", "response":"Paris", "question":"What is the capital of France?"} +{"query":"Who is the best tennis player of all time ?", "response":"Roger Federer", "question": "Who is the best tennis player of all time ?"} \ No newline at end of file diff --git a/scenarios/evaluate-app-endpoint/evaluate-target.ipynb b/scenarios/evaluate-app-endpoint/evaluate-target.ipynb index 36574089..796bb70b 100644 --- a/scenarios/evaluate-app-endpoint/evaluate-target.ipynb +++ b/scenarios/evaluate-app-endpoint/evaluate-target.ipynb @@ -20,7 +20,7 @@ "\n", "This tutorial uses the following Azure AI services:\n", "\n", - "- [promptflow-evals](https://microsoft.github.io/promptflow/reference/python-library-reference/promptflow-evals/promptflow.html)\n", + "- [azure-ai-evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk)\n", "\n", "## Time\n", "\n", @@ -28,7 +28,7 @@ "\n", "## About this example\n", "\n", - "This example demonstrates evaluating a target function using promptflow-evals\n", + "This example demonstrates evaluating a target function using azure-ai-evaluation\n", "\n", "## Before you begin\n", "\n", @@ -44,7 +44,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install promptflow-evals" + "%pip install azure-ai-evaluation" ] }, { @@ -57,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "257fd898-7ef2-4d89-872e-da9e426aaf0b", "metadata": {}, "outputs": [], @@ -66,9 +66,8 @@ "import os\n", "\n", "from pprint import pprint\n", - "from promptflow.core import AzureOpenAIModelConfiguration\n", - "from promptflow.evals.evaluate import evaluate\n", - "from promptflow.evals.evaluators import RelevanceEvaluator" + "from azure.ai.evaluation import evaluate\n", + "from azure.ai.evaluation import RelevanceEvaluator" ] }, { @@ -89,17 +88,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "fbfc3a3b", "metadata": {}, "outputs": [], "source": [ "# Use the following code to set the environment variables if not already set. If set, you can skip this step.\n", "\n", - "os.environ[\"AZURE_OPENAI_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"\"\n", - "os.environ[\"AZURE_OPENAI_DEPLOYMENT\"] = \"\"\n", - "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"\"" + "os.environ[\"AZURE_OPENAI_API_KEY\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_DEPLOYMENT\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"\"" ] }, { @@ -111,7 +110,7 @@ "source": [ "from askwiki import ask_wiki\n", "\n", - "ask_wiki(\"What is the capital of India?\")" + "ask_wiki(question=\"What is the capital of India?\")" ] }, { @@ -145,17 +144,16 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "58f76d0c-2d44-44e0-8296-110477c7e559", + "execution_count": 16, + "id": "665d0e98", "metadata": {}, "outputs": [], "source": [ - "configuration = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"AZURE_OPENAI_DEPLOYMENT\"],\n", - ")" + "model_config = {\n", + " \"azure_endpoint\": os.environ.get(\"AZURE_OPENAI_ENDPOINT\"),\n", + " \"api_key\": os.environ.get(\"AZURE_OPENAI_KEY\"),\n", + " \"azure_deployment\": os.environ.get(\"AZURE_OPENAI_DEPLOYMENT\"),\n", + " }" ] }, { @@ -173,12 +171,12 @@ "metadata": {}, "outputs": [], "source": [ - "relevance_evaluator = RelevanceEvaluator(model_config=configuration)\n", + "relevance_evaluator = RelevanceEvaluator(model_config)\n", "\n", "relevance_evaluator(\n", - " question=\"What is the capital of India?\",\n", - " answer=\"New Delhi is Capital of India\",\n", + " response=\"New Delhi is Capital of India\",\n", " context=\"India is a country in South Asia.\",\n", + " query=\"What is the capital of India?\"\n", ")" ] }, @@ -250,7 +248,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.11.9" } }, "nbformat": 4, diff --git a/scenarios/evaluate-model-endpoints/app_target.py b/scenarios/evaluate-model-endpoints/app_target.py index 7025b9a6..17b0a467 100644 --- a/scenarios/evaluate-model-endpoints/app_target.py +++ b/scenarios/evaluate-model-endpoints/app_target.py @@ -15,7 +15,7 @@ class Response(TypedDict): @trace def __call__(self: Self, question: str) -> Response: - if self.model_type == "gpt4-0613": + if self.model_type == "gpt4-0613" or self.model_type == "gpt-4o-mini": output = self.call_gpt4_endpoint(question) elif self.model_type == "gpt35-turbo": output = self.call_gpt35_turbo_endpoint(question) @@ -45,8 +45,8 @@ def call_gpt4_endpoint(self: Self, question: str) -> Response: payload = {"messages": [{"role": "user", "content": question}], "max_tokens": 500} output = self.query(endpoint=endpoint, headers=headers, payload=payload) - answer = output["choices"][0]["message"]["content"] - return {"question": question, "answer": answer} + answer = output["choices"][0]["message"]["content"] + return {"query": question, "response": answer} def call_gpt35_turbo_endpoint(self: Self, question: str) -> Response: endpoint = self.env["gpt35-turbo"]["endpoint"] @@ -58,7 +58,7 @@ def call_gpt35_turbo_endpoint(self: Self, question: str) -> Response: output = self.query(endpoint=endpoint, headers=headers, payload=payload) answer = output["choices"][0]["message"]["content"] - return {"question": question, "answer": answer} + return {"query": question, "response": answer} def call_tiny_llama_endpoint(self: Self, question: str) -> Response: endpoint = self.env["tiny_llama"]["endpoint"] @@ -75,7 +75,7 @@ def call_tiny_llama_endpoint(self: Self, question: str) -> Response: output = self.query(endpoint=endpoint, headers=headers, payload=payload) answer = output["choices"][0]["message"]["content"] - return {"question": question, "answer": answer} + return {"query": question, "response": answer} def call_phi3_mini_serverless_endpoint(self: Self, question: str) -> Response: endpoint = self.env["phi3_mini_serverless"]["endpoint"] @@ -87,7 +87,7 @@ def call_phi3_mini_serverless_endpoint(self: Self, question: str) -> Response: output = self.query(endpoint=endpoint, headers=headers, payload=payload) answer = output["choices"][0]["message"]["content"] - return {"question": question, "answer": answer} + return {"query": question, "response": answer} def call_gpt2_endpoint(self: Self, question: str) -> Response: endpoint = self.env["gpt2"]["endpoint"] @@ -101,7 +101,7 @@ def call_gpt2_endpoint(self: Self, question: str) -> Response: output = self.query(endpoint=endpoint, headers=headers, payload=payload) answer = output[0]["generated_text"] - return {"question": question, "answer": answer} + return {"query": question, "response": answer} def call_mistral_endpoint(self: Self, question: str) -> Response: endpoint = self.env["mistral7b"]["endpoint"] @@ -113,7 +113,7 @@ def call_mistral_endpoint(self: Self, question: str) -> Response: output = self.query(endpoint=endpoint, headers=headers, payload=payload) answer = output["choices"][0]["message"]["content"] - return {"question": question, "answer": answer} + return {"query": question, "response": answer} def call_default_endpoint(question: str) -> Response: - return {"question": "What is the capital of France?", "answer": "Paris"} + return {"query": "What is the capital of France?", "response": "Paris"} diff --git a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb index 6ecba4e5..adbea2bb 100644 --- a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb +++ b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb @@ -14,7 +14,7 @@ "\n", "This tutorial uses the following Azure AI services:\n", "\n", - "- [promptflow-evals](https://microsoft.github.io/promptflow/reference/python-library-reference/promptflow-evals/promptflow.html)\n", + "- [azure-ai-evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk)\n", "\n", "## Time\n", "\n", @@ -22,7 +22,7 @@ "\n", "## About this example\n", "\n", - "This example demonstrates evaluating model endpoints responses against provided prompts using promptflow-evals\n", + "This example demonstrates evaluating model endpoints responses against provided prompts using azure-ai-evaluation\n", "\n", "## Before you begin\n", "\n", @@ -33,11 +33,277 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n", + "Requirement already satisfied: azure-ai-evaluation in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (1.0.0b1)\n", + "Requirement already satisfied: promptflow-devkit>=1.15.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.15.1)\n", + "Requirement already satisfied: promptflow-core>=1.15.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.15.1)\n", + "Requirement already satisfied: pyjwt>=2.8.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (2.9.0)\n", + "Requirement already satisfied: azure-identity>=1.12.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.18.0)\n", + "Requirement already satisfied: azure-core>=1.30.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.31.0)\n", + "Requirement already satisfied: nltk>=3.9.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (3.9.1)\n", + "Requirement already satisfied: rouge-score>=0.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (0.1.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.26.4)\n", + "Requirement already satisfied: requests>=2.21.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core>=1.30.2->azure-ai-evaluation) (2.31.0)\n", + "Requirement already satisfied: six>=1.11.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core>=1.30.2->azure-ai-evaluation) (1.16.0)\n", + "Requirement already satisfied: typing-extensions>=4.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core>=1.30.2->azure-ai-evaluation) (4.12.2)\n", + "Requirement already satisfied: cryptography>=2.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity>=1.12.0->azure-ai-evaluation) (43.0.1)\n", + "Requirement already satisfied: msal>=1.30.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity>=1.12.0->azure-ai-evaluation) (1.31.0)\n", + "Requirement already satisfied: msal-extensions>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity>=1.12.0->azure-ai-evaluation) (1.2.0)\n", + "Requirement already satisfied: click in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (8.1.7)\n", + "Requirement already satisfied: joblib in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (1.4.2)\n", + "Requirement already satisfied: regex>=2021.8.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (2023.12.25)\n", + "Requirement already satisfied: tqdm in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (4.66.2)\n", + "Requirement already satisfied: docstring_parser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (0.16)\n", + "Requirement already satisfied: fastapi<1.0.0,>=0.109.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (0.115.0)\n", + "Requirement already satisfied: filetype>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (1.2.0)\n", + "Requirement already satisfied: flask<4.0.0,>=2.2.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (3.0.3)\n", + "Requirement already satisfied: jsonschema<5.0.0,>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (4.23.0)\n", + "Requirement already satisfied: promptflow-tracing==1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (1.15.1)\n", + "Requirement already satisfied: psutil in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (5.9.8)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (2.9.0.post0)\n", + "Requirement already satisfied: ruamel.yaml<1.0.0,>=0.17.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (0.18.6)\n", + "Requirement already satisfied: openai in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (1.47.1)\n", + "Requirement already satisfied: opentelemetry-sdk<2.0.0,>=1.22.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (1.27.0)\n", + "Requirement already satisfied: tiktoken>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (0.7.0)\n", + "Requirement already satisfied: argcomplete>=3.2.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.5.0)\n", + "Requirement already satisfied: azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.0.0b30)\n", + "Requirement already satisfied: colorama<0.5.0,>=0.4.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.4.6)\n", + "Requirement already satisfied: filelock<4.0.0,>=3.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.13.1)\n", + "Requirement already satisfied: flask-cors<5.0.0,>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.0.2)\n", + "Requirement already satisfied: flask-restx<2.0.0,>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.3.0)\n", + "Requirement already satisfied: gitpython<4.0.0,>=3.1.24 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.1.43)\n", + "Requirement already satisfied: httpx>=0.25.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.27.2)\n", + "Requirement already satisfied: keyring<25.0.0,>=24.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (24.3.1)\n", + "Requirement already satisfied: marshmallow<4.0.0,>=3.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.22.0)\n", + "Requirement already satisfied: opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", + "Requirement already satisfied: pandas<3.0.0,>=1.5.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.2.3)\n", + "Requirement already satisfied: pillow<11.0.0,>=10.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (10.4.0)\n", + "Requirement already satisfied: pydash<8.0.0,>=6.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (7.0.7)\n", + "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.0.1)\n", + "Requirement already satisfied: pywin32 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (306)\n", + "Requirement already satisfied: sqlalchemy<3.0.0,>=1.4.48 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.0.35)\n", + "Requirement already satisfied: strictyaml<2.0.0,>=1.5.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.7.3)\n", + "Requirement already satisfied: tabulate<1.0.0,>=0.9.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.9.0)\n", + "Requirement already satisfied: waitress<3.0.0,>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.1.2)\n", + "Requirement already satisfied: absl-py in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from rouge-score>=0.1.2->azure-ai-evaluation) (2.1.0)\n", + "Requirement already satisfied: fixedint==0.1.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.1.6)\n", + "Requirement already satisfied: msrest>=0.6.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.7.1)\n", + "Requirement already satisfied: opentelemetry-api~=1.26 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", + "Requirement already satisfied: cffi>=1.12 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cryptography>=2.5->azure-identity>=1.12.0->azure-ai-evaluation) (1.17.1)\n", + "Requirement already satisfied: starlette<0.39.0,>=0.37.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.38.6)\n", + "Requirement already satisfied: pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (2.9.2)\n", + "Requirement already satisfied: Werkzeug>=3.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (3.0.4)\n", + "Requirement already satisfied: Jinja2>=3.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (3.1.3)\n", + "Requirement already satisfied: itsdangerous>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (2.2.0)\n", + "Requirement already satisfied: blinker>=1.6.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (1.8.2)\n", + "Requirement already satisfied: aniso8601>=0.82 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (9.0.1)\n", + "Requirement already satisfied: pytz in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2024.2)\n", + "Requirement already satisfied: importlib-resources in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (6.4.5)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitpython<4.0.0,>=3.1.24->promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.0.11)\n", + "Requirement already satisfied: anyio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.6.0)\n", + "Requirement already satisfied: certifi in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2024.2.2)\n", + "Requirement already satisfied: httpcore==1.* in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.0.5)\n", + "Requirement already satisfied: idna in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.6)\n", + "Requirement already satisfied: sniffio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.3.1)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpcore==1.*->httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.14.0)\n", + "Requirement already satisfied: attrs>=22.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (24.2.0)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (2023.12.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.20.0)\n", + "Requirement already satisfied: jaraco.classes in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.4.0)\n", + "Requirement already satisfied: importlib-metadata>=4.11.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (8.4.0)\n", + "Requirement already satisfied: pywin32-ctypes>=0.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.2.3)\n", + "Requirement already satisfied: packaging>=17.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from marshmallow<4.0.0,>=3.5->promptflow-devkit>=1.15.0->azure-ai-evaluation) (24.0)\n", + "Requirement already satisfied: portalocker<3,>=1.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msal-extensions>=1.2.0->azure-identity>=1.12.0->azure-ai-evaluation) (2.10.1)\n", + "Requirement already satisfied: deprecated>=1.2.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.2.14)\n", + "Requirement already satisfied: googleapis-common-protos~=1.52 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.65.0)\n", + "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", + "Requirement already satisfied: opentelemetry-proto==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", + "Requirement already satisfied: protobuf<5.0,>=3.19 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-proto==1.27.0->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.25.5)\n", + "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2024.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core>=1.30.2->azure-ai-evaluation) (3.3.2)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core>=1.30.2->azure-ai-evaluation) (2.2.1)\n", + "Requirement already satisfied: ruamel.yaml.clib>=0.2.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from ruamel.yaml<1.0.0,>=0.17.10->promptflow-core>=1.15.0->azure-ai-evaluation) (0.2.8)\n", + "Requirement already satisfied: greenlet!=0.4.17 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from sqlalchemy<3.0.0,>=1.4.48->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.1.1)\n", + "Requirement already satisfied: pycparser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity>=1.12.0->azure-ai-evaluation) (2.22)\n", + "Requirement already satisfied: wrapt<2,>=1.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from deprecated>=1.2.6->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.16.0)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitdb<5,>=4.0.1->gitpython<4.0.0,>=3.1.24->promptflow-devkit>=1.15.0->azure-ai-evaluation) (5.0.1)\n", + "Requirement already satisfied: zipp>=0.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from importlib-metadata>=4.11.4->keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.20.2)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Jinja2>=3.1.2->flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (2.1.5)\n", + "Requirement already satisfied: isodate>=0.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msrest>=0.6.10->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.6.1)\n", + "Requirement already satisfied: requests-oauthlib>=0.5.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msrest>=0.6.10->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.0.0)\n", + "Requirement already satisfied: opentelemetry-semantic-conventions==0.48b0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-sdk<2.0.0,>=1.22.0->promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (0.48b0)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.23.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (2.23.4)\n", + "Requirement already satisfied: more-itertools in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jaraco.classes->keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (10.5.0)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (1.9.0)\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (0.5.0)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests-oauthlib>=0.5.0->msrest>=0.6.10->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.2.2)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[notice] A new release of pip is available: 24.0 -> 24.2\n", + "[notice] To update, run: C:\\Users\\sydneylister\\AppData\\Local\\Microsoft\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\python.exe -m pip install --upgrade pip\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n", + "Requirement already satisfied: promptflow-azure in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (1.15.1)\n", + "Requirement already satisfied: azure-ai-ml<2.0.0,>=1.14.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.20.0)\n", + "Requirement already satisfied: azure-core<2.0.0,>=1.26.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.31.0)\n", + "Requirement already satisfied: azure-cosmos<5.0.0,>=4.5.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (4.7.0)\n", + "Requirement already satisfied: azure-identity<2.0.0,>=1.12.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.18.0)\n", + "Requirement already satisfied: azure-storage-blob<13.0.0,>=12.17.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (12.23.0)\n", + "Requirement already satisfied: promptflow-devkit<2.0.0,>=1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.15.1)\n", + "Requirement already satisfied: pyjwt<3.0.0,>=2.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (2.9.0)\n", + "Requirement already satisfied: pyyaml>=5.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (6.0.1)\n", + "Requirement already satisfied: msrest>=0.6.18 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.7.1)\n", + "Requirement already satisfied: azure-mgmt-core>=1.3.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.4.0)\n", + "Requirement already satisfied: marshmallow>=3.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (3.22.0)\n", + "Requirement already satisfied: jsonschema>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.23.0)\n", + "Requirement already satisfied: tqdm in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.66.2)\n", + "Requirement already satisfied: strictyaml in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.7.3)\n", + "Requirement already satisfied: colorama in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.4.6)\n", + "Requirement already satisfied: azure-storage-file-share in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (12.18.0)\n", + "Requirement already satisfied: azure-storage-file-datalake>=12.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (12.17.0)\n", + "Requirement already satisfied: pydash>=6.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (7.0.7)\n", + "Requirement already satisfied: isodate in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.6.1)\n", + "Requirement already satisfied: azure-common>=1.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.1.28)\n", + "Requirement already satisfied: typing-extensions in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.12.2)\n", + "Requirement already satisfied: opencensus-ext-azure in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.1.13)\n", + "Requirement already satisfied: opencensus-ext-logging in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.1.1)\n", + "Requirement already satisfied: requests>=2.21.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core<2.0.0,>=1.26.4->promptflow-azure) (2.31.0)\n", + "Requirement already satisfied: six>=1.11.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core<2.0.0,>=1.26.4->promptflow-azure) (1.16.0)\n", + "Requirement already satisfied: cryptography>=2.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity<2.0.0,>=1.12.0->promptflow-azure) (43.0.1)\n", + "Requirement already satisfied: msal>=1.30.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity<2.0.0,>=1.12.0->promptflow-azure) (1.31.0)\n", + "Requirement already satisfied: msal-extensions>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity<2.0.0,>=1.12.0->promptflow-azure) (1.2.0)\n", + "Requirement already satisfied: argcomplete>=3.2.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.5.0)\n", + "Requirement already satisfied: azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.0.0b30)\n", + "Requirement already satisfied: filelock<4.0.0,>=3.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.13.1)\n", + "Requirement already satisfied: flask-cors<5.0.0,>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.0.2)\n", + "Requirement already satisfied: flask-restx<2.0.0,>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.3.0)\n", + "Requirement already satisfied: gitpython<4.0.0,>=3.1.24 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.1.43)\n", + "Requirement already satisfied: httpx>=0.25.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.27.2)\n", + "Requirement already satisfied: keyring<25.0.0,>=24.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (24.3.1)\n", + "Requirement already satisfied: opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", + "Requirement already satisfied: pandas<3.0.0,>=1.5.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.2.3)\n", + "Requirement already satisfied: pillow<11.0.0,>=10.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (10.4.0)\n", + "Requirement already satisfied: promptflow-core<2.0.0,>=1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.15.1)\n", + "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.0.1)\n", + "Requirement already satisfied: pywin32 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (306)\n", + "Requirement already satisfied: sqlalchemy<3.0.0,>=1.4.48 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.0.35)\n", + "Requirement already satisfied: tabulate<1.0.0,>=0.9.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.9.0)\n", + "Requirement already satisfied: waitress<3.0.0,>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.1.2)\n", + "Requirement already satisfied: aiohttp>=3.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (3.10.6)\n", + "Requirement already satisfied: fixedint==0.1.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.1.6)\n", + "Requirement already satisfied: opentelemetry-api~=1.26 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", + "Requirement already satisfied: opentelemetry-sdk~=1.26 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", + "Requirement already satisfied: psutil~=5.9 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (5.9.8)\n", + "Requirement already satisfied: cffi>=1.12 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cryptography>=2.5->azure-identity<2.0.0,>=1.12.0->promptflow-azure) (1.17.1)\n", + "Requirement already satisfied: Flask>=0.9 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.0.3)\n", + "Requirement already satisfied: aniso8601>=0.82 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (9.0.1)\n", + "Requirement already satisfied: werkzeug!=2.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.0.4)\n", + "Requirement already satisfied: pytz in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2024.2)\n", + "Requirement already satisfied: importlib-resources in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (6.4.5)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitpython<4.0.0,>=3.1.24->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.0.11)\n", + "Requirement already satisfied: anyio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.6.0)\n", + "Requirement already satisfied: certifi in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2024.2.2)\n", + "Requirement already satisfied: httpcore==1.* in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.0.5)\n", + "Requirement already satisfied: idna in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.6)\n", + "Requirement already satisfied: sniffio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.3.1)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpcore==1.*->httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.14.0)\n", + "Requirement already satisfied: attrs>=22.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (24.2.0)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2023.12.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.20.0)\n", + "Requirement already satisfied: jaraco.classes in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.4.0)\n", + "Requirement already satisfied: importlib-metadata>=4.11.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (8.4.0)\n", + "Requirement already satisfied: pywin32-ctypes>=0.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.2.3)\n", + "Requirement already satisfied: packaging>=17.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from marshmallow>=3.5->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (24.0)\n", + "Requirement already satisfied: portalocker<3,>=1.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msal-extensions>=1.2.0->azure-identity<2.0.0,>=1.12.0->promptflow-azure) (2.10.1)\n", + "Requirement already satisfied: requests-oauthlib>=0.5.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msrest>=0.6.18->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2.0.0)\n", + "Requirement already satisfied: deprecated>=1.2.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.2.14)\n", + "Requirement already satisfied: googleapis-common-protos~=1.52 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.65.0)\n", + "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", + "Requirement already satisfied: opentelemetry-proto==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", + "Requirement already satisfied: protobuf<5.0,>=3.19 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-proto==1.27.0->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.25.5)\n", + "Requirement already satisfied: numpy>=1.23.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.26.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.9.0.post0)\n", + "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2024.2)\n", + "Requirement already satisfied: docstring_parser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.16)\n", + "Requirement already satisfied: fastapi<1.0.0,>=0.109.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.115.0)\n", + "Requirement already satisfied: filetype>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.2.0)\n", + "Requirement already satisfied: promptflow-tracing==1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.15.1)\n", + "Requirement already satisfied: ruamel.yaml<1.0.0,>=0.17.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.18.6)\n", + "Requirement already satisfied: openai in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.47.1)\n", + "Requirement already satisfied: tiktoken>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.7.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core<2.0.0,>=1.26.4->promptflow-azure) (3.3.2)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core<2.0.0,>=1.26.4->promptflow-azure) (2.2.1)\n", + "Requirement already satisfied: greenlet!=0.4.17 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from sqlalchemy<3.0.0,>=1.4.48->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.1.1)\n", + "Requirement already satisfied: opencensus<1.0.0,>=0.11.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.11.4)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (2.4.0)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (1.3.1)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (1.4.1)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (6.1.0)\n", + "Requirement already satisfied: yarl<2.0,>=1.12.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (1.12.1)\n", + "Requirement already satisfied: pycparser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity<2.0.0,>=1.12.0->promptflow-azure) (2.22)\n", + "Requirement already satisfied: wrapt<2,>=1.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from deprecated>=1.2.6->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.16.0)\n", + "Requirement already satisfied: starlette<0.39.0,>=0.37.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.38.6)\n", + "Requirement already satisfied: pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.9.2)\n", + "Requirement already satisfied: Jinja2>=3.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.1.3)\n", + "Requirement already satisfied: itsdangerous>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.2.0)\n", + "Requirement already satisfied: click>=8.1.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (8.1.7)\n", + "Requirement already satisfied: blinker>=1.6.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.8.2)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitdb<5,>=4.0.1->gitpython<4.0.0,>=3.1.24->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (5.0.1)\n", + "Requirement already satisfied: zipp>=0.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from importlib-metadata>=4.11.4->keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.20.2)\n", + "Requirement already satisfied: opencensus-context>=0.1.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.1.3)\n", + "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2.20.0)\n", + "Requirement already satisfied: opentelemetry-semantic-conventions==0.48b0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-sdk~=1.26->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.48b0)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests-oauthlib>=0.5.0->msrest>=0.6.18->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (3.2.2)\n", + "Requirement already satisfied: ruamel.yaml.clib>=0.2.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from ruamel.yaml<1.0.0,>=0.17.10->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.2.8)\n", + "Requirement already satisfied: MarkupSafe>=2.1.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from werkzeug!=2.0.0->flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.1.5)\n", + "Requirement already satisfied: more-itertools in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jaraco.classes->keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (10.5.0)\n", + "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.24.0)\n", + "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2.35.0)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.23.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.23.4)\n", + "Requirement already satisfied: regex>=2022.1.18 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from tiktoken>=0.4.0->promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2023.12.25)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.9.0)\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.5.0)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (5.5.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.4.1)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.9)\n", + "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.6.1)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[notice] A new release of pip is available: 24.0 -> 24.2\n", + "[notice] To update, run: C:\\Users\\sydneylister\\AppData\\Local\\Microsoft\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\python.exe -m pip install --upgrade pip\n" + ] + } + ], "source": [ - "%pip install promptflow-evals\n", + "%pip install azure-ai-evaluation\n", "%pip install promptflow-azure" ] }, @@ -50,7 +316,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -75,7 +341,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -117,11 +383,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ - "azure_ai_project = {\"subscription_id\": \"***\", \"resource_group_name\": \"***\", \"project_name\": \"***\"}" + "azure_ai_project = {\"subscription_id\": \"\", \"resource_group_name\": \"\", \"project_name\": \"\"}" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Use the following code to set the environment variables if not already set. If set, you can skip this step.\n", + "\n", + "os.environ[\"AZURE_OPENAI_API_KEY\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_DEPLOYMENT\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"\"" ] }, { @@ -135,9 +417,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " question \\\n", + "0 What is the capital of France? \n", + "1 Which tent is the most waterproof? \n", + "2 Which camping table is the lightest? \n", + "3 How much does TrailWalker Hiking Shoes cost? \n", + "\n", + " context \\\n", + "0 France is the country in Europe. \n", + "1 #TrailMaster X4 Tent, price $250,## BrandOutdo... \n", + "2 #BaseCamp Folding Table, price $60,## BrandCam... \n", + "3 #TrailWalker Hiking Shoes, price $110## BrandT... \n", + "\n", + " ground_truth \n", + "0 Paris \n", + "1 The TrailMaster X4 tent has a rainfly waterpro... \n", + "2 The BaseCamp Folding Table has a weight of 15 lbs \n", + "3 The TrailWalker Hiking Shoes are priced at $110 \n" + ] + } + ], "source": [ "df = pd.read_json(\"data.jsonl\", lines=True)\n", "print(df.head())" @@ -153,18 +459,15 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ - "from promptflow.core import AzureOpenAIModelConfiguration\n", - "\n", - "configuration = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=\"https://ai-***.openai.azure.com\",\n", - " api_key=\"**\",\n", - " api_version=\"2023-03-15-preview\",\n", - " azure_deployment=\"gpt-35-turbo-16k\",\n", - ")" + "model_config = {\n", + " \"azure_endpoint\": os.environ.get(\"AZURE_OPENAI_ENDPOINT\"),\n", + " \"api_key\": os.environ.get(\"AZURE_OPENAI_KEY\"),\n", + " \"azure_deployment\": os.environ.get(\"AZURE_OPENAI_DEPLOYMENT\"),\n", + " }" ] }, { @@ -188,15 +491,103 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-09-26 16:53:21 -0700][promptflow][WARNING] - Found existing c:/azureai-samples/scenarios/evaluate-model-endpoints/flow.flex.yaml, will not respect it in runtime.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Prompt flow service has started...\n", + "You can view the traces in local from http://127.0.0.1:23333/v1.0/ui/traces/?#run=evaluate_model_endpoints_20240926_165321_352972\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-09-26 16:53:28 -0700][promptflow._sdk._orchestrator.run_submitter][INFO] - Submitting run evaluate_model_endpoints_20240926_165321_352972, log path: C:\\Users\\sydneylister\\.promptflow\\.runs\\evaluate_model_endpoints_20240926_165321_352972\\logs.txt\n", + "[2024-09-26 16:53:47 -0700][promptflow._sdk._orchestrator.run_submitter][WARNING] - 4 out of 4 runs failed in batch run.\n", + " Please check out C:/Users/sydneylister/.promptflow/.runs/evaluate_model_endpoints_20240926_165321_352972 for more details.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2024-09-26 16:53:28 -0700 39604 execution.bulk INFO Current thread is not main thread, skip signal handler registration in BatchEngine.\n", + "2024-09-26 16:53:28 -0700 39604 execution WARNING Starting run without column mapping may lead to unexpected results. Please consult the following documentation for more information: https://aka.ms/pf/column-mapping\n", + "2024-09-26 16:53:28 -0700 39604 execution.bulk INFO Current system's available memory is 10048.08203125MB, memory consumption of current process is 372.0625MB, estimated available worker count is 10048.08203125/372.0625 = 27\n", + "2024-09-26 16:53:28 -0700 39604 execution.bulk INFO Set process count to 4 by taking the minimum value among the factors of {'default_worker_count': 4, 'row_count': 4, 'estimated_worker_count_based_on_memory_usage': 27}.\n", + "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-32)-Process id(43356)-Line number(0) start execution.\n", + "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-33)-Process id(3604)-Line number(1) start execution.\n", + "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-35)-Process id(8800)-Line number(2) start execution.\n", + "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-34)-Process id(21616)-Line number(3) start execution.\n", + "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-32)-Process id(43356)-Line number(0) completed.\n", + "2024-09-26 16:53:38 -0700 39604 execution.bulk INFO Finished 1 / 4 lines.\n", + "2024-09-26 16:53:38 -0700 39604 execution.bulk INFO Average execution time for completed lines: 1.01 seconds. Estimated time for incomplete lines: 3.03 seconds.\n", + "2024-09-26 16:53:43 -0700 39604 execution.bulk INFO Process name(SpawnProcess-33)-Process id(3604)-Line number(1) completed.\n", + "2024-09-26 16:53:43 -0700 39604 execution.bulk INFO Finished 2 / 4 lines.\n", + "2024-09-26 16:53:43 -0700 39604 execution.bulk INFO Average execution time for completed lines: 3.03 seconds. Estimated time for incomplete lines: 6.06 seconds.\n", + "2024-09-26 16:53:44 -0700 39604 execution.bulk INFO Process name(SpawnProcess-34)-Process id(21616)-Line number(3) completed.\n", + "2024-09-26 16:53:45 -0700 39604 execution.bulk INFO Finished 3 / 4 lines.\n", + "2024-09-26 16:53:45 -0700 39604 execution.bulk INFO Average execution time for completed lines: 2.69 seconds. Estimated time for incomplete lines: 2.69 seconds.\n", + "2024-09-26 16:53:45 -0700 39604 execution.bulk INFO Process name(SpawnProcess-35)-Process id(8800)-Line number(2) completed.\n", + "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO Finished 4 / 4 lines.\n", + "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO Average execution time for completed lines: 2.27 seconds. Estimated time for incomplete lines: 0.0 seconds.\n", + "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [21616-SpawnProcess-34] will be terminated.\n", + "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [8800-SpawnProcess-35] will be terminated.\n", + "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [43356-SpawnProcess-32] will be terminated.\n", + "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [3604-SpawnProcess-33] will be terminated.\n", + "2024-09-26 16:53:46 -0700 21616 execution.bulk INFO The process [21616] has received a terminate signal.\n", + "2024-09-26 16:53:46 -0700 8800 execution.bulk INFO The process [8800] has received a terminate signal.\n", + "2024-09-26 16:53:46 -0700 43356 execution.bulk INFO The process [43356] has received a terminate signal.\n", + "2024-09-26 16:53:46 -0700 3604 execution.bulk INFO The process [3604] has received a terminate signal.\n", + "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 43356 terminated.\n", + "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 3604 terminated.\n", + "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 21616 terminated.\n", + "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 8800 terminated.\n", + "2024-09-26 16:53:47 -0700 39604 execution ERROR 4/4 flow run failed, indexes: [0,1,2,3], exception of index 0: Execution failure in 'ModelEndpoints.__call__': (ConnectionError) HTTPSConnectionPool(host='ai-***.**.azure.com', port=443): Max retries exceeded with url: /openai/deployments/gpt-4/chat/completions?api-version=2023-03-15-preview (Caused by NameResolutionError(\": Failed to resolve 'ai-***.**.azure.com' ([Errno 11001] getaddrinfo failed)\"))\n", + "======= Run Summary =======\n", + "\n", + "Run name: \"evaluate_model_endpoints_20240926_165321_352972\"\n", + "Run status: \"Completed\"\n", + "Start time: \"2024-09-26 16:53:21.348453-07:00\"\n", + "Duration: \"0:00:26.215496\"\n", + "Output path: \"C:\\Users\\sydneylister\\.promptflow\\.runs\\evaluate_model_endpoints_20240926_165321_352972\"\n", + "\n" + ] + }, + { + "ename": "EvaluationException", + "evalue": "Missing required inputs for evaluator content_safety : ['response'].", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mEvaluationException\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[25], line 35\u001b[0m\n\u001b[0;32m 33\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m model \u001b[38;5;129;01min\u001b[39;00m models:\n\u001b[0;32m 34\u001b[0m randomNum \u001b[38;5;241m=\u001b[39m random\u001b[38;5;241m.\u001b[39mrandint(\u001b[38;5;241m1111\u001b[39m, \u001b[38;5;241m9999\u001b[39m)\n\u001b[1;32m---> 35\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 36\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# azure_ai_project=azure_ai_project,\u001b[39;49;00m\n\u001b[0;32m 37\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluation_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mEval-Run-\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mrandomNum\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m-\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtitle\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 38\u001b[0m \u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 39\u001b[0m \u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mModelEndpoints\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv_var\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 40\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluators\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\n\u001b[0;32m 41\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontent_safety\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontent_safety_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 42\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcoherence\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mcoherence_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 43\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrelevance\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mrelevance_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 44\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgroundedness\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroundedness_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 45\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfluency\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mfluency_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 46\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msimilarity\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43msimilarity_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 47\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 48\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluator_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\n\u001b[0;32m 49\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontent_safety\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 50\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcoherence\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 51\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrelevance\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.query}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 52\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgroundedness\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[0;32m 53\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m 54\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m 55\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m 56\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 57\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfluency\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 58\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msimilarity\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 59\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 60\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:545\u001b[0m, in \u001b[0;36mevaluate\u001b[1;34m(evaluation_name, target, data, evaluators, evaluator_config, azure_ai_project, output_path, **kwargs)\u001b[0m\n\u001b[0;32m 531\u001b[0m error_message \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m 532\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe evaluation failed due to an error during multiprocess bootstrapping.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 533\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPlease ensure the evaluate API is properly guarded with the \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m block:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 534\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m if __name__ == \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 535\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m evaluate(...)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 536\u001b[0m )\n\u001b[0;32m 537\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m EvaluationException(\n\u001b[0;32m 538\u001b[0m message\u001b[38;5;241m=\u001b[39merror_message,\n\u001b[0;32m 539\u001b[0m internal_message\u001b[38;5;241m=\u001b[39merror_message,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 542\u001b[0m blame\u001b[38;5;241m=\u001b[39mErrorBlame\u001b[38;5;241m.\u001b[39mUNKNOWN,\n\u001b[0;32m 543\u001b[0m ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m--> 545\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:514\u001b[0m, in \u001b[0;36mevaluate\u001b[1;34m(evaluation_name, target, data, evaluators, evaluator_config, azure_ai_project, output_path, **kwargs)\u001b[0m\n\u001b[0;32m 448\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Evaluates target or data with built-in or custom evaluators. If both target and data are provided,\u001b[39;00m\n\u001b[0;32m 449\u001b[0m \u001b[38;5;124;03m data will be run through target function and then results will be evaluated.\u001b[39;00m\n\u001b[0;32m 450\u001b[0m \n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 511\u001b[0m \n\u001b[0;32m 512\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 513\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 514\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_evaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 515\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluation_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluation_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 516\u001b[0m \u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtarget\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 517\u001b[0m \u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 518\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluators\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluators\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 519\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluator_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluator_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 520\u001b[0m \u001b[43m \u001b[49m\u001b[43mazure_ai_project\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mazure_ai_project\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 521\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 522\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 523\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 524\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m 525\u001b[0m \u001b[38;5;66;03m# Handle multiprocess bootstrap error\u001b[39;00m\n\u001b[0;32m 526\u001b[0m bootstrap_error \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m 527\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn attempt has been made to start a new process before the\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 528\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcurrent process has finished its bootstrapping phase.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 529\u001b[0m )\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:605\u001b[0m, in \u001b[0;36m_evaluate\u001b[1;34m(evaluation_name, target, data, evaluators, evaluator_config, azure_ai_project, output_path, **kwargs)\u001b[0m\n\u001b[0;32m 601\u001b[0m evaluator_config[evaluator_name][col] \u001b[38;5;241m=\u001b[39m run_output \u001b[38;5;66;03m# pylint: disable=unnecessary-dict-index-lookup\u001b[39;00m\n\u001b[0;32m 603\u001b[0m \u001b[38;5;66;03m# After we have generated all columns we can check if we have\u001b[39;00m\n\u001b[0;32m 604\u001b[0m \u001b[38;5;66;03m# everything we need for evaluators.\u001b[39;00m\n\u001b[1;32m--> 605\u001b[0m \u001b[43m_validate_columns\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_data_df\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevaluators\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevaluator_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluator_config\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 607\u001b[0m \u001b[38;5;66;03m# Batch Run\u001b[39;00m\n\u001b[0;32m 608\u001b[0m evaluators_info \u001b[38;5;241m=\u001b[39m {}\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:314\u001b[0m, in \u001b[0;36m_validate_columns\u001b[1;34m(df, evaluators, target, evaluator_config)\u001b[0m\n\u001b[0;32m 311\u001b[0m new_df \u001b[38;5;241m=\u001b[39m _apply_column_mapping(df, mapping_config)\n\u001b[0;32m 313\u001b[0m \u001b[38;5;66;03m# Validate input data for evaluator\u001b[39;00m\n\u001b[1;32m--> 314\u001b[0m \u001b[43m_validate_input_data_for_evaluator\u001b[49m\u001b[43m(\u001b[49m\u001b[43mevaluator\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevaluator_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnew_df\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:165\u001b[0m, in \u001b[0;36m_validate_input_data_for_evaluator\u001b[1;34m(evaluator, evaluator_name, df_data, is_target_fn)\u001b[0m\n\u001b[0;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_target_fn:\n\u001b[0;32m 164\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMissing required inputs for evaluator \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mevaluator_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmissing_inputs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m--> 165\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m EvaluationException(\n\u001b[0;32m 166\u001b[0m message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[0;32m 167\u001b[0m internal_message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[0;32m 168\u001b[0m target\u001b[38;5;241m=\u001b[39mErrorTarget\u001b[38;5;241m.\u001b[39mEVALUATE,\n\u001b[0;32m 169\u001b[0m category\u001b[38;5;241m=\u001b[39mErrorCategory\u001b[38;5;241m.\u001b[39mMISSING_FIELD,\n\u001b[0;32m 170\u001b[0m blame\u001b[38;5;241m=\u001b[39mErrorBlame\u001b[38;5;241m.\u001b[39mUSER_ERROR,\n\u001b[0;32m 171\u001b[0m )\n\u001b[0;32m 172\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMissing required inputs for target : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmissing_inputs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 173\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m EvaluationException(\n\u001b[0;32m 174\u001b[0m message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[0;32m 175\u001b[0m internal_message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 178\u001b[0m blame\u001b[38;5;241m=\u001b[39mErrorBlame\u001b[38;5;241m.\u001b[39mUSER_ERROR,\n\u001b[0;32m 179\u001b[0m )\n", + "\u001b[1;31mEvaluationException\u001b[0m: Missing required inputs for evaluator content_safety : ['response']." + ] + } + ], "source": [ "from app_target import ModelEndpoints\n", "import pathlib\n", "\n", - "from promptflow.evals.evaluate import evaluate\n", - "from promptflow.evals.evaluators import (\n", + "from azure.ai.evaluation import evaluate\n", + "from azure.ai.evaluation import (\n", " ContentSafetyEvaluator,\n", " RelevanceEvaluator,\n", " CoherenceEvaluator,\n", @@ -206,12 +597,12 @@ ")\n", "\n", "\n", - "content_safety_evaluator = ContentSafetyEvaluator(project_scope=azure_ai_project)\n", - "relevance_evaluator = RelevanceEvaluator(model_config=configuration)\n", - "coherence_evaluator = CoherenceEvaluator(model_config=configuration)\n", - "groundedness_evaluator = GroundednessEvaluator(model_config=configuration)\n", - "fluency_evaluator = FluencyEvaluator(model_config=configuration)\n", - "similarity_evaluator = SimilarityEvaluator(model_config=configuration)\n", + "content_safety_evaluator = ContentSafetyEvaluator(azure_ai_project)\n", + "relevance_evaluator = RelevanceEvaluator(model_config)\n", + "coherence_evaluator = CoherenceEvaluator(model_config)\n", + "groundedness_evaluator = GroundednessEvaluator(model_config)\n", + "fluency_evaluator = FluencyEvaluator(model_config)\n", + "similarity_evaluator = SimilarityEvaluator(model_config)\n", "\n", "models = [\n", " \"gpt4-0613\",\n", @@ -227,7 +618,6 @@ "for model in models:\n", " randomNum = random.randint(1111, 9999)\n", " results = evaluate(\n", - " azure_ai_project=azure_ai_project,\n", " evaluation_name=\"Eval-Run-\" + str(randomNum) + \"-\" + model.title(),\n", " data=path,\n", " target=ModelEndpoints(env_var, model),\n", @@ -240,16 +630,16 @@ " \"similarity\": similarity_evaluator,\n", " },\n", " evaluator_config={\n", - " \"content_safety\": {\"question\": \"${data.question}\", \"answer\": \"${target.answer}\"},\n", - " \"coherence\": {\"answer\": \"${target.answer}\", \"question\": \"${data.question}\"},\n", - " \"relevance\": {\"answer\": \"${target.answer}\", \"context\": \"${data.context}\", \"question\": \"${data.question}\"},\n", + " \"content_safety\": {\"query\": \"${data.question}\", \"response\": \"${target.response}\"},\n", + " \"coherence\": {\"response\": \"${target.response}\", \"query\": \"${data.question}\"},\n", + " \"relevance\": {\"response\": \"${target.query}\", \"context\": \"${data.context}\", \"query\": \"${data.question}\"},\n", " \"groundedness\": {\n", - " \"answer\": \"${target.answer}\",\n", + " \"response\": \"${target.response}\",\n", " \"context\": \"${data.context}\",\n", - " \"question\": \"${data.question}\",\n", + " \"query\": \"${data.question}\",\n", " },\n", - " \"fluency\": {\"answer\": \"${target.answer}\", \"context\": \"${data.context}\", \"question\": \"${data.question}\"},\n", - " \"similarity\": {\"answer\": \"${target.answer}\", \"context\": \"${data.context}\", \"question\": \"${data.question}\"},\n", + " \"fluency\": {\"response\": \"${target.response}\", \"context\": \"${data.context}\", \"query\": \"${data.question}\"},\n", + " \"similarity\": {\"response\": \"${target.response}\", \"context\": \"${data.context}\", \"query\": \"${data.question}\"},\n", " },\n", " )" ] @@ -295,7 +685,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.11.9" } }, "nbformat": 4, diff --git a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb index 4881a040..8a807308 100644 --- a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb +++ b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb @@ -13,7 +13,7 @@ "This tutorial uses the following Azure AI services:\n", "\n", "- [Azure AI Safety Evaluation](https://aka.ms/azureaistudiosafetyeval)\n", - "- [promptflow-evals](https://microsoft.github.io/promptflow/reference/python-library-reference/promptflow-evals/promptflow.html)\n", + "- [azure-ai-evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk)\n", "\n", "## Time\n", "\n", @@ -36,7 +36,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install promptflow-evals" + "%pip install azure-ai-evaluation" ] }, { @@ -48,13 +48,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 72, "metadata": {}, "outputs": [], "source": [ "from pathlib import Path\n", - "from azure.identity import DefaultAzureCredential\n", - "from promptflow.evals.synthetic import AdversarialSimulator, AdversarialScenario\n", + "from azure.ai.evaluation.simulator import AdversarialSimulator, AdversarialScenario\n", "from typing import List, Dict, Any, Optional" ] }, @@ -78,22 +77,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 75, + "metadata": {}, + "outputs": [], + "source": [ + "azure_ai_project = {\"subscription_id\": \"\", \"resource_group_name\": \"\", \"project_name\": \"\"}" + ] + }, + { + "cell_type": "code", + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", - "os.environ[\"AZURE_OPENAI_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"\"\n", - "os.environ[\"AZURE_OPENAI_DEPLOYMENT\"] = \"\"\n", - "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"\"\n", - "azure_ai_project = {\n", - " \"subscription_id\": \"\",\n", - " \"resource_group_name\": \"\",\n", - " \"project_name\": \"\",\n", - " \"credential\": DefaultAzureCredential(),\n", - "}" + "# Use the following code to set the environment variables if not already set. If set, you can skip this step.\n", + "\n", + "os.environ[\"AZURE_OPENAI_API_KEY\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_DEPLOYMENT\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"\"" ] }, { @@ -141,7 +145,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 78, "metadata": {}, "outputs": [], "source": [ @@ -217,15 +221,11 @@ "metadata": {}, "outputs": [], "source": [ - "from promptflow.evals.evaluators import (\n", + "from azure.ai.evaluation import (\n", " ContentSafetyEvaluator,\n", ")\n", - "from promptflow.evals.evaluate import evaluate\n", "\n", - "# credential is acquired within evaluate\n", - "azure_ai_project.pop(\"credential\", None)\n", - "\n", - "content_safety_evaluator = ContentSafetyEvaluator(project_scope=azure_ai_project)" + "content_safety_evaluator = ContentSafetyEvaluator(azure_ai_project)" ] }, { @@ -237,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 84, "metadata": {}, "outputs": [], "source": [ @@ -245,7 +245,7 @@ "\n", "json_line_as_json = json.loads(outputs.to_eval_qa_json_lines())\n", "content_safety_eval_result = content_safety_evaluator(\n", - " question=json_line_as_json[\"question\"], answer=json_line_as_json[\"answer\"]\n", + " query=json_line_as_json[\"query\"], response=json_line_as_json[\"response\"]\n", ")\n", "# {'violence': 'Safe',\n", "# 'violence_score': 0,\n", @@ -287,26 +287,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Run the evaluate method with the content safety evaluator and blocklist evaluator" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "results = evaluate(\n", - " data=\"outputs.jsonl\",\n", - " evaluators={\"content_safety\": content_safety_evaluator, \"blocklist\": blocklist_evaluator},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### View the results of the evaluation" + "### Run the blocklist evaluator" ] }, { @@ -315,7 +296,10 @@ "metadata": {}, "outputs": [], "source": [ - "results" + "json_lines = outputs.to_eval_qa_json_lines()\n", + "for line in json_lines.split(\"\\n\"):\n", + " if line:\n", + " print(blocklist_evaluator(answer=json.loads(line)['response']))" ] } ], @@ -334,7 +318,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.11.9" } }, "nbformat": 4, diff --git a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb index cff321b3..b5a30761 100644 --- a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb +++ b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb @@ -13,7 +13,7 @@ "This tutorial uses the following Azure AI services:\n", "\n", "- [Azure AI Safety Evaluation](https://aka.ms/azureaistudiosafetyeval)\n", - "- [promptflow-evals](https://microsoft.github.io/promptflow/reference/python-library-reference/promptflow-evals/promptflow.html)\n", + "- [azure-ai-evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk)\n", "\n", "## Time\n", "\n", @@ -37,7 +37,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install promptflow-evals\n", + "%pip install azure-ai-evaluation\n", "%pip install requests" ] }, @@ -50,14 +50,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "import json\n", "from pathlib import Path\n", - "from azure.identity import DefaultAzureCredential\n", - "from promptflow.evals.synthetic import AdversarialSimulator, AdversarialScenario\n", + "from azure.ai.evaluation.simulator import AdversarialSimulator, AdversarialScenario\n", "import requests\n", "from typing import Optional, List, Dict, Any" ] @@ -69,7 +68,7 @@ "## Target function\n", "The target function for this sample uses a call to call the endpoint.\n", "\n", - "Make sure you retrive the `key`, `endpoint` and `azure_model_deployment` from Azure AI studio" + "Make sure you retrive the `api_key`, `endpoint` and `azure_model_deployment` from Azure AI studio and update them in the `call_endpoint` function below. " ] }, { @@ -78,25 +77,20 @@ "metadata": {}, "outputs": [], "source": [ - "azure_ai_project = {\n", - " \"subscription_id\": \"\",\n", - " \"resource_group_name\": \"\",\n", - " \"project_name\": \"\",\n", - " \"credential\": DefaultAzureCredential(),\n", - "}" + "azure_ai_project = {\"subscription_id\": \"\", \"resource_group_name\": \"\", \"project_name\": \"\"}" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ "def call_endpoint(query: str) -> dict:\n", " data = {\"query\": query}\n", " body = json.dumps(data)\n", - " api_key = \"\"\n", - " endpoint = \"\"\n", + " api_key=\"\"\n", + " endpoint=\"\"\n", " azure_model_deployment = \"\"\n", "\n", " if not api_key:\n", @@ -117,6 +111,7 @@ " print(err.response.text)\n", "\n", " json_output = json.loads(result)\n", + " print(json_output)\n", "\n", " return {\n", " \"answer\": json_output[\"reply\"],\n", @@ -133,7 +128,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ @@ -151,11 +146,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "metadata": {}, "outputs": [], "source": [ - "## define a callback that formats the interaction between the simulator and the ask wiki application\n", + "## define a callback that formats the interaction between the simulator and the online endpoint\n", "\n", "\n", "async def callback(\n", @@ -169,13 +164,11 @@ " latest_message = messages_list[-1]\n", " query = latest_message[\"content\"]\n", " context = None\n", - " response_from_ask_wiki = call_endpoint(query)\n", " # we are formatting the response to follow the openAI chat protocol format\n", " formatted_response = {\n", - " \"content\": response_from_ask_wiki[\"answer\"],\n", + " \"content\": \"I cannot answer\",\n", " \"role\": \"assistant\",\n", " \"context\": {\n", - " \"citations\": response_from_ask_wiki[\"context\"],\n", " },\n", " }\n", " messages[\"messages\"].append(formatted_response)\n", @@ -202,11 +195,10 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ - "outputs.to_eval_qa_json_lines()\n", "with Path.open(\"outputs.jsonl\", \"w\") as f:\n", " f.write(outputs.to_eval_qa_json_lines())" ] @@ -222,19 +214,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 47, "metadata": {}, "outputs": [], "source": [ - "from promptflow.evals.evaluators import (\n", + "from azure.ai.evaluation import (\n", " ContentSafetyEvaluator,\n", ")\n", - "from promptflow.evals.evaluate import evaluate\n", + "from azure.ai.evaluation import evaluate\n", "\n", "# credential is acquired within evaluate\n", "azure_ai_project.pop(\"credential\", None)\n", "\n", - "content_safety_evaluator = ContentSafetyEvaluator(project_scope=azure_ai_project)" + "content_safety_evaluator = ContentSafetyEvaluator(azure_ai_project)" ] }, { @@ -246,13 +238,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ "json_line_as_json = json.loads(outputs.to_eval_qa_json_lines())\n", "content_safety_eval_result = content_safety_evaluator(\n", - " question=json_line_as_json[\"question\"], answer=json_line_as_json[\"answer\"]\n", + " query=json_line_as_json[\"query\"], response=json_line_as_json[\"response\"]\n", ")\n", "# {'violence': 'Safe',\n", "# 'violence_score': 0,\n", @@ -316,7 +308,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.11.9" } }, "nbformat": 4, From 3c9826967d7a0cda4e022da8237c54b9e1f31359 Mon Sep 17 00:00:00 2001 From: Sydney Date: Thu, 26 Sep 2024 16:58:15 -0700 Subject: [PATCH 02/10] clear local variables --- .../evaluate-model-endpoints/app_target.py | 2 +- .../evaluate-models-target.ipynb | 390 +----------------- 2 files changed, 7 insertions(+), 385 deletions(-) diff --git a/scenarios/evaluate-model-endpoints/app_target.py b/scenarios/evaluate-model-endpoints/app_target.py index 17b0a467..516c6d87 100644 --- a/scenarios/evaluate-model-endpoints/app_target.py +++ b/scenarios/evaluate-model-endpoints/app_target.py @@ -15,7 +15,7 @@ class Response(TypedDict): @trace def __call__(self: Self, question: str) -> Response: - if self.model_type == "gpt4-0613" or self.model_type == "gpt-4o-mini": + if self.model_type == "gpt4-0613": output = self.call_gpt4_endpoint(question) elif self.model_type == "gpt35-turbo": output = self.call_gpt35_turbo_endpoint(question) diff --git a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb index adbea2bb..e6bcdb20 100644 --- a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb +++ b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb @@ -33,275 +33,9 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n", - "Requirement already satisfied: azure-ai-evaluation in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (1.0.0b1)\n", - "Requirement already satisfied: promptflow-devkit>=1.15.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.15.1)\n", - "Requirement already satisfied: promptflow-core>=1.15.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.15.1)\n", - "Requirement already satisfied: pyjwt>=2.8.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (2.9.0)\n", - "Requirement already satisfied: azure-identity>=1.12.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.18.0)\n", - "Requirement already satisfied: azure-core>=1.30.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.31.0)\n", - "Requirement already satisfied: nltk>=3.9.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (3.9.1)\n", - "Requirement already satisfied: rouge-score>=0.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (0.1.2)\n", - "Requirement already satisfied: numpy>=1.23.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.26.4)\n", - "Requirement already satisfied: requests>=2.21.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core>=1.30.2->azure-ai-evaluation) (2.31.0)\n", - "Requirement already satisfied: six>=1.11.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core>=1.30.2->azure-ai-evaluation) (1.16.0)\n", - "Requirement already satisfied: typing-extensions>=4.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core>=1.30.2->azure-ai-evaluation) (4.12.2)\n", - "Requirement already satisfied: cryptography>=2.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity>=1.12.0->azure-ai-evaluation) (43.0.1)\n", - "Requirement already satisfied: msal>=1.30.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity>=1.12.0->azure-ai-evaluation) (1.31.0)\n", - "Requirement already satisfied: msal-extensions>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity>=1.12.0->azure-ai-evaluation) (1.2.0)\n", - "Requirement already satisfied: click in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (8.1.7)\n", - "Requirement already satisfied: joblib in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (1.4.2)\n", - "Requirement already satisfied: regex>=2021.8.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (2023.12.25)\n", - "Requirement already satisfied: tqdm in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (4.66.2)\n", - "Requirement already satisfied: docstring_parser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (0.16)\n", - "Requirement already satisfied: fastapi<1.0.0,>=0.109.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (0.115.0)\n", - "Requirement already satisfied: filetype>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (1.2.0)\n", - "Requirement already satisfied: flask<4.0.0,>=2.2.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (3.0.3)\n", - "Requirement already satisfied: jsonschema<5.0.0,>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (4.23.0)\n", - "Requirement already satisfied: promptflow-tracing==1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (1.15.1)\n", - "Requirement already satisfied: psutil in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (5.9.8)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (2.9.0.post0)\n", - "Requirement already satisfied: ruamel.yaml<1.0.0,>=0.17.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (0.18.6)\n", - "Requirement already satisfied: openai in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (1.47.1)\n", - "Requirement already satisfied: opentelemetry-sdk<2.0.0,>=1.22.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (1.27.0)\n", - "Requirement already satisfied: tiktoken>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (0.7.0)\n", - "Requirement already satisfied: argcomplete>=3.2.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.5.0)\n", - "Requirement already satisfied: azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.0.0b30)\n", - "Requirement already satisfied: colorama<0.5.0,>=0.4.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.4.6)\n", - "Requirement already satisfied: filelock<4.0.0,>=3.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.13.1)\n", - "Requirement already satisfied: flask-cors<5.0.0,>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.0.2)\n", - "Requirement already satisfied: flask-restx<2.0.0,>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.3.0)\n", - "Requirement already satisfied: gitpython<4.0.0,>=3.1.24 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.1.43)\n", - "Requirement already satisfied: httpx>=0.25.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.27.2)\n", - "Requirement already satisfied: keyring<25.0.0,>=24.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (24.3.1)\n", - "Requirement already satisfied: marshmallow<4.0.0,>=3.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.22.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", - "Requirement already satisfied: pandas<3.0.0,>=1.5.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.2.3)\n", - "Requirement already satisfied: pillow<11.0.0,>=10.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (10.4.0)\n", - "Requirement already satisfied: pydash<8.0.0,>=6.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (7.0.7)\n", - "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.0.1)\n", - "Requirement already satisfied: pywin32 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (306)\n", - "Requirement already satisfied: sqlalchemy<3.0.0,>=1.4.48 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.0.35)\n", - "Requirement already satisfied: strictyaml<2.0.0,>=1.5.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.7.3)\n", - "Requirement already satisfied: tabulate<1.0.0,>=0.9.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.9.0)\n", - "Requirement already satisfied: waitress<3.0.0,>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.1.2)\n", - "Requirement already satisfied: absl-py in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from rouge-score>=0.1.2->azure-ai-evaluation) (2.1.0)\n", - "Requirement already satisfied: fixedint==0.1.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.1.6)\n", - "Requirement already satisfied: msrest>=0.6.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.7.1)\n", - "Requirement already satisfied: opentelemetry-api~=1.26 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", - "Requirement already satisfied: cffi>=1.12 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cryptography>=2.5->azure-identity>=1.12.0->azure-ai-evaluation) (1.17.1)\n", - "Requirement already satisfied: starlette<0.39.0,>=0.37.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.38.6)\n", - "Requirement already satisfied: pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (2.9.2)\n", - "Requirement already satisfied: Werkzeug>=3.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (3.0.4)\n", - "Requirement already satisfied: Jinja2>=3.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (3.1.3)\n", - "Requirement already satisfied: itsdangerous>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (2.2.0)\n", - "Requirement already satisfied: blinker>=1.6.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (1.8.2)\n", - "Requirement already satisfied: aniso8601>=0.82 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (9.0.1)\n", - "Requirement already satisfied: pytz in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2024.2)\n", - "Requirement already satisfied: importlib-resources in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (6.4.5)\n", - "Requirement already satisfied: gitdb<5,>=4.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitpython<4.0.0,>=3.1.24->promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.0.11)\n", - "Requirement already satisfied: anyio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.6.0)\n", - "Requirement already satisfied: certifi in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2024.2.2)\n", - "Requirement already satisfied: httpcore==1.* in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.0.5)\n", - "Requirement already satisfied: idna in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.6)\n", - "Requirement already satisfied: sniffio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.3.1)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpcore==1.*->httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.14.0)\n", - "Requirement already satisfied: attrs>=22.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (24.2.0)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (2023.12.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.20.0)\n", - "Requirement already satisfied: jaraco.classes in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.4.0)\n", - "Requirement already satisfied: importlib-metadata>=4.11.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (8.4.0)\n", - "Requirement already satisfied: pywin32-ctypes>=0.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.2.3)\n", - "Requirement already satisfied: packaging>=17.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from marshmallow<4.0.0,>=3.5->promptflow-devkit>=1.15.0->azure-ai-evaluation) (24.0)\n", - "Requirement already satisfied: portalocker<3,>=1.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msal-extensions>=1.2.0->azure-identity>=1.12.0->azure-ai-evaluation) (2.10.1)\n", - "Requirement already satisfied: deprecated>=1.2.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.2.14)\n", - "Requirement already satisfied: googleapis-common-protos~=1.52 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.65.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", - "Requirement already satisfied: opentelemetry-proto==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", - "Requirement already satisfied: protobuf<5.0,>=3.19 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-proto==1.27.0->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.25.5)\n", - "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2024.2)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core>=1.30.2->azure-ai-evaluation) (3.3.2)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core>=1.30.2->azure-ai-evaluation) (2.2.1)\n", - "Requirement already satisfied: ruamel.yaml.clib>=0.2.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from ruamel.yaml<1.0.0,>=0.17.10->promptflow-core>=1.15.0->azure-ai-evaluation) (0.2.8)\n", - "Requirement already satisfied: greenlet!=0.4.17 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from sqlalchemy<3.0.0,>=1.4.48->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.1.1)\n", - "Requirement already satisfied: pycparser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity>=1.12.0->azure-ai-evaluation) (2.22)\n", - "Requirement already satisfied: wrapt<2,>=1.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from deprecated>=1.2.6->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.16.0)\n", - "Requirement already satisfied: smmap<6,>=3.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitdb<5,>=4.0.1->gitpython<4.0.0,>=3.1.24->promptflow-devkit>=1.15.0->azure-ai-evaluation) (5.0.1)\n", - "Requirement already satisfied: zipp>=0.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from importlib-metadata>=4.11.4->keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.20.2)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Jinja2>=3.1.2->flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (2.1.5)\n", - "Requirement already satisfied: isodate>=0.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msrest>=0.6.10->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.6.1)\n", - "Requirement already satisfied: requests-oauthlib>=0.5.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msrest>=0.6.10->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.0.0)\n", - "Requirement already satisfied: opentelemetry-semantic-conventions==0.48b0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-sdk<2.0.0,>=1.22.0->promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (0.48b0)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.23.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (2.23.4)\n", - "Requirement already satisfied: more-itertools in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jaraco.classes->keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (10.5.0)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (1.9.0)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (0.5.0)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests-oauthlib>=0.5.0->msrest>=0.6.10->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.2.2)\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "[notice] A new release of pip is available: 24.0 -> 24.2\n", - "[notice] To update, run: C:\\Users\\sydneylister\\AppData\\Local\\Microsoft\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\python.exe -m pip install --upgrade pip\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n", - "Requirement already satisfied: promptflow-azure in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (1.15.1)\n", - "Requirement already satisfied: azure-ai-ml<2.0.0,>=1.14.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.20.0)\n", - "Requirement already satisfied: azure-core<2.0.0,>=1.26.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.31.0)\n", - "Requirement already satisfied: azure-cosmos<5.0.0,>=4.5.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (4.7.0)\n", - "Requirement already satisfied: azure-identity<2.0.0,>=1.12.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.18.0)\n", - "Requirement already satisfied: azure-storage-blob<13.0.0,>=12.17.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (12.23.0)\n", - "Requirement already satisfied: promptflow-devkit<2.0.0,>=1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.15.1)\n", - "Requirement already satisfied: pyjwt<3.0.0,>=2.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (2.9.0)\n", - "Requirement already satisfied: pyyaml>=5.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (6.0.1)\n", - "Requirement already satisfied: msrest>=0.6.18 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.7.1)\n", - "Requirement already satisfied: azure-mgmt-core>=1.3.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.4.0)\n", - "Requirement already satisfied: marshmallow>=3.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (3.22.0)\n", - "Requirement already satisfied: jsonschema>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.23.0)\n", - "Requirement already satisfied: tqdm in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.66.2)\n", - "Requirement already satisfied: strictyaml in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.7.3)\n", - "Requirement already satisfied: colorama in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.4.6)\n", - "Requirement already satisfied: azure-storage-file-share in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (12.18.0)\n", - "Requirement already satisfied: azure-storage-file-datalake>=12.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (12.17.0)\n", - "Requirement already satisfied: pydash>=6.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (7.0.7)\n", - "Requirement already satisfied: isodate in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.6.1)\n", - "Requirement already satisfied: azure-common>=1.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.1.28)\n", - "Requirement already satisfied: typing-extensions in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.12.2)\n", - "Requirement already satisfied: opencensus-ext-azure in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.1.13)\n", - "Requirement already satisfied: opencensus-ext-logging in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.1.1)\n", - "Requirement already satisfied: requests>=2.21.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core<2.0.0,>=1.26.4->promptflow-azure) (2.31.0)\n", - "Requirement already satisfied: six>=1.11.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core<2.0.0,>=1.26.4->promptflow-azure) (1.16.0)\n", - "Requirement already satisfied: cryptography>=2.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity<2.0.0,>=1.12.0->promptflow-azure) (43.0.1)\n", - "Requirement already satisfied: msal>=1.30.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity<2.0.0,>=1.12.0->promptflow-azure) (1.31.0)\n", - "Requirement already satisfied: msal-extensions>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity<2.0.0,>=1.12.0->promptflow-azure) (1.2.0)\n", - "Requirement already satisfied: argcomplete>=3.2.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.5.0)\n", - "Requirement already satisfied: azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.0.0b30)\n", - "Requirement already satisfied: filelock<4.0.0,>=3.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.13.1)\n", - "Requirement already satisfied: flask-cors<5.0.0,>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.0.2)\n", - "Requirement already satisfied: flask-restx<2.0.0,>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.3.0)\n", - "Requirement already satisfied: gitpython<4.0.0,>=3.1.24 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.1.43)\n", - "Requirement already satisfied: httpx>=0.25.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.27.2)\n", - "Requirement already satisfied: keyring<25.0.0,>=24.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (24.3.1)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", - "Requirement already satisfied: pandas<3.0.0,>=1.5.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.2.3)\n", - "Requirement already satisfied: pillow<11.0.0,>=10.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (10.4.0)\n", - "Requirement already satisfied: promptflow-core<2.0.0,>=1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.15.1)\n", - "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.0.1)\n", - "Requirement already satisfied: pywin32 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (306)\n", - "Requirement already satisfied: sqlalchemy<3.0.0,>=1.4.48 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.0.35)\n", - "Requirement already satisfied: tabulate<1.0.0,>=0.9.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.9.0)\n", - "Requirement already satisfied: waitress<3.0.0,>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.1.2)\n", - "Requirement already satisfied: aiohttp>=3.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (3.10.6)\n", - "Requirement already satisfied: fixedint==0.1.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.1.6)\n", - "Requirement already satisfied: opentelemetry-api~=1.26 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", - "Requirement already satisfied: opentelemetry-sdk~=1.26 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", - "Requirement already satisfied: psutil~=5.9 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (5.9.8)\n", - "Requirement already satisfied: cffi>=1.12 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cryptography>=2.5->azure-identity<2.0.0,>=1.12.0->promptflow-azure) (1.17.1)\n", - "Requirement already satisfied: Flask>=0.9 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.0.3)\n", - "Requirement already satisfied: aniso8601>=0.82 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (9.0.1)\n", - "Requirement already satisfied: werkzeug!=2.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.0.4)\n", - "Requirement already satisfied: pytz in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2024.2)\n", - "Requirement already satisfied: importlib-resources in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (6.4.5)\n", - "Requirement already satisfied: gitdb<5,>=4.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitpython<4.0.0,>=3.1.24->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.0.11)\n", - "Requirement already satisfied: anyio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.6.0)\n", - "Requirement already satisfied: certifi in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2024.2.2)\n", - "Requirement already satisfied: httpcore==1.* in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.0.5)\n", - "Requirement already satisfied: idna in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.6)\n", - "Requirement already satisfied: sniffio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.3.1)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpcore==1.*->httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.14.0)\n", - "Requirement already satisfied: attrs>=22.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (24.2.0)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2023.12.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.20.0)\n", - "Requirement already satisfied: jaraco.classes in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.4.0)\n", - "Requirement already satisfied: importlib-metadata>=4.11.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (8.4.0)\n", - "Requirement already satisfied: pywin32-ctypes>=0.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.2.3)\n", - "Requirement already satisfied: packaging>=17.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from marshmallow>=3.5->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (24.0)\n", - "Requirement already satisfied: portalocker<3,>=1.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msal-extensions>=1.2.0->azure-identity<2.0.0,>=1.12.0->promptflow-azure) (2.10.1)\n", - "Requirement already satisfied: requests-oauthlib>=0.5.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msrest>=0.6.18->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2.0.0)\n", - "Requirement already satisfied: deprecated>=1.2.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.2.14)\n", - "Requirement already satisfied: googleapis-common-protos~=1.52 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.65.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", - "Requirement already satisfied: opentelemetry-proto==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", - "Requirement already satisfied: protobuf<5.0,>=3.19 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-proto==1.27.0->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.25.5)\n", - "Requirement already satisfied: numpy>=1.23.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.26.4)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.9.0.post0)\n", - "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2024.2)\n", - "Requirement already satisfied: docstring_parser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.16)\n", - "Requirement already satisfied: fastapi<1.0.0,>=0.109.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.115.0)\n", - "Requirement already satisfied: filetype>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.2.0)\n", - "Requirement already satisfied: promptflow-tracing==1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.15.1)\n", - "Requirement already satisfied: ruamel.yaml<1.0.0,>=0.17.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.18.6)\n", - "Requirement already satisfied: openai in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.47.1)\n", - "Requirement already satisfied: tiktoken>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.7.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core<2.0.0,>=1.26.4->promptflow-azure) (3.3.2)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core<2.0.0,>=1.26.4->promptflow-azure) (2.2.1)\n", - "Requirement already satisfied: greenlet!=0.4.17 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from sqlalchemy<3.0.0,>=1.4.48->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.1.1)\n", - "Requirement already satisfied: opencensus<1.0.0,>=0.11.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.11.4)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (2.4.0)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (1.3.1)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (1.4.1)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (6.1.0)\n", - "Requirement already satisfied: yarl<2.0,>=1.12.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (1.12.1)\n", - "Requirement already satisfied: pycparser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity<2.0.0,>=1.12.0->promptflow-azure) (2.22)\n", - "Requirement already satisfied: wrapt<2,>=1.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from deprecated>=1.2.6->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.16.0)\n", - "Requirement already satisfied: starlette<0.39.0,>=0.37.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.38.6)\n", - "Requirement already satisfied: pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.9.2)\n", - "Requirement already satisfied: Jinja2>=3.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.1.3)\n", - "Requirement already satisfied: itsdangerous>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.2.0)\n", - "Requirement already satisfied: click>=8.1.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (8.1.7)\n", - "Requirement already satisfied: blinker>=1.6.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.8.2)\n", - "Requirement already satisfied: smmap<6,>=3.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitdb<5,>=4.0.1->gitpython<4.0.0,>=3.1.24->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (5.0.1)\n", - "Requirement already satisfied: zipp>=0.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from importlib-metadata>=4.11.4->keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.20.2)\n", - "Requirement already satisfied: opencensus-context>=0.1.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.1.3)\n", - "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2.20.0)\n", - "Requirement already satisfied: opentelemetry-semantic-conventions==0.48b0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-sdk~=1.26->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.48b0)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests-oauthlib>=0.5.0->msrest>=0.6.18->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (3.2.2)\n", - "Requirement already satisfied: ruamel.yaml.clib>=0.2.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from ruamel.yaml<1.0.0,>=0.17.10->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.2.8)\n", - "Requirement already satisfied: MarkupSafe>=2.1.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from werkzeug!=2.0.0->flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.1.5)\n", - "Requirement already satisfied: more-itertools in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jaraco.classes->keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (10.5.0)\n", - "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.24.0)\n", - "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2.35.0)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.23.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.23.4)\n", - "Requirement already satisfied: regex>=2022.1.18 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from tiktoken>=0.4.0->promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2023.12.25)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.9.0)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.5.0)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (5.5.0)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.4.1)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.9)\n", - "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.6.1)\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "[notice] A new release of pip is available: 24.0 -> 24.2\n", - "[notice] To update, run: C:\\Users\\sydneylister\\AppData\\Local\\Microsoft\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\python.exe -m pip install --upgrade pip\n" - ] - } - ], + "outputs": [], "source": [ "%pip install azure-ai-evaluation\n", "%pip install promptflow-azure" @@ -417,33 +151,9 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " question \\\n", - "0 What is the capital of France? \n", - "1 Which tent is the most waterproof? \n", - "2 Which camping table is the lightest? \n", - "3 How much does TrailWalker Hiking Shoes cost? \n", - "\n", - " context \\\n", - "0 France is the country in Europe. \n", - "1 #TrailMaster X4 Tent, price $250,## BrandOutdo... \n", - "2 #BaseCamp Folding Table, price $60,## BrandCam... \n", - "3 #TrailWalker Hiking Shoes, price $110## BrandT... \n", - "\n", - " ground_truth \n", - "0 Paris \n", - "1 The TrailMaster X4 tent has a rainfly waterpro... \n", - "2 The BaseCamp Folding Table has a weight of 15 lbs \n", - "3 The TrailWalker Hiking Shoes are priced at $110 \n" - ] - } - ], + "outputs": [], "source": [ "df = pd.read_json(\"data.jsonl\", lines=True)\n", "print(df.head())" @@ -491,97 +201,9 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[2024-09-26 16:53:21 -0700][promptflow][WARNING] - Found existing c:/azureai-samples/scenarios/evaluate-model-endpoints/flow.flex.yaml, will not respect it in runtime.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Prompt flow service has started...\n", - "You can view the traces in local from http://127.0.0.1:23333/v1.0/ui/traces/?#run=evaluate_model_endpoints_20240926_165321_352972\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[2024-09-26 16:53:28 -0700][promptflow._sdk._orchestrator.run_submitter][INFO] - Submitting run evaluate_model_endpoints_20240926_165321_352972, log path: C:\\Users\\sydneylister\\.promptflow\\.runs\\evaluate_model_endpoints_20240926_165321_352972\\logs.txt\n", - "[2024-09-26 16:53:47 -0700][promptflow._sdk._orchestrator.run_submitter][WARNING] - 4 out of 4 runs failed in batch run.\n", - " Please check out C:/Users/sydneylister/.promptflow/.runs/evaluate_model_endpoints_20240926_165321_352972 for more details.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2024-09-26 16:53:28 -0700 39604 execution.bulk INFO Current thread is not main thread, skip signal handler registration in BatchEngine.\n", - "2024-09-26 16:53:28 -0700 39604 execution WARNING Starting run without column mapping may lead to unexpected results. Please consult the following documentation for more information: https://aka.ms/pf/column-mapping\n", - "2024-09-26 16:53:28 -0700 39604 execution.bulk INFO Current system's available memory is 10048.08203125MB, memory consumption of current process is 372.0625MB, estimated available worker count is 10048.08203125/372.0625 = 27\n", - "2024-09-26 16:53:28 -0700 39604 execution.bulk INFO Set process count to 4 by taking the minimum value among the factors of {'default_worker_count': 4, 'row_count': 4, 'estimated_worker_count_based_on_memory_usage': 27}.\n", - "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-32)-Process id(43356)-Line number(0) start execution.\n", - "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-33)-Process id(3604)-Line number(1) start execution.\n", - "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-35)-Process id(8800)-Line number(2) start execution.\n", - "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-34)-Process id(21616)-Line number(3) start execution.\n", - "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-32)-Process id(43356)-Line number(0) completed.\n", - "2024-09-26 16:53:38 -0700 39604 execution.bulk INFO Finished 1 / 4 lines.\n", - "2024-09-26 16:53:38 -0700 39604 execution.bulk INFO Average execution time for completed lines: 1.01 seconds. Estimated time for incomplete lines: 3.03 seconds.\n", - "2024-09-26 16:53:43 -0700 39604 execution.bulk INFO Process name(SpawnProcess-33)-Process id(3604)-Line number(1) completed.\n", - "2024-09-26 16:53:43 -0700 39604 execution.bulk INFO Finished 2 / 4 lines.\n", - "2024-09-26 16:53:43 -0700 39604 execution.bulk INFO Average execution time for completed lines: 3.03 seconds. Estimated time for incomplete lines: 6.06 seconds.\n", - "2024-09-26 16:53:44 -0700 39604 execution.bulk INFO Process name(SpawnProcess-34)-Process id(21616)-Line number(3) completed.\n", - "2024-09-26 16:53:45 -0700 39604 execution.bulk INFO Finished 3 / 4 lines.\n", - "2024-09-26 16:53:45 -0700 39604 execution.bulk INFO Average execution time for completed lines: 2.69 seconds. Estimated time for incomplete lines: 2.69 seconds.\n", - "2024-09-26 16:53:45 -0700 39604 execution.bulk INFO Process name(SpawnProcess-35)-Process id(8800)-Line number(2) completed.\n", - "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO Finished 4 / 4 lines.\n", - "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO Average execution time for completed lines: 2.27 seconds. Estimated time for incomplete lines: 0.0 seconds.\n", - "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [21616-SpawnProcess-34] will be terminated.\n", - "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [8800-SpawnProcess-35] will be terminated.\n", - "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [43356-SpawnProcess-32] will be terminated.\n", - "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [3604-SpawnProcess-33] will be terminated.\n", - "2024-09-26 16:53:46 -0700 21616 execution.bulk INFO The process [21616] has received a terminate signal.\n", - "2024-09-26 16:53:46 -0700 8800 execution.bulk INFO The process [8800] has received a terminate signal.\n", - "2024-09-26 16:53:46 -0700 43356 execution.bulk INFO The process [43356] has received a terminate signal.\n", - "2024-09-26 16:53:46 -0700 3604 execution.bulk INFO The process [3604] has received a terminate signal.\n", - "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 43356 terminated.\n", - "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 3604 terminated.\n", - "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 21616 terminated.\n", - "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 8800 terminated.\n", - "2024-09-26 16:53:47 -0700 39604 execution ERROR 4/4 flow run failed, indexes: [0,1,2,3], exception of index 0: Execution failure in 'ModelEndpoints.__call__': (ConnectionError) HTTPSConnectionPool(host='ai-***.**.azure.com', port=443): Max retries exceeded with url: /openai/deployments/gpt-4/chat/completions?api-version=2023-03-15-preview (Caused by NameResolutionError(\": Failed to resolve 'ai-***.**.azure.com' ([Errno 11001] getaddrinfo failed)\"))\n", - "======= Run Summary =======\n", - "\n", - "Run name: \"evaluate_model_endpoints_20240926_165321_352972\"\n", - "Run status: \"Completed\"\n", - "Start time: \"2024-09-26 16:53:21.348453-07:00\"\n", - "Duration: \"0:00:26.215496\"\n", - "Output path: \"C:\\Users\\sydneylister\\.promptflow\\.runs\\evaluate_model_endpoints_20240926_165321_352972\"\n", - "\n" - ] - }, - { - "ename": "EvaluationException", - "evalue": "Missing required inputs for evaluator content_safety : ['response'].", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mEvaluationException\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[25], line 35\u001b[0m\n\u001b[0;32m 33\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m model \u001b[38;5;129;01min\u001b[39;00m models:\n\u001b[0;32m 34\u001b[0m randomNum \u001b[38;5;241m=\u001b[39m random\u001b[38;5;241m.\u001b[39mrandint(\u001b[38;5;241m1111\u001b[39m, \u001b[38;5;241m9999\u001b[39m)\n\u001b[1;32m---> 35\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 36\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# azure_ai_project=azure_ai_project,\u001b[39;49;00m\n\u001b[0;32m 37\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluation_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mEval-Run-\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mrandomNum\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m-\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtitle\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 38\u001b[0m \u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 39\u001b[0m \u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mModelEndpoints\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv_var\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 40\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluators\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\n\u001b[0;32m 41\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontent_safety\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontent_safety_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 42\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcoherence\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mcoherence_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 43\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrelevance\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mrelevance_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 44\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgroundedness\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroundedness_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 45\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfluency\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mfluency_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 46\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msimilarity\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43msimilarity_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 47\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 48\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluator_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\n\u001b[0;32m 49\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontent_safety\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 50\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcoherence\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 51\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrelevance\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.query}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 52\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgroundedness\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[0;32m 53\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m 54\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m 55\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m 56\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 57\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfluency\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 58\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msimilarity\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 59\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 60\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:545\u001b[0m, in \u001b[0;36mevaluate\u001b[1;34m(evaluation_name, target, data, evaluators, evaluator_config, azure_ai_project, output_path, **kwargs)\u001b[0m\n\u001b[0;32m 531\u001b[0m error_message \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m 532\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe evaluation failed due to an error during multiprocess bootstrapping.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 533\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPlease ensure the evaluate API is properly guarded with the \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m block:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 534\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m if __name__ == \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 535\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m evaluate(...)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 536\u001b[0m )\n\u001b[0;32m 537\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m EvaluationException(\n\u001b[0;32m 538\u001b[0m message\u001b[38;5;241m=\u001b[39merror_message,\n\u001b[0;32m 539\u001b[0m internal_message\u001b[38;5;241m=\u001b[39merror_message,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 542\u001b[0m blame\u001b[38;5;241m=\u001b[39mErrorBlame\u001b[38;5;241m.\u001b[39mUNKNOWN,\n\u001b[0;32m 543\u001b[0m ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m--> 545\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:514\u001b[0m, in \u001b[0;36mevaluate\u001b[1;34m(evaluation_name, target, data, evaluators, evaluator_config, azure_ai_project, output_path, **kwargs)\u001b[0m\n\u001b[0;32m 448\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Evaluates target or data with built-in or custom evaluators. If both target and data are provided,\u001b[39;00m\n\u001b[0;32m 449\u001b[0m \u001b[38;5;124;03m data will be run through target function and then results will be evaluated.\u001b[39;00m\n\u001b[0;32m 450\u001b[0m \n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 511\u001b[0m \n\u001b[0;32m 512\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 513\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 514\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_evaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 515\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluation_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluation_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 516\u001b[0m \u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtarget\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 517\u001b[0m \u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 518\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluators\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluators\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 519\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluator_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluator_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 520\u001b[0m \u001b[43m \u001b[49m\u001b[43mazure_ai_project\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mazure_ai_project\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 521\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 522\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 523\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 524\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m 525\u001b[0m \u001b[38;5;66;03m# Handle multiprocess bootstrap error\u001b[39;00m\n\u001b[0;32m 526\u001b[0m bootstrap_error \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m 527\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn attempt has been made to start a new process before the\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 528\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcurrent process has finished its bootstrapping phase.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 529\u001b[0m )\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:605\u001b[0m, in \u001b[0;36m_evaluate\u001b[1;34m(evaluation_name, target, data, evaluators, evaluator_config, azure_ai_project, output_path, **kwargs)\u001b[0m\n\u001b[0;32m 601\u001b[0m evaluator_config[evaluator_name][col] \u001b[38;5;241m=\u001b[39m run_output \u001b[38;5;66;03m# pylint: disable=unnecessary-dict-index-lookup\u001b[39;00m\n\u001b[0;32m 603\u001b[0m \u001b[38;5;66;03m# After we have generated all columns we can check if we have\u001b[39;00m\n\u001b[0;32m 604\u001b[0m \u001b[38;5;66;03m# everything we need for evaluators.\u001b[39;00m\n\u001b[1;32m--> 605\u001b[0m \u001b[43m_validate_columns\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_data_df\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevaluators\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevaluator_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluator_config\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 607\u001b[0m \u001b[38;5;66;03m# Batch Run\u001b[39;00m\n\u001b[0;32m 608\u001b[0m evaluators_info \u001b[38;5;241m=\u001b[39m {}\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:314\u001b[0m, in \u001b[0;36m_validate_columns\u001b[1;34m(df, evaluators, target, evaluator_config)\u001b[0m\n\u001b[0;32m 311\u001b[0m new_df \u001b[38;5;241m=\u001b[39m _apply_column_mapping(df, mapping_config)\n\u001b[0;32m 313\u001b[0m \u001b[38;5;66;03m# Validate input data for evaluator\u001b[39;00m\n\u001b[1;32m--> 314\u001b[0m \u001b[43m_validate_input_data_for_evaluator\u001b[49m\u001b[43m(\u001b[49m\u001b[43mevaluator\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevaluator_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnew_df\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:165\u001b[0m, in \u001b[0;36m_validate_input_data_for_evaluator\u001b[1;34m(evaluator, evaluator_name, df_data, is_target_fn)\u001b[0m\n\u001b[0;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_target_fn:\n\u001b[0;32m 164\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMissing required inputs for evaluator \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mevaluator_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmissing_inputs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m--> 165\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m EvaluationException(\n\u001b[0;32m 166\u001b[0m message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[0;32m 167\u001b[0m internal_message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[0;32m 168\u001b[0m target\u001b[38;5;241m=\u001b[39mErrorTarget\u001b[38;5;241m.\u001b[39mEVALUATE,\n\u001b[0;32m 169\u001b[0m category\u001b[38;5;241m=\u001b[39mErrorCategory\u001b[38;5;241m.\u001b[39mMISSING_FIELD,\n\u001b[0;32m 170\u001b[0m blame\u001b[38;5;241m=\u001b[39mErrorBlame\u001b[38;5;241m.\u001b[39mUSER_ERROR,\n\u001b[0;32m 171\u001b[0m )\n\u001b[0;32m 172\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMissing required inputs for target : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmissing_inputs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 173\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m EvaluationException(\n\u001b[0;32m 174\u001b[0m message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[0;32m 175\u001b[0m internal_message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 178\u001b[0m blame\u001b[38;5;241m=\u001b[39mErrorBlame\u001b[38;5;241m.\u001b[39mUSER_ERROR,\n\u001b[0;32m 179\u001b[0m )\n", - "\u001b[1;31mEvaluationException\u001b[0m: Missing required inputs for evaluator content_safety : ['response']." - ] - } - ], + "outputs": [], "source": [ "from app_target import ModelEndpoints\n", "import pathlib\n", From 2ccdfb21780b683febeedfc6d004cde7109d75dc Mon Sep 17 00:00:00 2001 From: Sydney Date: Fri, 27 Sep 2024 13:39:40 -0700 Subject: [PATCH 03/10] fix errors and remove 'question' col from data --- scenarios/evaluate-app-endpoint/askwiki.py | 14 ++++---- scenarios/evaluate-app-endpoint/data.jsonl | 6 ++-- .../evaluate-target.ipynb | 21 ++++++----- .../evaluate-model-endpoints/app_target.py | 2 +- .../evaluate-models-target.ipynb | 27 +++++++------- .../simulate_and_evaluate_ask_wiki.ipynb | 21 ++++++----- ...imulate_and_evaluate_online_endpoint.ipynb | 35 +++++++++---------- 7 files changed, 65 insertions(+), 61 deletions(-) diff --git a/scenarios/evaluate-app-endpoint/askwiki.py b/scenarios/evaluate-app-endpoint/askwiki.py index fe9523f0..203261b6 100644 --- a/scenarios/evaluate-app-endpoint/askwiki.py +++ b/scenarios/evaluate-app-endpoint/askwiki.py @@ -160,10 +160,10 @@ def format(doc: dict) -> str: # Function to perform augmented QA -def augemented_qa(question: str, context: str) -> str: +def augemented_qa(query: str, context: str) -> str: system_message = system_message_template.render(contexts=context) - messages = [{"role": "system", "content": system_message}, {"role": "user", "content": question}] + messages = [{"role": "system", "content": system_message}, {"role": "user", "content": query}] with AzureOpenAI( azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], @@ -181,17 +181,17 @@ def augemented_qa(question: str, context: str) -> str: class Response(TypedDict): - answer: str + response: str context: str -def ask_wiki(question: str) -> Response: - url_list = get_wiki_url(question, count=2) +def ask_wiki(query: str) -> Response: + url_list = get_wiki_url(query, count=2) search_result = search_result_from_url(url_list, count=10) context = process_search_result(search_result) - answer = augemented_qa(question, context) + response = augemented_qa(query, context) - return {"answer": answer, "context": str(context)} + return {"response": response, "context": str(context)} # Main function diff --git a/scenarios/evaluate-app-endpoint/data.jsonl b/scenarios/evaluate-app-endpoint/data.jsonl index b6ae357f..37f5c4ca 100644 --- a/scenarios/evaluate-app-endpoint/data.jsonl +++ b/scenarios/evaluate-app-endpoint/data.jsonl @@ -1,3 +1,3 @@ -{"query":"When was United Stated found ?", "response":"1776", "question":"When was United Stated found ?"} -{"query":"What is the capital of France?", "response":"Paris", "question":"What is the capital of France?"} -{"query":"Who is the best tennis player of all time ?", "response":"Roger Federer", "question": "Who is the best tennis player of all time ?"} \ No newline at end of file +{"query":"When was United Stated found ?", "response":"1776"} +{"query":"What is the capital of France?", "response":"Paris"} +{"query":"Who is the best tennis player of all time ?", "response":"Roger Federer"} \ No newline at end of file diff --git a/scenarios/evaluate-app-endpoint/evaluate-target.ipynb b/scenarios/evaluate-app-endpoint/evaluate-target.ipynb index 796bb70b..8bf00ad7 100644 --- a/scenarios/evaluate-app-endpoint/evaluate-target.ipynb +++ b/scenarios/evaluate-app-endpoint/evaluate-target.ipynb @@ -57,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "257fd898-7ef2-4d89-872e-da9e426aaf0b", "metadata": {}, "outputs": [], @@ -88,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "fbfc3a3b", "metadata": {}, "outputs": [], @@ -110,7 +110,7 @@ "source": [ "from askwiki import ask_wiki\n", "\n", - "ask_wiki(question=\"What is the capital of India?\")" + "ask_wiki(query=\"What is the capital of India?\")" ] }, { @@ -144,16 +144,16 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "665d0e98", "metadata": {}, "outputs": [], "source": [ "model_config = {\n", - " \"azure_endpoint\": os.environ.get(\"AZURE_OPENAI_ENDPOINT\"),\n", - " \"api_key\": os.environ.get(\"AZURE_OPENAI_KEY\"),\n", - " \"azure_deployment\": os.environ.get(\"AZURE_OPENAI_DEPLOYMENT\"),\n", - " }" + " \"azure_endpoint\": os.environ.get(\"AZURE_OPENAI_ENDPOINT\"),\n", + " \"api_key\": os.environ.get(\"AZURE_OPENAI_KEY\"),\n", + " \"azure_deployment\": os.environ.get(\"AZURE_OPENAI_DEPLOYMENT\"),\n", + "}" ] }, { @@ -176,7 +176,7 @@ "relevance_evaluator(\n", " response=\"New Delhi is Capital of India\",\n", " context=\"India is a country in South Asia.\",\n", - " query=\"What is the capital of India?\"\n", + " query=\"What is the capital of India?\",\n", ")" ] }, @@ -248,8 +248,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" + "pygments_lexer": "ipython3" } }, "nbformat": 4, diff --git a/scenarios/evaluate-model-endpoints/app_target.py b/scenarios/evaluate-model-endpoints/app_target.py index 516c6d87..759c03e8 100644 --- a/scenarios/evaluate-model-endpoints/app_target.py +++ b/scenarios/evaluate-model-endpoints/app_target.py @@ -45,7 +45,7 @@ def call_gpt4_endpoint(self: Self, question: str) -> Response: payload = {"messages": [{"role": "user", "content": question}], "max_tokens": 500} output = self.query(endpoint=endpoint, headers=headers, payload=payload) - answer = output["choices"][0]["message"]["content"] + answer = output["choices"][0]["message"]["content"] return {"query": question, "response": answer} def call_gpt35_turbo_endpoint(self: Self, question: str) -> Response: diff --git a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb index e6bcdb20..8fa44d27 100644 --- a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb +++ b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -75,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -117,16 +117,20 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "azure_ai_project = {\"subscription_id\": \"\", \"resource_group_name\": \"\", \"project_name\": \"\"}" + "azure_ai_project = {\n", + " \"subscription_id\": \"\",\n", + " \"resource_group_name\": \"\",\n", + " \"project_name\": \"\",\n", + "}" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -169,15 +173,15 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model_config = {\n", - " \"azure_endpoint\": os.environ.get(\"AZURE_OPENAI_ENDPOINT\"),\n", - " \"api_key\": os.environ.get(\"AZURE_OPENAI_KEY\"),\n", - " \"azure_deployment\": os.environ.get(\"AZURE_OPENAI_DEPLOYMENT\"),\n", - " }" + " \"azure_endpoint\": os.environ.get(\"AZURE_OPENAI_ENDPOINT\"),\n", + " \"api_key\": os.environ.get(\"AZURE_OPENAI_KEY\"),\n", + " \"azure_deployment\": os.environ.get(\"AZURE_OPENAI_DEPLOYMENT\"),\n", + "}" ] }, { @@ -307,8 +311,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" + "pygments_lexer": "ipython3" } }, "nbformat": 4, diff --git a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb index 8a807308..3302b63f 100644 --- a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb +++ b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb @@ -48,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -77,16 +77,20 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "azure_ai_project = {\"subscription_id\": \"\", \"resource_group_name\": \"\", \"project_name\": \"\"}" + "azure_ai_project = {\n", + " \"subscription_id\": \"\",\n", + " \"resource_group_name\": \"\",\n", + " \"project_name\": \"\",\n", + "}" ] }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -145,7 +149,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -237,7 +241,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -299,7 +303,7 @@ "json_lines = outputs.to_eval_qa_json_lines()\n", "for line in json_lines.split(\"\\n\"):\n", " if line:\n", - " print(blocklist_evaluator(answer=json.loads(line)['response']))" + " print(blocklist_evaluator(answer=json.loads(line)[\"response\"]))" ] } ], @@ -318,8 +322,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" + "pygments_lexer": "ipython3" } }, "nbformat": 4, diff --git a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb index b5a30761..b4229ad3 100644 --- a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb +++ b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -77,20 +77,24 @@ "metadata": {}, "outputs": [], "source": [ - "azure_ai_project = {\"subscription_id\": \"\", \"resource_group_name\": \"\", \"project_name\": \"\"}" + "azure_ai_project = {\n", + " \"subscription_id\": \"\",\n", + " \"resource_group_name\": \"\",\n", + " \"project_name\": \"\",\n", + "}" ] }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def call_endpoint(query: str) -> dict:\n", " data = {\"query\": query}\n", " body = json.dumps(data)\n", - " api_key=\"\"\n", - " endpoint=\"\"\n", + " api_key = \"\"\n", + " endpoint = \"\"\n", " azure_model_deployment = \"\"\n", "\n", " if not api_key:\n", @@ -128,7 +132,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -146,7 +150,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -160,19 +164,15 @@ " context: Optional[Dict[str, Any]] = None,\n", ") -> dict:\n", " messages_list = messages[\"messages\"]\n", - " # get last message\n", - " latest_message = messages_list[-1]\n", - " query = latest_message[\"content\"]\n", " context = None\n", " # we are formatting the response to follow the openAI chat protocol format\n", " formatted_response = {\n", " \"content\": \"I cannot answer\",\n", " \"role\": \"assistant\",\n", - " \"context\": {\n", - " },\n", + " \"context\": {},\n", " }\n", " messages[\"messages\"].append(formatted_response)\n", - " return {\"messages\": messages[\"messages\"], \"stream\": stream, \"session_state\": session_state, \"context\": context}" + " return {\"messages\": messages_list, \"stream\": stream, \"session_state\": session_state, \"context\": context}" ] }, { @@ -195,7 +195,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -214,7 +214,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -238,7 +238,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -308,8 +308,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" + "pygments_lexer": "ipython3" } }, "nbformat": 4, From fc46d6c8415ddfa8c8f574105be7ffae74117ead Mon Sep 17 00:00:00 2001 From: Sydney Date: Fri, 27 Sep 2024 13:51:50 -0700 Subject: [PATCH 04/10] small fix in evaluator config --- scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb index 8fa44d27..486a5405 100644 --- a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb +++ b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb @@ -258,7 +258,7 @@ " evaluator_config={\n", " \"content_safety\": {\"query\": \"${data.question}\", \"response\": \"${target.response}\"},\n", " \"coherence\": {\"response\": \"${target.response}\", \"query\": \"${data.question}\"},\n", - " \"relevance\": {\"response\": \"${target.query}\", \"context\": \"${data.context}\", \"query\": \"${data.question}\"},\n", + " \"relevance\": {\"response\": \"${target.response}\", \"context\": \"${data.context}\", \"query\": \"${data.question}\"},\n", " \"groundedness\": {\n", " \"response\": \"${target.response}\",\n", " \"context\": \"${data.context}\",\n", From 0ab12072e881253491aa33aec6fc9b6547fd1609 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Mon, 18 Nov 2024 16:27:12 -0800 Subject: [PATCH 05/10] init --- scenarios/evaluate/README.md | 41 +++++++++++++----- scenarios/evaluate/Simulators/README.md | 24 ++++++++++ .../Simulate_Adversarial_Data}/README.md | 8 ++-- .../Simulate_Adversarial.ipynb} | 0 .../README.md | 2 +- .../Simulate_From_Azure_Search_Index.ipynb} | 4 +- .../README.md | 0 .../Simulate_From_Conversation_Starter.ipynb} | 0 .../Simulate_From_Input_Text}/README.md | 2 +- .../Simulate_From_Input_Text.ipynb} | 0 .../user_override.prompty | 0 .../Simulate_Evaluate_Groundedness}/README.md | 0 .../Simulate_Evaluate_Groundedness.ipynb} | 0 .../AI_Judge_Evaluators_Quality.ipynb} | 4 +- .../AI_Judge_Evaluators_Quality}/README.md | 8 ++-- .../AI_Judge_Evaluators_Quality}/data.jsonl | 0 .../AI_Judge_Evaluators_Safety_Risks.ipynb} | 0 .../README.md | 0 .../AutomatedEvaluationAzureAIFoundry.jpg | Bin 0 -> 123430 bytes .../Custom_Evaluators.ipynb} | 0 .../Custom_Evaluators}/blocklist.py | 0 .../Custom_Evaluators}/data.jsonl | 0 .../NLP_Evaluators/NLP_Evaluators.ipynb} | 0 .../NLP_Evaluators}/README.md | 0 .../NLP_Evaluators}/data.jsonl | 0 .../Supported_Evaluation_Metrics/README.md | 29 +++++++++++++ .../Evaluate_App_Endpoint.ipynb} | 15 +------ .../Evaluate_App_Endpoint}/README.md | 4 +- .../Evaluate_App_Endpoint}/askwiki.py | 0 .../Evaluate_App_Endpoint}/data.jsonl | 0 .../system-message.jinja2 | 0 .../Evaluate_Base_Model_Endpoint.ipynb} | 2 +- .../Evaluate_Base_Model_Endpoint}/README.md | 4 +- .../Evaluate_Base_Model_Endpoint}/data.jsonl | 0 .../Supported_Evaluation_Targets/README.md | 39 +++++++++++++++++ 35 files changed, 142 insertions(+), 44 deletions(-) create mode 100644 scenarios/evaluate/Simulators/README.md rename scenarios/evaluate/{simulate_adversarial => Simulators/Simulate_Adversarial_Data}/README.md (86%) rename scenarios/evaluate/{simulate_adversarial/simulate_adversarial.ipynb => Simulators/Simulate_Adversarial_Data/Simulate_Adversarial.ipynb} (100%) rename scenarios/evaluate/{simulate_input_index => Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index}/README.md (89%) rename scenarios/evaluate/{simulate_input_index/simulate_input_index.ipynb => Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/Simulate_From_Azure_Search_Index.ipynb} (99%) rename scenarios/evaluate/{simulate_conversation_starter => Simulators/Simulate_Context-Relevant_Data/Simulate_From_Conversation_Starter}/README.md (100%) rename scenarios/evaluate/{simulate_conversation_starter/simulate_conversation_starter.ipynb => Simulators/Simulate_Context-Relevant_Data/Simulate_From_Conversation_Starter/Simulate_From_Conversation_Starter.ipynb} (100%) rename scenarios/evaluate/{simulate_input_text => Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text}/README.md (98%) rename scenarios/evaluate/{simulate_input_text/simulate_input_text.ipynb => Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/Simulate_From_Input_Text.ipynb} (100%) rename scenarios/evaluate/{simulate_input_text => Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text}/user_override.prompty (100%) rename scenarios/evaluate/{simulate_evaluate_groundedness => Simulators/Simulate_Evaluate_Groundedness}/README.md (100%) rename scenarios/evaluate/{simulate_evaluate_groundedness/simulate_evaluate_groundedness.ipynb => Simulators/Simulate_Evaluate_Groundedness/Simulate_Evaluate_Groundedness.ipynb} (100%) rename scenarios/evaluate/{evaluate_qualitative_metrics/evaluate_qualitative_metrics.ipynb => Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality/AI_Judge_Evaluators_Quality.ipynb} (97%) rename scenarios/evaluate/{evaluate_qualitative_metrics => Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality}/README.md (51%) rename scenarios/evaluate/{evaluate_endpoints => Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality}/data.jsonl (100%) rename scenarios/evaluate/{evaluate_safety_risk/evaluate_safety_risk.ipynb => Supported_Evaluation_Metrics/AI_Judge_Evaluators_Safety_Risks/AI_Judge_Evaluators_Safety_Risks.ipynb} (100%) rename scenarios/evaluate/{evaluate_safety_risk => Supported_Evaluation_Metrics/AI_Judge_Evaluators_Safety_Risks}/README.md (100%) create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/AutomatedEvaluationAzureAIFoundry.jpg rename scenarios/evaluate/{evaluate_custom/evaluate_custom.ipynb => Supported_Evaluation_Metrics/Custom_Evaluators/Custom_Evaluators.ipynb} (100%) rename scenarios/evaluate/{evaluate_custom => Supported_Evaluation_Metrics/Custom_Evaluators}/blocklist.py (100%) rename scenarios/evaluate/{evaluate_app => Supported_Evaluation_Metrics/Custom_Evaluators}/data.jsonl (100%) rename scenarios/evaluate/{evaluate_nlp_metrics/evaluate_nlp.ipynb => Supported_Evaluation_Metrics/NLP_Evaluators/NLP_Evaluators.ipynb} (100%) rename scenarios/evaluate/{evaluate_nlp_metrics => Supported_Evaluation_Metrics/NLP_Evaluators}/README.md (100%) rename scenarios/evaluate/{evaluate_nlp_metrics => Supported_Evaluation_Metrics/NLP_Evaluators}/data.jsonl (100%) create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/README.md rename scenarios/evaluate/{evaluate_app/evaluate_app.ipynb => Supported_Evaluation_Targets/Evaluate_App_Endpoint/Evaluate_App_Endpoint.ipynb} (96%) rename scenarios/evaluate/{evaluate_app => Supported_Evaluation_Targets/Evaluate_App_Endpoint}/README.md (85%) rename scenarios/evaluate/{evaluate_app => Supported_Evaluation_Targets/Evaluate_App_Endpoint}/askwiki.py (100%) rename scenarios/evaluate/{evaluate_custom => Supported_Evaluation_Targets/Evaluate_App_Endpoint}/data.jsonl (100%) rename scenarios/evaluate/{evaluate_app => Supported_Evaluation_Targets/Evaluate_App_Endpoint}/system-message.jinja2 (100%) rename scenarios/evaluate/{evaluate_endpoints/evaluate_endpoints.ipynb => Supported_Evaluation_Targets/Evaluate_Base_Model_Endpoint/Evaluate_Base_Model_Endpoint.ipynb} (99%) rename scenarios/evaluate/{evaluate_endpoints => Supported_Evaluation_Targets/Evaluate_Base_Model_Endpoint}/README.md (87%) rename scenarios/evaluate/{evaluate_qualitative_metrics => Supported_Evaluation_Targets/Evaluate_Base_Model_Endpoint}/data.jsonl (100%) create mode 100644 scenarios/evaluate/Supported_Evaluation_Targets/README.md diff --git a/scenarios/evaluate/README.md b/scenarios/evaluate/README.md index 15d1d4fc..257ea7bb 100644 --- a/scenarios/evaluate/README.md +++ b/scenarios/evaluate/README.md @@ -12,7 +12,24 @@ description: Evaluate. ### Overview -This tutorial provides a step-by-step guide on how to evaluate Generative AI models with Azure. Each of these samples uses the `azure-ai-evaluation` SDK. +This tutorial provides a step-by-step guide on how to evaluate Generative AI base models or AI Applications with Azure. Each of these samples uses the [`azure-ai-evaluation`](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk) SDK. + +When selecting a base model for building an application—or after building an AI application (such as a Retrieval-Augmented Generation (RAG) system or a multi-agent framework)—evaluation plays a pivotal role. Effective evaluation ensures that the chosen or developed AI model or application meets the intended safety, quality, and performance benchmarks. + +In both cases, running evaluations requires specific tools, methods, and datasets. Here’s a breakdown of the key components involved: + +* Testing with Evaluation Datasets + + - Bring Your Own Data: Use datasets tailored to your application or domain. + - Redteaming Queries: Design adversarial prompts to test robustness. + - ![Azure AI Simulators](./Simulators/): Leverage Azure AI's context-specific or adversarial dataset generators to create relevant test cases. + +* Selecting the Appropriate Evaluators or Building Custom Ones + + - Pre-Built Evaluators: Azure AI provides a range of [generation safety](\Supported_Evaluation_Metrics\AI_Judge_Evaluators_Safety_Risks\) and [quality/NLP evaluators](\Supported_Evaluation_Metrics\AI_Judge_Evaluators_Quality\) ready for immediate use. + - [Custom Evaluators](\Supported_Evaluation_Metrics\Custom_Evaluators\): Using the Azure AI Evaluation SDK, you can design and implement evaluators that align with the unique requirements of your application. + +* Generating and Visualizing Evaluation Results: Azure AI Evaluation SDK enables you to evaluate the target functions (such as [endpoints of your AI application](\Supported_Evaluation_Targets\Evaluate_App_Endpoint\) or your [model endpoints](\Supported_Evaluation_Targets\Evaluate_Base_Model_Endpoint\) on your dataset with either built-in or custom evaluators. You can run evaluations remotely in the cloud or locally on your own machine. ### Objective @@ -26,17 +43,17 @@ The main objective of this tutorial is to help users understand the process of e | Sample name | adversarial | simulator | conversation starter | index | raw text | against model endpoint | against app | qualitative metrics | custom metrics | quantitative NLP metrics | |----------------------------------------|-------------|-----------|---------------------|-------|----------|-----------------------|-------------|---------------------|----------------|----------------------| -| simulate_adversarial.ipynb | X | X | | | | X | | | | | -| simulate_conversation_starter.ipynb | | X | X | | | X | | | | | -| simulate_input_index.ipynb | | X | | X | | X | | | | | -| simulate_input_text.ipynb | | X | | | X | X | | | | | -| evaluate_endpoints.ipynb | | | | | | X | | X | | | -| evaluate_app.ipynb | | | | | | | X | X | | | -| evaluate_qualitative.ipynb | | | | | | X | | X | | | -| evaluate_custom.ipynb | | | | | | X | | | X | | -| evaluate_quantitative.ipynb | | | | | | X | | | | X | -| evaluate_safety_risk.ipynb | X | | | | | X | | | | | -| simulate_and_evaluate_endpoint.py | | X | | | X | X | | X | | | +| [Simulate_Adversarial.ipynb](scenarios\evaluate\Simulators\Simulate_Adversarial_Data\simulate_adversarial.ipynb) | X | X | | | | X | | | | | +| [Simulate_From_Conversation_Starter.ipynb](scenarios\evaluate\Simulators\Simulate_Context-Relevant_Data\Simulate_From_Conversation_Starter\Simulate_From_Conversation_Starter.ipynb) | | X | X | | | X | | | | | +| [Simulate_From_Azure_Search_Index.ipynb](scenarios\evaluate\Simulators\Simulate_Context-Relevant_Data\Simulate_From_Azure_Search_Index\Simulate_From_Azure_Search_Index.ipynb) | | X | | X | | X | | | | | +| [Simulate_From_Input_Text.ipynb](scenarios\evaluate\Simulators\Simulate_Context-Relevant_Data\Simulate_From_Input_Text\Simulate_From_Input_Text.ipynb) | | X | | | X | X | | | | | +| [Evaluate_Base_Model_Endpoint.ipynb](scenarios\evaluate\Supported_Evaluation_Targets\Evaluate_Base_Model_Endpoint\Evaluate_Base_Model_Endpoint.ipynb) | | | | | | X | | X | | | +| [Evaluate_App_Endpoint.ipynb](scenarios\evaluate\Supported_Evaluation_Targets\Evaluate_App_Endpoint\Evaluate_App_Endpoint.ipynb) | | | | | | | X | X | | | +| [AI_Judge_evaluators_quality.ipynb](scenarios\evaluate\Supported_Evaluation_Metrics\AI_Judge_evaluators_quality\AI_Judge_evaluators_quality.ipynb) | | | | | | X | | X | | | +| [evaluate_custom.ipynb](scenarios\evaluate\Supported_Evaluation_Metrics\custom_evaluators\custom_evaluators.ipynb) | | | | | | X | | | X | | +| [NLP_evaluators.ipynb](scenarios\evaluate\Supported_Evaluation_Metrics\NLP_evaluators\NLP_evaluators.ipynb) | | | | | | X | | | | X | +| [AI_Judge_evaluators_safety_risks.ipynb](scenarios\evaluate\Supported_Evaluation_Metrics\AI_Judge_evaluators_safety_risks\AI_Judge_evaluators_safety_risks.ipynb) | X | | | | | X | | | | | +| [simulate_evaluate_groundedness.py](scenarios\evaluate\Simulators\simulate_evaluate_groundedness\simulate_evaluate_groundedness.ipynb) | | X | | | X | X | | X | | | diff --git a/scenarios/evaluate/Simulators/README.md b/scenarios/evaluate/Simulators/README.md new file mode 100644 index 00000000..3eb1273b --- /dev/null +++ b/scenarios/evaluate/Simulators/README.md @@ -0,0 +1,24 @@ +--- +page_type: sample +languages: +- python +products: +- ai-services +- azure-openai +description: Evaluate. +--- + +## Simulate Evaluation (Test) Data + + + +Relevant, robust evaluation data is essential for effective evaluations. This data can be generated manually, can include production data, or can be assembled with the help of AI. There are two main types of evaluation data: + +- Bring Your Own Data: You can create and update a “golden dataset” with realistic customer questions or inputs paired with expert answers, ensuring quality for generative AI experiences. This dataset can also include samples from production data, offering a realistic evaluation dataset derived from actual queries your AI application has encountered. +* Simulators: If evaluation data is not available, simulators can play a crucial role in generating evaluation data by creating both topic-related and adversarial queries. + - Context-related simulators test the AI system’s ability to handle relevant interactions within a specific context, ensuring it performs well under typical use scenarios. + + - Adversarial simulators, on the other hand, generate queries designed to challenge the AI system, mimicking potential security threats or attempting to provoke undesirable behaviors. This approach helps identify the model's limitations and prepares it to perform well in unexpected or hostile conditions. + +Azure AI Studio provides tools for both topic-related and adversarial simulations, enabling comprehensive evaluation and enhancing confidence in deployment. For topic-related simulations, Azure AI enables you to simulate relevant conversations using [your data](\Simulate_Context-Relevant_Data\Simulate_From_Input_Text\Simulate_From_Input_Text.ipynb), [your Azure Search Index](\Simulate_Context-Relevant_Data\Simulate_From_Azure_Search_Index\Simulate_From_Azure_Search_Index.ipynb), or [your pre-defined conversation starters](\Simulate_Context-Relevant_Data\Simulate_From_Conversation_Starter\Simulate_From_Conversation_Starter.ipynb). + diff --git a/scenarios/evaluate/simulate_adversarial/README.md b/scenarios/evaluate/Simulators/Simulate_Adversarial_Data/README.md similarity index 86% rename from scenarios/evaluate/simulate_adversarial/README.md rename to scenarios/evaluate/Simulators/Simulate_Adversarial_Data/README.md index ff039bbb..55e6cd6f 100644 --- a/scenarios/evaluate/simulate_adversarial/README.md +++ b/scenarios/evaluate/Simulators/Simulate_Adversarial_Data/README.md @@ -5,18 +5,18 @@ languages: products: - ai-services - azure-openai -description: Simulator which simulates adversarial questions to ask wiki a custom application +description: Simulator which simulates adversarial questions --- -## Adversarial Simulator for Custom Application (askwiki) +## Adversarial Simulator ### Overview -This tutorial provides a step-by-step guide on how to use the adversarial simulator to simulate against a custom application +This tutorial provides a step-by-step guide on how to use the adversarial simulator ### Objective -The main objective of this tutorial is to help users understand the process of creating and using an adversarial simulator and use it with a custom application +The main objective of this tutorial is to help users understand the process of creating and using an adversarial simulator By the end of this tutorial, you should be able to: - Use the simulator - Run the simulator to have an adversarial question answering scenario diff --git a/scenarios/evaluate/simulate_adversarial/simulate_adversarial.ipynb b/scenarios/evaluate/Simulators/Simulate_Adversarial_Data/Simulate_Adversarial.ipynb similarity index 100% rename from scenarios/evaluate/simulate_adversarial/simulate_adversarial.ipynb rename to scenarios/evaluate/Simulators/Simulate_Adversarial_Data/Simulate_Adversarial.ipynb diff --git a/scenarios/evaluate/simulate_input_index/README.md b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/README.md similarity index 89% rename from scenarios/evaluate/simulate_input_index/README.md rename to scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/README.md index a459aefc..6a06c407 100644 --- a/scenarios/evaluate/simulate_input_index/README.md +++ b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/README.md @@ -4,7 +4,7 @@ languages: - python products: - azure-openai -description: Use the Simulator to generate high-quality query and response interactions with your AI applications from your data using LLMs." +description: Simulate from Azure Search Index --- ## Generate Query and Response from your Azure Search Index diff --git a/scenarios/evaluate/simulate_input_index/simulate_input_index.ipynb b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/Simulate_From_Azure_Search_Index.ipynb similarity index 99% rename from scenarios/evaluate/simulate_input_index/simulate_input_index.ipynb rename to scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/Simulate_From_Azure_Search_Index.ipynb index bfdb2d12..21cca016 100644 --- a/scenarios/evaluate/simulate_input_index/simulate_input_index.ipynb +++ b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/Simulate_From_Azure_Search_Index.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Simulate Queries and Responses from input text" + "# Simulate Queries and Responses from Azure Search Index" ] }, { @@ -13,7 +13,7 @@ "source": [ "## Objective\n", "\n", - "Use the Simulator to generate high-quality queries and responses from your data using LLMs.\n", + "Use the Simulator to generate high-quality queries and responses from your data in Azure Search using LLMs.\n", "\n", "This tutorial uses the following Azure AI services:\n", "\n", diff --git a/scenarios/evaluate/simulate_conversation_starter/README.md b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Conversation_Starter/README.md similarity index 100% rename from scenarios/evaluate/simulate_conversation_starter/README.md rename to scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Conversation_Starter/README.md diff --git a/scenarios/evaluate/simulate_conversation_starter/simulate_conversation_starter.ipynb b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Conversation_Starter/Simulate_From_Conversation_Starter.ipynb similarity index 100% rename from scenarios/evaluate/simulate_conversation_starter/simulate_conversation_starter.ipynb rename to scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Conversation_Starter/Simulate_From_Conversation_Starter.ipynb diff --git a/scenarios/evaluate/simulate_input_text/README.md b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/README.md similarity index 98% rename from scenarios/evaluate/simulate_input_text/README.md rename to scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/README.md index 1a13f717..d2e7f8bf 100644 --- a/scenarios/evaluate/simulate_input_text/README.md +++ b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/README.md @@ -4,7 +4,7 @@ languages: - python products: - azure-openai -description: Use the Simulator to generate high-quality query and response interactions with your AI applications from your data using LLMs." +description: Use the Simulator to generate high-quality query and response interactions with your AI applications from your data using LLMs --- ## Generate Query and Response from your data diff --git a/scenarios/evaluate/simulate_input_text/simulate_input_text.ipynb b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/Simulate_From_Input_Text.ipynb similarity index 100% rename from scenarios/evaluate/simulate_input_text/simulate_input_text.ipynb rename to scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/Simulate_From_Input_Text.ipynb diff --git a/scenarios/evaluate/simulate_input_text/user_override.prompty b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/user_override.prompty similarity index 100% rename from scenarios/evaluate/simulate_input_text/user_override.prompty rename to scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/user_override.prompty diff --git a/scenarios/evaluate/simulate_evaluate_groundedness/README.md b/scenarios/evaluate/Simulators/Simulate_Evaluate_Groundedness/README.md similarity index 100% rename from scenarios/evaluate/simulate_evaluate_groundedness/README.md rename to scenarios/evaluate/Simulators/Simulate_Evaluate_Groundedness/README.md diff --git a/scenarios/evaluate/simulate_evaluate_groundedness/simulate_evaluate_groundedness.ipynb b/scenarios/evaluate/Simulators/Simulate_Evaluate_Groundedness/Simulate_Evaluate_Groundedness.ipynb similarity index 100% rename from scenarios/evaluate/simulate_evaluate_groundedness/simulate_evaluate_groundedness.ipynb rename to scenarios/evaluate/Simulators/Simulate_Evaluate_Groundedness/Simulate_Evaluate_Groundedness.ipynb diff --git a/scenarios/evaluate/evaluate_qualitative_metrics/evaluate_qualitative_metrics.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality/AI_Judge_Evaluators_Quality.ipynb similarity index 97% rename from scenarios/evaluate/evaluate_qualitative_metrics/evaluate_qualitative_metrics.ipynb rename to scenarios/evaluate/Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality/AI_Judge_Evaluators_Quality.ipynb index ce75eb58..f950d1fb 100644 --- a/scenarios/evaluate/evaluate_qualitative_metrics/evaluate_qualitative_metrics.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality/AI_Judge_Evaluators_Quality.ipynb @@ -4,13 +4,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Evaluate model endpoints using Prompt Flow Eval APIs\n", + "# Evaluate using AI as Judge Quality Evaluators with Azure AI Evaluation SDK\n", "\n", "## Objective\n", "\n", "This tutorial provides a step-by-step guide on how to evaluate prompts against variety of model endpoints deployed on Azure AI Platform or non Azure AI platforms. \n", "\n", - "This guide uses Python Class as an application target which is passed to Evaluate API provided by PromptFlow SDK to evaluate results generated by LLM models against provided prompts. \n", + "This guide uses Python Class as an application target which is passed to Evaluate API provided by Azure AI Evaluation SDK to evaluate results generated by LLM models against provided prompts. \n", "\n", "This tutorial uses the following Azure AI services:\n", "\n", diff --git a/scenarios/evaluate/evaluate_qualitative_metrics/README.md b/scenarios/evaluate/Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality/README.md similarity index 51% rename from scenarios/evaluate/evaluate_qualitative_metrics/README.md rename to scenarios/evaluate/Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality/README.md index def15bfc..9ea9fbb3 100644 --- a/scenarios/evaluate/evaluate_qualitative_metrics/README.md +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality/README.md @@ -5,18 +5,18 @@ languages: products: - ai-services - azure-openai -description: Evaluating qualitative metrics +description: Evaluate using AI as Judge Quality Evaluators with Azure AI Evaluation SDK --- -## Evaluating qualitative metrics +## Evaluate using AI as Judge Quality Evaluators with Azure AI Evaluation SDK ### Overview -This tutorial provides a step-by-step guide on how to evaluate prompts against variety of model endpoint using qualitative metrics. +This tutorial provides a step-by-step guide on how to evaluate prompts against variety of model endpoint using AI as Judge evaluators. ### Objective -The main objective of this tutorial is to help users understand the process of evaluating model endpoints using qualitative metrics. By the end of this tutorial, you should be able to: +The main objective of this tutorial is to help users understand the process of evaluating model endpoints using AI as Judge quality evaluators. By the end of this tutorial, you should be able to: - Learn about evaluations - Evaluate prompt against model endpoint of your choice. diff --git a/scenarios/evaluate/evaluate_endpoints/data.jsonl b/scenarios/evaluate/Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality/data.jsonl similarity index 100% rename from scenarios/evaluate/evaluate_endpoints/data.jsonl rename to scenarios/evaluate/Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality/data.jsonl diff --git a/scenarios/evaluate/evaluate_safety_risk/evaluate_safety_risk.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/AI_Judge_Evaluators_Safety_Risks/AI_Judge_Evaluators_Safety_Risks.ipynb similarity index 100% rename from scenarios/evaluate/evaluate_safety_risk/evaluate_safety_risk.ipynb rename to scenarios/evaluate/Supported_Evaluation_Metrics/AI_Judge_Evaluators_Safety_Risks/AI_Judge_Evaluators_Safety_Risks.ipynb diff --git a/scenarios/evaluate/evaluate_safety_risk/README.md b/scenarios/evaluate/Supported_Evaluation_Metrics/AI_Judge_Evaluators_Safety_Risks/README.md similarity index 100% rename from scenarios/evaluate/evaluate_safety_risk/README.md rename to scenarios/evaluate/Supported_Evaluation_Metrics/AI_Judge_Evaluators_Safety_Risks/README.md diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/AutomatedEvaluationAzureAIFoundry.jpg b/scenarios/evaluate/Supported_Evaluation_Metrics/AutomatedEvaluationAzureAIFoundry.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2aa7e1f61825e4708182171eef61166b15a1c848 GIT binary patch literal 123430 zcmeFZc|4Tw-za|DvL(sBPK9hqvSl45d$Lo+RFXXj*=MAXeG8vd$SyI-R>IhYBuj*{ zO0=lovhIp_8J{r8ibaoyK_-Pe0xuXaE0&O!%G42%sRIyyQi z75syCryxCufu4Tv8+ZzdKdCPqdkHfH91Ec@B^?`LOYXXiL@=pe@dt^@4s2l)ulCc0w=+qGJNP-P`crUv%_fkNcQeSlQUYgt~(eJskrBJtM>3uEFdGa35miVmfq0 z@ytGM%d5;s{di8@O?$>7ezv-e*J=nap>*wb6e}Aazkr~S*0Na(%$(J`?P;^NaY9%g1`|Me)Rps=X8 zq_nL3`Ky}Ry84FKZyMV>I=i}i-u3nkkBojA8~;2pIX92}_I+V-Y550XZGB^txJBCD z*@KG?V)z5rzaaZBaB%^+=ouLq7@7CrqN5K27Xud~(-Fmeht60sU-jcYdg?9<&)KwR z)orZeN>+H@Yqy8k_#~9)Bnf+<{f6wn2Uyhq6=eSc>_6a|g!VJg0pc-mL2!u5F{UCj zOvsdw%@5!;B%3VK!c>G3ZZ@TUa)9cn#eTbxwnGNlU~ky2IG-IhX%AnYD5DL%*@bck zY|HZ5;gf3rn~?$N5^ebRd_!X9Acg2zk#qhd6B;L%x4klo^0>SU4c>TOYr&NF&xjKW z42=FBK>wcd<@c0FMfvQg$!x^>*K&B>3Cv=Rt%EfEII(yfEctujt*xH1E+3}u<}S4O z7Wok6`4v2OhbmhvE(A4DD}w+mQyWFJvA1CFxE)&)Y5Eyr-3(>RcQ^&-h1vc!Q*1qO zB%eLjzUlV_E0ni0YVY}{Jaqo#YwNVIhGj!y*W%s-7YrB;XSLU4|4Hh8@PYH<#8#0FpqGsTZt$Iqk|lCN#ikiu)cCh`BD&fKx2*8IJV zVtvSo_PKc%%8Q{y!=}`J@8h|R1=9>JVP zg2DdZV5s;ThPrzKV#qr^yY7#*s3#n@W5=!Hp=_K#YS4hPM7}<=K7`ST zkB1)PtpAMI{2qzPij*Y(`Ka?rjk4kUp`hUZ{gLrMD;MRnA}7=+o4y}X>dgmNh1biP zm$l32T|$1}&iHQvSVX?Q3$1?w(4>Q%djI*XGX>vU_ZfHv?a#-Q`oZ6LE0fM=ZTb59 zZU$}~w*Kk4Sd*0n<1X}v7KT$*fKoO*|7Rn{zv=mjNid&v7pgr0U#zi0xnD*N+Lg&j z)Atagd-k;a?{V8uKtddQ3NQ_Mll;K=Jm+*UfvTy6{(H=o^4p4ha)^g&sKviW9#em_ z-DJfEFd%9e!j{7Ljt#H&DkY+Sla|Q@@mLz?Kn_Y~1(E#9kIM=o*`)WbqOZE_#>RP} zCeYF7twis>Vv+FI-Sj}Vjm-ff;*@LGZ5DO2!rtB-W1x;z$;hSks?NaRH4JDj#{C~0{g>Y|GVXNJ+Itu@aUOjLJ^Tj=BNs|?AelX^nmC*VQ@|SkaivEx7NW*}?*H|9c4;=>c&I`i0aS z6KOhsV|^X~I6x)0k(neUqx=x8_4^i7RFGB%et_2&73DJxNJ{>hxHtK~b3Gy!7gkCF z{N(jzMIK5V8UT-(j-^Mk>6tDY>+bt!Gyeh8zr**x`CQy3y+;vusF*(90g};uwnw?z zYfp8VPDs;(pFGx)`4F3f%qVm%>+XeoMxNT*f=H&~Ju>J+dsH$89LZA)KI!uSrj!TZ z53@NynnM7#+J6Y}zx@51&$^&v?gjUBS${vw^AHj`|ItGlDu|bcN``Yx{u7z_GL$WK z_jxE8|IOMlS&x4|M{gx${fB3{XZ|LT&DJhN*al)|8~%}g$ZS9mApMxf9-#kRW`IB5 z6cquMFdM!8TZA4$wO>ac{oyEXWh-tVZvPPUe?_AksY5tZUAiKiDcFvc{~pio3C)w? zk921Wi;v_pH1Jrl0Z6ikGcd~FRRCcQCNg6BNteDxJx|PXiDa&XX*@VvhD)_=Ni#79aQFd}KT4s=t=NH7 zVsLT%Lb~_KJLIx~E!#l>t262%{?2iKg9gBp&i_s4!b*t#seZ`UMH~ly%A9mRci$UA zZYF2^*ZVGJ9xXnLMXuGg>a;n{H_N(Wai;Z=fO#GkJd_8FH7osFx~;2qq2hw4y8C3@ z_q4-)>+;|H1*l&9%)=cj$jWr5lhNm?{Uc5v_n<|XFz>Mg;QuTzZI40zpRS_fqI?D| zE30u$tipp_9#W8xuireHdx<94@Y-KQ$%~)hAn{_KNqJhNl*p_Fv!3xKXF$JmlN4!HnH@)*k9-=Obku+6)A02o}o)9EVlO3i-Nai zrKi8-XYSlfBiSzureIwC3wNP54T>gO@)6?TXAnEJ8wRaK(#%>}`cPj}`rvqO=Yd^_ zv4YgF3$d!TSGu%r`L8zPaf462UvyU2{grVyw{I@CeohzWH)NJp-~C?w@QoH$Y98D- z?*W_-?jeVWooq>&tMCzWtavRHCg&Ladg|cq*EgPKOSupB{~V6gAr-`vV8r{>7UW@y z6t$?T1?C|HkEv32uc|!EJw5qFrF-HmpJZ0x<)j{%!lEuL+`b#hybDFez~WG~S!wuw z1AOe~fbBZ=1i2)uoz?Kl?-E51<8;|G1isbW3bnT;-$af$pQb(ym|-299pFAccH`6a zr#I~_YjY#+^0LStO;hX_dxJw{A`Z032Pd?`h2+CgMIK|>NQo%oJmGlN;k4OPMpdFF zahEe_N@X7MIr5b7q1AxdO2dMA{Gt8# zbBNkiTsBrtg}|3k;H9|qLTl7Xv)x0oDQ;Zang0$V&}9>CNPP;QEfq?BHviU_jdFPy z7qFFk)w}WQ6Eo(tV}1z_j86;BzQXzj{?KJ!x?9-n<~5n&(IPe)$R3fd-TWe>=WXV- zvWsraX+188r!Z#J>~pcBpR~@D!ybg%$G+GwbyD^E%i5ssb@GpeZ@gS*V~oEduk|zt z40=LTnId$*LOk$^arRy}wQO4WTk!n%6POcp#AZ0N%=;JmwY)>H3&M1`cp~S-B8}~8 zF+iz~MBe;m)ybQf{7LF_)tB4+HkYGWMZANv{VWXp*4-jmAXtzUN|y4Pu!Tz_TayaS zvb)8E@VU?CV2o96F<}T!YWaTAT;r0YHN{V(vg0+hj$Npss+`c##oS8MY^X`-~+VYlK7{ z?bf{-MN^N}BQ+N!nJ@*Cotl3=saNY2g#SFM_`Goc^~7?HN!t#9R^g0*H4nfDLvub7x_;j4X}dFQ9L;VHHpE>bUm}GOui)QvJDs7T=N&hFS8$>dpAjO05;yO3bB15f zWxlPWG>q~vpMqgI9_C8pZ^p_u#6HY6+W5-%rA;>Z-Ax8#snU055b~KNf{h0Bh()D& zPp(Gn+-jHhF%IrdYa^e^K$dG>fgwF&=Lw42A{&fX+!EQy_FYKXVG8$LvngV1^-zb& z$gEew<2X*1_){PZb8Ln@KbawDXVfs8ZM)D0H{vfC7ksuTkC!GD+-Jr;$Cnr|WuKPb zE59*U=4lV11P$yg(>{ER745A;w_@rSyHlbyGOHdF*4ypz!Twfu4NkYQhF&tyZnJQ# zNhj-SmRYp=ujp#|(pW(xD3#!~3pww^(xSYG2QXr&*(W2CJ-EBB$9)>r>M^dq7B;C~ zM4?DF$`T-60mYh73fNLC_~=YKO<}xAjj8*wa)y49r~l-O7m|a5j)7f|d4p7pfoZGM}iJA6^o=89kRU!|60>806xQ^0+mkk(Z@^!!-JG z3^U1)(S8EO8SdgmU~g??j`M!xFmCrMo}juRJbDODZ$y!1cvjGYqSj(OitzoM#|I{*uS2=LB@S2#Zw*C3!K98e=^7~VtKiip$hLGjs zSo3a$ZKL8rYVsI-M?;8ui{wRMHX@bJIth2>h*X})e(qS=>DVpk|LNi#*TeUI$*sY$ z{FEZAMOu#+j4H|jA^RZgXTu_5rItFL!LLRnj!Q#3Qa(TtkhD zJp*H}QJ)dSCj@hx+8mtwN3QhC;o~Qi-!9>vIUN;rF|bSVJ1&vdtG;nGQs;5Ot8m8( zvwV{MrBU1`u`oHADwj)ivX{hfK#^y<4J*SP$$ZGKI@;7i!wW5k zmhHy5`kTd67*1Wl3iwZHtLJkzN}~m+6}yl-d&)$|oG(R!7<%;Qc#X563)71XfsdLo zM;=3248C6lq_p+J-TVmS#7R>0jmhrlImZEp3*zsMDwPGSZxbgQu711XKc$Y-g_Q)@ zFJM~XBpLps+Pnw#Eu5sDlJ+Ax-!7SzMI=sqoj7j<4IFIG>=mT5NnCt#t>qE54!k}^ z06D0`ivNY4Z5F>UhE`!;_fZawt{o}pK6>Lzfy}{ssp=BR?hLUY2@`b><%t}22x@=I z->&0OUq{Uhsz7|bPI+Pb-cOo1n@y2h38m~GK`I1in3H~BxSox?nCW7FUSD$Dfpx!F z$TgFYt6R8Ut($|*$W&T4yk4y{Ws-T`u5fM_5+L~-3~i7j)PG?gn^(VyGDy`p!J6g9 zcs1or4)l$782CHNfrA3bA$#Phd)<5|gF$l3nl}2{*h(*goBx2znWRLeRDqX0Fqy81 zZJ`H4HrJyC7FrlcaBZqomYW4B>h6gW>;CG=UJ|zj`o1K$0vWe9xcT7%-tVj^vNu_G5 z5nD)~Drq1w!>FPlKnfioe)FRK zbGd~&&*m!C97!quNjE&Kc$lfp`-%%=iX34a*zZf>Dx;N~ zdAuH5V#33}U|g3x#YBu=eA)agn(_XFX2_#12w5F}N3Ip)GB}wtLd?T+bWz6qJe(y) zBrdAR30$lV5tlh0vgmRJ?;pj|`_q}M%Y;M)Kc=x}R{or+_325RCnP;Y@-}|L)O|NT zq=!s&UEgn__%iVu^YN34YXn68#brbuO=r)bhU?+cI489ZVKd>5L+az2MIn|a;3 z+4M*v=~4rrSH~04*UMl-k!6$lq1IWi6;gC`Ms@fof`pWWfHI#d6?xc&tTh%iZw_Ce zAYJ>bbe;PVUh}Z~Fucuxm+g?&r6BB6e*Fx~y7;fk^5VKt!KI@ouHUhW4SyA0$Y#tS zuH8!?2{-i6MfaJ4C0Q= zqxO?#+NbI9Y7aIe*e#-94T#=n)22SkDKb<0^$V^zMh-Mki;3tVZBvVep^cvvnH;M`2m?9^c-jK|d_h0gCx;rKYJocsNcE1fk;Kk0hSUe>=hNDomQh2aNEf#cLV z(UAH^ay;PkdsR!%w4e3yZ#t&lclO;|&s0Rb$`PyanwRJ6u&sV*<{H5mbn;#4k+jiM zdGF$(X4Co>IJFk!iBsY+5XCKj8O2FCQc6nhCU#YwaD6*+;#ZD3WAxd3`^%u#B?GTT zUs4D^PG%Q6*PsvLm@tsI{uU2WO}gZf=I%3iTUc`LVFFW0Y-zAs+qgsX*Y?Q{H?gN- z3(Cdy_j@KA7h2F%5af6c3M1d^u;VWjKU)qJoi2T0LO|#}?@+Gp?iZ?N{oZ#!yffK& zK2qmZ@qAM@(VFz;MfhzJ|J;m_*Vve9PDz2}`qh!bfV+44-l@jkx!Rum!kt>7bHWS8 zpu6Ss@O&|N<+;+Tfl<3atI+uYeUu9C%`?;50I*Mp{`J5 zQt3d^S3O4d!Zu}-zNL#<>yjB#;`MTOsc+ynQ^b((F+9qc1HlL*{C8^7*k9Qx$Iw20 z63_Y@Q}_Glck|q#xHJ3;4Mh4-Zcs5Ta=}xUb$CqcDyMn*54(E4^&dO+2}LbMKW`CY z0^;Rg$rhyB`|s;z9?5%7qo-UO^MSjPVWQ!P#th?6FRs3haCZEu=q7PP{gSTx*{ZR_ z(vv*b(|VNYR+a6)Z^dC|pXCX+=mbx$0w0JzTzV`kE&W=KCT~--%oA4EQ#S-2?(kwu zu(p7jQ&Yqk>J!RUmtu+!9z)EGZtJH$@76SpO*wrd>Y{EmA=hB!vuo%_<{wVHcV}TD z?z_;TCeb2No^B(5qfR;f7svOfHO;SSo>XVNb9nsBvM%fJ8&o_}=M~8jzu1YCA4T|- zq`AmXp=xc7ZNUVU&==hL0%Mn}gEu`r9~oNW%p zDbsCJ{mIthUU^p8o$LhRZc9h)<O8uM0Ob|z~_!jF{38ltr4kOd3j=W zKTvq1nMBx>7&AekqoTLQukeG4Fs};;@g+~E8EO{^?m~AzGB4>PwU8ruqB;^wuzJjc zT5Y9y{`gwlA^FfTfsH9M&jq>bi7(X82)(u_l@b^DHol^bJd7MetRbhRcA@HcpySWP zM8bWKswc62o3&m}-G!c;Z|UnqYDs~pR;+RkKH=Y^Q6KxKMPJ8I`Q(4z_@ zX?{wMLMLp8n6L{O_8=rsZNfnIu91p?|J;b8#ro-rFjJ%vQ>C;hq!tj(W%%|fjNvYH zoyy#!BS_+$8_=9T*RUSk&80Slux{b6dD7PvWFop1{ucEBTO7J6e$rOr30?28NWOT= zI<*u&Tb441;Jr*ToI953RyE`J-lt@2HM1nT_``4(;Rw@*;4c?gmx}zI&!ET_$SEDs zI^RP-LRH@$h)Gyz3UWReCyzbvcKjT@{^bM3vCg%0JLCG{=47ztHiP3W+FtcE4nkF) z*W4=q2hPvGj!#4vlj+j$Ud#}lJ;YPPa4(9JcLi0C>hVDl14u^#u?ifM9u1RE?8k`x z2P$~d_390d#IY<*U{i7;wa4PO^N5_(5-m%zMY#Gn<;;8qhq!su$fk=6PpHJ8} zq_uWq3C&lF&lm*eoBQY8;;NeX@jp9vs1I-Thm?3U@TV5H9#aj*-mqvrX!LH8F~Xgp zq@YENdLL9!6hvtwGhBfK3|$)LvEIj50%qzBcC>2=?-1{8I|PsUB`Pj4k2M03!xC!< zsFj+M;Ir8FoI7!4I@Kzj{z*4ji`lp=l5#IsV^>8MmFYG!fz<~^d?qnMPhyKO$?-fX zXqKeqL_pZohCQ_s73&VcDc7wF+Ju~Wl#oidoyVSf!9SkBzB)xF=8_b!(uJgZ?st0Q z43g09#@EqeA#2mhhT`?O_jH@fx-fL@lkvRpeWZb+a3smGFnC1YE{6S^oJvLgV*R5F z5^*WFIgwcA<`6^NBeI7F{MCm94_6O2vA0+=WA=&VVfBm868#p>8*2KuT1NF~MJnmS zOo`}0H0d~;l7WbwA@b(|%|9MbvL<#?t96c@rya$Yq(xM?`@I=x$({?y>3#4cN>4B~ zZAoI_HVA~6ab9XTUVKV>qe5JOA%3tE#`$BN!`kd*uMn@|6{F|7&?)~m19j$63ue$6 zfIpAfR$664jei18QKm7%db}U@U&fqe2zoR&73?a690g9^fnDeoMFtk_y-8A@@U5z> zLVnD8za-k95cxQ&Mg-R*D$THAl8!k+J4Ty^)iQUPbN{eMFjC|z^k<)He1DcqHp8!b z`rhVQ8-9E4$K6;>8+y8LIYfoYbz&!}ntXa0&RORqY=~bd1xPnZHvXmJC>N8Z*&6@h zQF4F<%N5`I7N0uJxkYh+x@^vIeh2> z%ePOflfv_(D25h};L`5I)P9{u?ha$ZQFlylbsjgmb*Amk<=I8Sd`|s>ma|mgyQjpV z>O`icMzw5q9&mSr(WvCPvd_J0uSvIHc^-A;uhkZ4Vdanv$?CbVfb>^~=gS=#wy_BcXJq;kkQS zaMn+=MvZi>v4$Mw;U>>_9kQ3OX9cc{F(Yf~$i)gfrMpnTE+k6{q$Z%g?Yw}m94SXF zof=L^o~@#whluCQJlQ}YtD=1uW!31)&I#cOSyX#MFZz7#i8T)chpo>v3$ z_=l;pZ$rv3BqB3b7gl3S`cC#h^==||q0f3|V1w7tpZh46sTJX7F7ty^VHe+ye0NN6 z53JBMR}nZ>)AhY@mVELm$$}ZT|I%6pb%;_ys36+yLZHGvgj%^WirH}LLiTmg%HdyO zeHbCQ>mxpeDe^hDD}}T}qWZTZOfExkApb9&PmX7_Pahhq%WiYh>GP_|rTZL>DMrGP zT{^5O#CYn9!qITMSnC)iKlF6bjE33aTegnc_F?5c1=wQbTZ+L)L>G*cVp`a#Bu{(n z=b5*vVdVXe?2sDqjkdX%_W}Po&Z;5kp=ULRhd3p zMV-wRx{vnXXDGb>a~B@ZVT4I8-Hkpk?$mgt_4}5Ec-=id0;VyQB9^y^;Xu&*+dbJBBS0RreyF-tahbUck zHY~V&$y81x+g-0KV>Oz0yBF+`fIpB;U?4vIIQI#MZLAxSFZsmBYJa;aEt322X2(fi z5(hF4*-Ja{8n3ouTX6Af=f}(DgcT3WB|ok-na_PSHyZW9HdePZq)!K5x9D5yJHv=nA!U-4DY7G!K(bwMSlbqXT56PF z_ZY|ZW*54|BRu1=$?=k5Q-z$Mwv)XhM7i@XJ_AwB(y5QkE9!OkC#jG7%xhNGD3ygb zzqs7gR-D8L(1bkX@G~?vhLYtwa^Lk_u&nae&dnY-lPhzkFnQbKn2_q6+s6FV54#ZA zArgK?rBcd#yta3x&hetJ@-}p;?e??6TSX_oa57$J7!;S12<2GB6#n$@+r4`h-{X=Pw{A;uaZ})Q4Iu z!|CT$*V{GAZJ4Gpq9cQxn5?B8AEouqIm+MA zKH)*sDque?A`vM|-}yv+mOD#j2UbLz$*YNpEphM|6FZ$e+&v~4!*b-~CX>;S0k19@ z4$&bV??Svyas|PL#IbQ@wBW>}a22We{SAcLl!IO#-=`6`UtUq|eS4UQ9u`KCq);oH zfsFoovn5%AJQ_whp9pqVr&x4Q;yiTQ1oEwLD?BVkNBd$hb}r_nLSoiSvp|2*Cu37L z=l$ylvf_M`+)wWCZo(-%3G>Apc{NgzD|4ys;r0^z{E;GjNkoTXfww`gm7B;#;nGw{iXf43H6g z;d%U4+a=1)vA&%JOwe(JN1!maA&w|3v%)T?xQYE}{|V09#9T-LE)$yM?^I{ntg$nE z&h=_~Qf+47hU@9WJMr5yAcuiXfO8`UR2{T}SaxEf!?f!<4=43iEf{^!W~*1C^`eWA z;-s1jc(ur;Gw2^9mGw_cCrq`56?^C|KwDB!SPn#a1Tq`GQ`8jV>ygSW-?bO3gOnXVP1(Da~co#nXM<@znTy`u{oYj2_vcw1Tx7QN8Ez%+x4g z9ERE>wJ>ve?Xby_%Xq#xu-Y!fZ#ZX~aBJuGMN*y5Rj%vU>j)30d+Jzs^d0IWokO%w zxuT@yIWxnat!eq}HoTe4D7OW{;cKf+$6+S|d)BmCCXM?8r{@la2n<(-Jqe`LuRH|2 znbZ6{QU@s7sN!%CuiSq-`ymg@eo<|PbmQl#S0xuG`_5u??nc2rfPiuKF9bJdBF1eT zPwnhBo!LL#T(&=CO@jIj71@ub>}x3?Y~5RFIZTqej2|l$>UDiS(eEMKP*^TVIlwuG z$cDd!C4Hv?=g>oem!#BbMa`O?pZqcW4c@0kb{gKt@YU`-@TpB{Y#vxgOdO(?KZYje zvHzx{X^dvzT~3aW+x=}@?B7#5$Cm~>;L#5Dk;sGYxeQ+JRRJLR>Roj-?RLg@ROrQ^ zFQ(j^=nt^Zr5z2h+8|u!JSV^*{u1RYf%svZVF@r*blHgu>Q5e<*4ZBM0(__P4=3gF* zs)*t8r@GryP zB!4?#F)&sn`;wiGAU^qCzE}3n^KI*k69(DxYzAHH$J8==&ws;8STMgtwJG`*y`^S( zU%UNYzRvXBh}3DTxtDN075vP9%D$69Wd+!oW1V?K0T7o=$jzH4BG%GO^T?tvP#Xct zbXyn-PX^5l)&<|(1QBtu<#-0FO&tF1Nidn#{t>m5Lo_EzmpndEOqL|IT@^gp>!FEi zY~adp<_8urVO-v^eHS_yfKo(GHe_ufosquB3s3khXXbOhK}w){H3{a_s4C2csr~mL z%Ap3@1<J4cd>ETs)53c%1&eNAQ@cEyI^|XlHx@_Mabs zkX5Ag89lz2KOA;G7guFVO&$7aAuC>Yt0?5{X^1|_&j#A#Xp zjpM{vSenexw8A9mM@^g*R{Xu(Q&L z#Q4ZA!~%pM55BJnHuv;|nQ4b5sp6u$czPJs&O`E&NST!Pnfq67U(mUXEeAP_DY9?) z4ZL}ed7@%)Y1>JJ9{cAWrY{XyKUs5ZZqSYHlc2mcxWGw|aiIwzbc*qd?c=x8ujiT8 zr6q`@pEof^_s)=H_+n=9D0yy`%=h#K_#$*aKHw^o@+|?SpbG7NPpwHV7t=V|0 z0}15WBr4iwh|EAthval&8PkV$p@%78*rjM@N989`10&p|aOw=Oi$ZbVu7LCa>x)9E zX*-5Z^n?&Z@t{7#d_UrlLQM)qtw&)%x5ex;_`wr28122z(JN5rFoWEVej{X`V{JN5)JrVa@8 z^TbCTvvn0&8hxtz6rx;I8iAZTjo;!DOI`G^Fv zRq%z5?!20rlz;&P$#mFCLfpqxR%7wAOIU3d*L^KlfgU#4g?c3k7-}r{E@V91@|yG$ z(AK^OMPvYh#*l(8>?sUyEId(2YQ_rNhM!SQtor%G&UF&S(X)>JBwx1;+lR!@UZhI)vaDKA0-R*F(|1C-dp{*>6B(mv;{dF63q zR+vHlA%DCThBtx5HX6nw6@wExg)(0`IRzuz)sLUMcU0Z=cFPsmG+PnFR3I|)_j9O! zJ=clB8}35eA~XrqXC06NGceYL6-_JPsoC)qxr(L9hphfrQ6BP51fw_m6N|*)LKG1H zMfq>%aN^QRFFL-Pw$DJhf7=YOaT%PCr16qP1K>k9b|F7ZCQ_Z&idYL9gRTBR>@Ys0 zBTJd^C0c!C?gP02%UuX?HogMmCTeP4m)G~H$?WiO{-n`WVfvAiJ?5!gdBo@tIOUMu zoH^0{Zj&XJT;``*d0Y7>I1JH!;&MvG6I82QD{8g`xJ<`LkH~tCB>4FoZ))+Qrq|L8 zjAw#Q2%SH0Ke2#%rs56?Z;VJ(vrGK|nxmOouj`{^9e=)DPJ3FJV0B76E}s*iFRFne zEmRs6u0Yr~CNfxjSyYwKLm_1U)w^@LhjO4<6L=v6e;S|K3eKm%$!x(B$@^sa1E)3J z_A~faFEGgvI<*Qjn~oBw^tthu7mfW%cjCM>x@xOs-<`zCyBWXqT~KW*Na+E5DG^xm z`!3clULNzOXH0Ts;8Dx0?lr5gHFiNlo`HC_1xR#MG;ORGFV`B|jud^%j!g>Xvk88@ z@UmV@xo0Rvwq;wpVjAwwnJffb*~a(S5RmnAU5(p!vaY_|3StaTQyAUUj+ClHUZeHH z_l4_`{b?UH(48=`M<*yMZUH9(rM$QOF&AH8MN^?u%=%)Lx?<~q5BsUeQUmmkzZJ!KcND%$c zDsz${AXJ4#=mwj4IgLEe&RBB(S7I!xqd-uoV}4f(Z#u z)+SwO@1B%itz(z26E*m5znB|hp?*9plErd~DI5fsOyQ)Yz+#d@85boRbAuJJt8Ce(f^j~VqXouoi(9YgI%%FX(cj@w7yydFE(6K82$`{3*Hn(FBT zQxfNMzlhUp=)8#hRKVVDgA=Q>fys|wklM91yLRrQgDS9wEomLUj)0AEtCt~Bo|;C3 z!b3?6`J{U`_y%I$1MIqdpqrV)nksjNn2AK|37c{O$ln?0vjd;0x$q@Dpr)AZfmTOG zY7Np@Ve{l>uUYd4wWir__YB5Z>U%$oW~tmRq%nayb!)}1&~?l`c+J*bxEoc;a)iQK zP@mvYcE09I_qhKT-$1AQ1^a8KH_-a|jLSGg0=(A#!3m5PW-8HL!@JV2RHvNpu4`TL zkS}-I@q0hcq0^>#cjCxS;ZBGsM9peKlSw+uCTESXCKUQO4tcI;M6xB`i?B|-NlgVI zwykm1z+71Ku|9d}9^&KXn<0h9V&_wzyTqS$gfdb!KyB;2FUxYuEOzdmH|Fy~4_R}C zgza6UG-I{~kc#2#<0mlJ;Zvd7N)~DP)vrE%A5utgTd2#dFU)HN+z}nhOwoX0`KkL` zm;s>VLE1U=L25A^FpCJSpFuCX%ih7@nNNivAA2QVi0iq4Hfh>*%hb>60*tpp@;Rq% zuk^8~GpB-X3L;2XWDp5(YN8!#XTQo$9+{uw3==}RnX~@PI4?Ph!*xGJ{aoz0e+@NF z=SX8hWWnmPDR~V*QdX|jfMDQiKIhas)F&x&LEiaTsuikHW`l0Mhr+{7GvlA2ViLnm{b~68*AsFn;91p{hBa;e85+ z2MJ6N!;~NT6Q9Ko+elY&MzI*oDa>rj3|9bIb5MkIY(5~?qUq$1J89L>`-42G3gytV z9U-b}K4)j%RBYQxECmzRu5(=}@@fpTj_-Rb$rZNFKWoLM?BH zF>oOdZ-m&EpO)W+Uh>n%lbaQ2?1H<{VOUJw<~l%rzM-zz-ZBFu-y)J4(PgOat!eOw zig`iFrsl?}^_B@iE~TaC&DzG=KSAE5a?S~Uk^0xn#n;gez*G4opF8Ot-Y4NJKdy8C zJkd)`eY6tYM;#A*5Z4M_^iZY-g| zRNqW4f_F5s^t%uQ(aA0HTA;i?j{6x=IDJ`j5<$uguvrvysy!9rYCrPb5N@5bvrc2# z2A)@J51c4!PL(ntO=}zLFwthxXS~j}`4k6#FfL7dsj4UOKIjRN^{{U-g#1hj>S*-6 zA0W)ZZ&;gPKD=_(G0e-|M%EzyB*eSj1+;kt$*=GtIx5`P$2)81)3@Q}hT8`DA%y45 zXQzuppdHv%MDeZn*$0cNHF`zj1)VPF(Szhm<~BG&$8?`i|L9oWER5Aqqqm||-|MPOk)1s($sEjgf>PW{I8j+s40)(R|b<9bm!R7pPCd4PAYq#}|j~$qpUR&Gn;< zuTr*B8)+&VDyNLbE>4K*iSIx4^4;N0z*b;=#%NOhjS-3)p0;3h{zOi5P7R9vnz}B` zB9>4A{5X5x_?ats^!*mG$$_{))+yU&pnq~LDb);{cYtz)(EK005kz$WN79*V6Ad2~ zHett9uXbE=esWB4LIm)G_YY#FC(0YNW-w-92%B>QVkCpbrv)G0Q)ce7z-Cds|C1&kx0 zj$#ImbVRb9|7fNf$y>g6>PCh`5)v%f>uE1Y{79!SDGAB*JiHOfUo2K@smt6<4_-hD zqOnQh?SXsMGl1w2Qj5}(o@|JzeLeld>t5NUuGsR2FFskezM0IIT^3b-VX850ntSG) zZUfHgnKeI!c-Z`_R3IRt=G|@U+lNX$qc2Apih4~ogP#1B0P0(~8_tk;mvnp*6}ys7 z?$U4_cJ>%jF6T<6PH>04D(g!RIqa$*!4Eup5mcKd@({(I1S=vHcIfEf``vex*NQiR zPd~}m0G&6O;P$qv7Nomi)U&S%68LO+5)z-N7o5^I=G{F z${9I5sn-U0=+@vXrdZBNh8`hWa~L6}!)j#XfwxPFxtX3>a%)r9+KLO?iwhss#Tn{P zc@3bv(IlzdN-Uhui3F0LZs{tHrDT0#%m~Z=j6fybZ_)eBB`=l@9$t^9(ii1(_W7vI zm*U>cIV$6a-KUSgQAB5oewh|Atv{29>gkQkY>ZgA7O`V*i|3}XZeeHQ;0LpN5F(4| zr-l3iPdp8}E!8wCB63&8>YU#OR=r2SfvT_&5A_lb43#B=^T-r0D59gjz@AgLsQjUzny zbfCOjg+2J%2#yi;Hae)$;KIr`!&Y366b|+I=iK509bVd?vjBcDUKzMc%{SkhElzF^ z4b%j@=vYX4WX>3%f>X9&Ud*QO7=l7%UZ<~!Bjqe!=aJNd{q=D^Wy|#|EuqRG>_=t9 zgAYfZ{r2sEy7J(JLN}t$08J2YqrpIn*}Sj<-EviLglc7> z)(xvSY?V75F7G-A8ZR7tUawW+s+6st>p>z=Mt5bjXbctn{{cH)`cOC>{+=GSQs+dt z4ZgmCaKM-XpRH8g(VR-sneDii{`|?x+EFo=C$lMPk%kUrPAsaO1BOL+n7On~jxFI< z3*XyI(}jg(|MmUy7ITAN+VtB#kcXLvP3%I%Y-$=MC@^BSS<0dD_<6Z9(#>)5+3B_8 zgoU%bgpD*h=F^+#J9ytV)T9!z>MihKqbJIo+@0xbd~0_mp`3+L2~hlQX1d zqrCGDEZ>>D`B2NU#AB@Ydz*(FRx|in9ICOb;fgeoz6#Z$369vVld&+$36Qw?)<^Xy zh?MnE95|5eCi-Eu_PF$`n?60aET803#uv?T#bjQ}c|vSYMV?Uj$&ZgM>OaY+o4&a^ zDqra=oKymwVnd*w3Pfp1*2D-BT%{q-+p420v<~|f(Ox4lZH4YX z&;hwD&QpXF;48}xx!A!cE5;{-djdV#kR68*EIIoiEAfwgw+A0 ziQ*sAgl34I=_po`Z(Gj#XEUYKwb{nxz;vU~#`F0FZX$E-xg~E`<7cX~d*F%i1J?=J ziPPr1;g*eWV>F$36@2nz&sWp+Cp#Va;^@m}exWEMWkf@oVNv0;(wss(jh(`ZI6&$L zx^!$`v7aB&w^kqiZtBj{Ax(wTT5c+9Go9$q>+@R=;NIJmpsRB*czfl1g4sDw*rc5o zEjhjOJ?+w8a5~4NH=ru~*O0ErR;L0Z@G%cHaaIL8+FDH_D?KEOrb5fECUtxf;o3m= zCZH$K!~|+a%hAE-B$&yVFQwkSa?|&KV??87nenVKeavYsSEpcc@e(iM(RhfmfM0FP zn}idqYEmh#Z<~fFqHc`Yd^_xhyors`9}U(r3w4VPD2ImA5^t$XXcFnE8A!QsZqj%E zd6-bJJz3Z<G;ICcM^1>V2hzzP-0q&28Q{-KWt*q0$U<7;qwG#djW_Boy-$o5$dI zthnmZ6uTx)A+$%b)5KJ~`(os!pp&vj5?qOC_po{UDIsJpy`m`+zX_gLb2DqH{^gIH zg~T-B;2s8kC22!8X6q(uFnSa$y_Svbp>Sj7n>B-xILbW~mj~qA0-#qbhAy>g-u|lyx8aY>3qjrR&#UuW@xTd7 zm=Rfa<|5li7ZI4AuFU{tf2riE9v^XS@(RWKrs`rc6EfW^R5B$_ISeOfU7x&e-W-kl zCI3zElK2%pnVoy~zzOJEjmNx%ccCLO2RRdT_J48qoD4WsTYuYQi!CQQCvW@4QfElT z)c3EiBfdW$$8f=YFmXBIEdmk3U3PJ&wz)4T3v6^8)`D%sB+=jd@d7SObu8uqryy2L z$&99WdpmOx645biReNeYcIc?{4hB5{cu`ZYj2zSKO@gbQ=*je)q49{||`Vjo(Dmu{i zO+y0i)@9$svlVX@HSfOD;?CJonD34boA-^T$qxs7Mvj!tc-X6s;a6>E#5?;cvJUT0 zSC?S>vFy&UzD{HPN;PajheZB(o-Kk8%z@@`VA^(D21adHqVvvvPh7S^?;h zcxi?KaE6Ha2MtbQ$6^k+tLFKR1!~KBd71huimyvtZU1$fC&HM%#R)juH)-!@4k77- zA9s%$|CqDg2|XOzdK{uD(Ksa}!ylyptMX^bhsRF~Ce^Jl`8Jg3F$ z5$_Mr%Jgrp`nXq>-AI*@ef(Fn;AugL1D>RLAk#1RN{Uxei9*>pl_Ju}d`fiog`p9v zj7!b0GpuUgtV=Pzi>p_E`X&{9x|JRA{Pf;wT#)#vMX$gJK5!2+L^qsW>v&^wieDIL zf%X%7=5h1L8wt&--Wvw*p?&h6h*&U&Zm+w)eg?Q12fCgGTbqw6%O6$q;2R!v7|sk*>{p}` zjde;ejvnp|YyGr?Fp@Mgh3bVMX%Ayan-OowhRDI&$f3NNZ9MYX?Rw`UMu*?jH~qEB z<$Tn?De?QoEGxsUS8`wUH6q~xEie)@vG#v(_nvW0E?xU*5EVfbMT&qBl@0N(3p=rK^ZYjr1zgNvHvmcqX83-FrXJd*1W=e>mrS zh>16o+;dM^v*udYx`r4*Q1Hye%h!>(OO4;3Z{vv12-U6|31hmXYr-&&uhwg(>Zl5C z94>^-Q*T`+h-0L0;#9`Q^>HswbD6SYQwI8ta@o>!xbhj&t~l@76#+0e=HoO(DX0)PdSQYHF8~+Bh|g>vO6B#E!%77Fe#9__ky1iWw_*6WhaS-e~X8pho8s! zC}E@zjKb$zg@=_@4S7lJMWy}6pMJ^jwNR8kJwfXTPw6V6XeEes2(QrjCmih*8Cw#` zU1>o)7#mWPB^l%smz6o{$T%jnmZuT9M_c)JRZL*64Km%`C+WR&c$ zESmbO%0W>~iLcw{wYFIC<(tcI^^7ty%{XnuO^-dk!-UDA`(R#t;@Npv<#QG(<0`jf zFb0_IaBy_})lPzVw(Cvq4X$!e4tD;v1nOmBIg94&8FcAhDPaAfdJChPB|-2~qi{NW z#;TLZ;V-?9yM#Mjo~fR>ecSHL+uK~FTw&^n8sK8!Q6w4#tQ~nTmSacbYBvfT8Dj%1 zF;3aj(}9?dxXU_12#B+YqVB5*b3n!QRq3o9!Bb{=wZSQJ@7p-Ho~qPBI&hP+>C2T1 z6{WQHr=Bh8(q-8lR=fhTruEx~OQCP2+WL-+2xlfw^;ka`L-428bfj8dY5AB*`*6V5 zz zj2uX0XP_8|moV4QBgbOqzKuio<4xeO1_Z^bwx`u?h+HLI>(jICRVw>;5$eJm3GDFLr;=E@u?1+UKK1i0`-K4I&--^n8 zg@rOhQ=<9z>bCpQJRLZsrltV#(j~Zq;pj+E!mHu-;X6k>N|`r3T=vfN^fXOf%(BjV zF>liFAmH4?M+RXu37G&Ogm;Af4_6$#`KPflKw&KgM z7kdPud>;aCUw>nQZ)$X$=6Eql-=Jr z&67S_0h?z-d~s~T>%znEPcKqs=DC-|VCmMviMd7Wm$t^$2EY3GWcI4bMvR7BG~*_& zN7u3AG6sW=qGGk_Zh9-DJIeVemzzn{&0G8)I{5uKjRvFUuu2i4f2PSa?6*b1xnA(; zTj{4k3$T%N`0LUJ5!9x}!%-}=KWLNmPRla)^evmJ(r9)~e&R~ti;+^_ z!4k3l;1*@G;;qY?w$~!hu`d;7y=8Cpq#lZsgPugzidR#*&W=FqQH;}G(w_lXh(;G% z{I)Wlu6X#4bGOfReF6jJ+QDuwQ3T}}RsENniH#iEjakzoCmYwgwH}9`zr}EcYWEq4 z_l0-g7HNPhb8gfw7-rM9+n|2Llky@?#mJcMPvV|I+?D-^<+DBJd?vlFxqCd^G> zAD-B_NbkB|8MmkTId1=0bzW9gQ>oM)&8FyRk}0RJk@aEY0~o8fQ+xMn%z&N72>ewW zFycoD?8G<%Y}Af!{H@rVA`yC_u2W#0gI zWY?`7bq*LL{b+CcVb8(K7F*-|Do577{_ zg+=)ZhMsv}T2gbfZTn;zv@Fa848)%w`L-Z(Rlf|#;z^@M;B#o~_?lB`9O=;fk@sD~ z%;uYUbAIk+`djI3gQip;l9kv6d!|73$nYe10Z|j<<}U-BadO{jT@;1f4*x0C z^oIru88h8^?OIPksqjHoQvi#75BoG<(wjA-I1gP!s{Z4y0Ffe1X#obelAHjYVpyk}2|8kVdjDlL{aC%Rv2yTZrKn z=-@q|@5$fVegS7!bykVtK7fwF7?F*PEOOJ_<*udjL)7QzeERvGmI7M+b~q4H+3ty} z?aDw>2aH3$)M@3!ChsE~WEr8I3C06x`J!bRxUX=xXU?f$?ge1O%4&za-IU0TrEH+f&gS z7>XpVZEYo3_Zw|^)+`UzQ*2*COmQlcYb0mOfR zW5o~LdH!c#lHa#89@wXVc-4R zf`CMpA8n@s)QGH${+cM%N*z8sw)Ov(YTQmQ$rh{U%?#s++&Umav@>^FA?({CM=dy5 z;7Z~iyw}!s3{Zs;yiyR*>iFRPC}*hBJ+C(qCob;`VqP;dY;1=4!dwP7d}+h4e4}}} zSDE+Ku4*@|tT*gVs9wt$Z?I6ho&Q?lM)hHARc(X*9s|_8)VpT-D_)E*tM5@l#Q@a0sIz7?OLyeXdgU*&9J8}wEAz{}LYt5k#J@OBxjK8aBrN(fXUa3i6}sRTTrKGDRX01E<<;w`OD zqOU>KtY}Cxf(?}dfP7DCfe;0}d3cjIfd&K}GsXcUN3aiIM4-Uqobg5Ck>NeaQ5^@s zGKfUh>ErOoR_UL&FvTcgV#&8wm4zYOrLh_$M&_M6Cqiqc5?pzI-5Eq-c7AW2c%<#O z-_ylykpjOxs=;Q14Kn$GjUYG}c#`d(`~5-w{5kL-Iwblw)G&CEBj7>Q{(6wb9}kkU z^B~!35Z6os_h0o11}y>7ZQ96<;6eWGWx$vTH0=R<76$Od=y6cP?I#92fRozfCtfE7 zF8)?S&?U&xQ`=p#TnpRjI^KV6Yfwe>_HD>*{aY15TUYAuo{zqHu{>z>J`=I%+ zg8}}}!SL@D)F%&y|KC0} zAEqEvyE$>V>(Ipsy>yTDvBd+;TOtU3FBrCnUVdc%bSeIIy?xDskl{0m?7rl&^Vi3& zT^Gz?V@v%ub@aCKnDQ;BaBxFM785}xMnt@nPnvCQ2It%q6J8A$0s4U#V>7pMv%V1fzHa&^9iU_I>R{!FAB!8N0SWZo1msxD`K5*X z^ScH_^}-yM2CkPG+-g$1c&mP^Y%IHV92Lf#SKd5o>3`}ZdohfO>1oZrJOjF|DNHg! z7Vk1T?3=(`j9vF#c~_JtFnsD{NVKKndi-FZrQ_hUa8Cr7*5pa^|B9ij4XoGS&B;rr z?b(m~!y0YR{y!{-elftY7Nl3xI?9%M{9|lbdS|pd%?RKkjp|s=&I7VhoOr zB7KIgs1ttLvlC$dU#Gt%hisQ38`kX~?c=wiY^WJMMNC3Y#p8dNSNU{~-!9~!Nb+c# zUGZ`poS>eT^ORtTnXQH5b+03r*d<(FuK^-Qiqi;+AdnB+I9T-tIB-8o?)kl>2(Xz& z%)}$;-N;d0vhjDC|B{|WcNkRv^%{t!{q+-|ncDj3NP%tOhp{ioW?XV@(+C1GIPt1T zE7&HlDHV8-m%to9nFfZFm;~D61s=2t&30AoGBv6EK>@?9i=h23U|}Rieb9b@heT~R z58C(gx3}PzwrlV1Da=P%55SaD#AMiSzd>%(*K2?sk>WUl{PA3M(caQ968&)Bksme7 z*{&IEROqkoR*;*@9yn31F65`>B_2rxtu?Aoe%$*t*l5z{#1(Z9I$*p_!1{6m^JXk) z=do(6G&Xr<$&Zy!ek{O_eO&R^{TImX+60O&;#F&`;D0Y!y5{?{roWd&{#g<t7Y>hpS7q6gQ0`Wi+GPR zD&@QZ0($>fLqq@Fllg!Xkqm2SI|#&y?o7|7;B(}wJb6tzIxe2b)_b|AZOTK632`zPgO~rX7~~Is)R&4)cK_u{loRUEGwk=+HOYl zc^?^zX+rHIuyxMcR%*QTj79WH$Fa-yF_%?VMzHkL9)7godF3bz3#VA4NaJEq0*}wU z>{0v~nc~RD0X#V8ii6Mjp+gAQ4ompDF%`R97KzfUtq`xTlQ_oPKpoKlXBLaRUEO6E zT$|oB@zl&b?dh>IKBb(enjLs|Q!I%7Tix{Q9{T$aWWV`THS&oDKU%CwXqqn60 z8gFH{;^a8MMEOVe0(WEh&*AE{0LT*l$U2ih?*@Fp|6CBcGuz0u`qy%P-qOnK3&O!5c)4O2pF542l{Jf?-T{2`frm>oAqBO+w>n(c-w^7gJtLt zn1iK%%|Wtl+pg=MhH0Hf1%jg^Ps=~;()MIWuBca%8((*4vj3>6-8`7*muG(&q3x&o zTTyLIssM6NlP#F!PYbr)rchu($u{MO37ri0mX;@5rTwHp%p=$jb4h-pq?)OCSI(bC z(H1zkN@TnGw{pOG{y!@RT8M1AOaC<8D7-4nN*Fmd_}86n{`Jnl!-rXk08_s2rzxN9 z2U~=HSq-*-^8W)R)BKyE?796;i*+@$!wpD(HJszAV7%&`m&nCr({2Tb>w^E5x|Ql! zu*iPbPNirEaY3MsEl|8u)Z2ab^xTJwRLP7JiBwKRyHQvJ4_)GqV3)Hjt+DyoCKSAsp|qr{-_Oss2x<}dFpov z2JF@u06HQay!NVV@)e$X41ust*dK4uHoAXgQUH3hCI^{a<-f!8* z)YtT3cS)(Qq!KHiuGSo6SEc`2Dc~iWr6Ipyz#={UVWrvh($WC45}0+-lArlgcc^`IlT0s!VZW#zISv{ymc3Oi@-w03d52qnJ*a0Dl1#Rwu~ubRZs0R+=XN z*EAC1-q8+P(`p;?^X7x3lUMVf6I* z_jw@QKa}Z?XI-2&?KNc|TilRF`9s(E5y(v~GdYg&?Puo1J(8$Y!vThQsh{n|gb&ok zBJ?eGi7~B3eiHM8HG|=^_Ul^Hri1_d{wNR?=O}_aCrp8H#Vj)o(C}Zl+AdH2Jwm_4 z>Ti3~b^weHz|Ob5DRH1DcJQJM~vy1*7;#2kMRyA0;56AW$1Tl1d;` zlaBNvZ?C#yw`Eg!K1}w4sJH$_LsI8E}sXRB03xTC!BEWwIy@9B9m{{gL-kB58;B!oUNSF zA)n^%eQf4R)V3~O==@;aKOAA%pf^@|HE|&+e7WK~MC9FOA!huWkMJ?U;a0nZrG8nM z93sEn!!Xq~)G_0Zb7>|EKZEz1O|%w12p=Ijl^H&y5YB5gp%X<;_4hGS<;mn;P|jQr zLNVRC^&RLbihXq=A)l^$GA`3S^n%TDK*ObM%&QABSA@T#}~wxjL1% zm4;)K3YQ2Nl#8_^i18VB$-^}hRfg{a8@21?6LLx?iFD32T{faQdT*-A{XPwPt(4O5 zzPQRY_QphK%^S-U!O4lpc05jEGcb=t4qtR872`^(#%$n?F&T>nUAAJ>ZHmv|@1ILe zLER8|)C_l$_vKfBAJ!`Un)wa+c_aIq(;e7AE%&5nD0Ke9h&6QY4326&w!2_cUpbjz zhJ^>jYqvEpJyIM|vcF2Jk#umcMn^TP!Hj~+P$un-zIARclE#X7`tw`xh6tE~mJgqR zJU`3s`zNETS+E+7bo8zV%}F1-j`jIO7rx;JDRa~?k#2j$3=HgaSVa>9+JkSwUKdAQ zFMbjF1V3X2Ymg~N^;9dJeuq2CFeBo|j?uX0UgV&am#Sx(436YL$4hO;b~qxzkq+Cl znT%CatM-LOYErDEZ(Ga1=A&SJB{0*xPqE{qMj%oVXz{lfqRPb(j*7TvO zhMd6${rj1ZvC!FN>dg0)re4Z8c_*IR+3Du1;sPf|fy0v#GK_;}+8=9m6;+RT?{U~{ zYh0wVvVHG}HWm%;7Fs?-^?8m88tRcbwirMC62#9`%9A#4f}Xf|E%T+`>-Y90?ZwOR3Ad$@IohaHe%_gycU#q;HitD!4d*R` zCYiPhFO7B)_hsjd9iVD3Li^eJeu_cLL{Poj-+WiX{7ZVS9$r{hzlh4FIG%Kt$ zD;7BtaDDT<*Yx`I3YzLj^tF_-q>Y#{j|_{tXO#>lMs$4&p0i&uU)MPruc)!COa|dl z{xttFW&%{Me#MY*(7!JMYjgSmq~75rJGUC5Z;=L3s`oQh=@wWIJi%(f(M9gU$fE7& z>bW2OlgW?u;g8rI2mjV};68M~<`AEeD_`wQF~Fht+m%1Q@E-$zmaaagud$ziw(M}r zSEK&?NtyR^NiJMiLB(oYVoK%R&FBiK|9l$Eyr90V2)le35ID-|CY#D%b#a_b@mJ4` zh&79b`$gGSh`?uc9^r<=>QT{_SS8GLKC`j^weYX#fgEVGh%%_S-S{TX@l@ABsQPG{~dCv3H2S~eHe)o8%MI&Sh(A9 z$oMY$%&){#X@x|V>+jun8+GiMV(7BZ+*vx_hsK(p__|_4#y?a<1diTSV^vSTkX_s^ zq~X78qESB_ApRXfx0+T_DkPNlcz>L>m!baV={aP&_?S|?r{gF*09mm5B~{fgM2+E< zEdJ}ID+8I3)!p|f#?$Y`>0)*B(JYOG_H{k~ygpZya&cEK_i~=_ z5jSz9z309QyWN6n;1XV~)e;DHQ@aBL?@1_E+LC_MU?eR_y^6G)sa{?2^s)=9lV*2O zp6~ojtPp5!K3msRK76|ZHmjKuB-kbcI1J>3rT_f;Uo1oai=yC;6bd3BkqKgS1`ruR zpl_EqgLbE5Oi z0hUEm1H)0=?w>5JX3(Bl86Ses4W+r4E%S5&R%F=}D7HVFz?*!1{&m2!)%^jev!$*N zNmC>don|@5ku%MgvTvViczq;XLq{h?e;zN*eyaLWsEP6~9QaKSZoJe;4Ok{QsJZv+kO@~DS=}WdTRd&;M)N{Mzr=GukN=NJe zB_wDoP8kSF=Lz?ax(O|aXVF&u-3I1JNkXYqOV7c=c(DaltY^);`L7N}(b=97sSL%- zH?~iw0;|#fpXbv2bf15|>-)P_kTf_)n~6b2!=y+9%p1(#u6(9?Lyj!y`{7IfC|LIQ zuY!VFN-BRj@%;RNKOx%pBLz=hc{{3pr3)g)4K`N_jPloegb1+Ip&5Of!l53X=WVZ8 z-mc5QSRK^WxXZMHZ5oj@0T`1M)VADg# z(OIT~(yAj|?HlNm3u?aV*Tbo4-N(P86aB2!i@Hw6lBzV~THeI%c{XO|3*2ZBE7EI+ zr+T9HQmglB?@8}BQYl#bKvm*#7@swJO7>Xyq;THj!Z(f|651oCr#IZp1_j5sZihy7 zMOj>+&D;$&+H4;653U(Ki;XHC$UQItds7C~b>uc`4qYUuc}$*t@t`WrZNW>biG|C) z&|s6MJ)$Vaa`+pjt!ozetjlyC>;9F3W3+9TDBQJ_ioq!O+lq-P{reTLh5~xa8{qEX zz#a$C&VC%t_V>Nj=N$^#_FsW~NFWIVUAlT7wZ*yuC0;Pb6k~$C$X7&vT`2*iHPvla zNaImJ)QJHs11<8mg}BM62YI#r>vae|q?GYxV6de@4=0Ky%{d^=6Qr$L2l8%UzQ7oSSuvx-0Ve9yub;velZ(DXB=J4f5&W5HXf^%6S&ALj`cK5-+e2)vx(Z*} zON~aFyB!&3T)buVA%nx=QANSh0~0IFrjBf_KK=zymMwY{Afmr~-FhA0IPbuomc65+Sj?a=bAz?q@mByi+i1yJ2URQI&uwnI!^!fbk^;n!Mltq z4%X*OZCe&)ZHYx`mOaw+3ytwb;83t|^_h(dC%2gS6SzB_V;B_EqH1Vrdl7z5e5&O9 zS?|UWU!*@5RtECvpu&TTc5XzNQ2nQ9rWXQ&=Wut2Dh0=1cBL({El!1t-j}&$LuFXd zb%RS4A$hfOweMo{ct`a-IDcoUkAF-mC5ke6b+@_xTeh5epjKEa#6l%P1UsD6 zR0Xz^>iUfHPT-Qp@0S>MV4eR^^p9`>kO zWv;V=Gx4tQzXiL*FG+x^hK{JYvv}xy`goiWxAE!ep5&l!B%><;aKF#(c}sANE=>P` z6AsZ}+svh2K%xms-O$HitHh)caa(7QDTFxbW0M*)Gy8MkllJX->cMKEZhFxH$;F66 zib{z`0Wsd!E1BQ$I`&nUhEgZ;HZ-s=>V>!_-bgn{_h1Sl$2F0vn?tUCN)DY>V+Hja z;=+~IwjG=@;z+O)lQNF8Oc>BO=B_I=p}6t<(Tj5;5KLJ+{OPT89~DpLNfw6a_>>Rh zFa%UVroS&&$Bc&)mU4s{MWDkWCr57M%pWvMo{HzxPlIBF3 zhGg+^=kt=R5^Z0(8<;Iuckg~+vHP&Uy{D9ZkJEv#U#k-lLakl6e&dFUv{=i10W!+c z6gtG>z-aDUW%^=e&dx)`bxAWu_QME##NWOPW1U4EHT&Iro$R@fPP|P!)6&hW8lTO6zWN_#WPUa zC#|~oUb@XN$72?%Rifbgw(OVBSd@v;iQ{%%xe5o3)J;$+U{g{fEPaFQ@_{oFe`g~8 zIXW3fhlGKm$UD3PAje0S0vR|au?4Z87af0H6a~>F@*o~13qbs>IQcim*WTg(EFJm7 zR&ql>`N&ZjUG3;31B~SJ3%@OIvlRg}OP(T;MSkA2K>!ZyLzvpYYR~uGwJ9#%|0Mrh zkLU)9BvV-BR_#oDDa^8JM^{VWr~M?G`+<)1NU-Na(o3@wUa`b?o5IkPoXPvCLQ%s5 zaqz&=fMQ7@F{VrfbWP5=^>>%I1i{Gx@;0N3v9eDEIc5r!ZOLA|4t@s`tW6jM;CW{OYEMQco-481oZP#$8X_>HT>%^CBZh? z)6uG5Rp~a%l2LceQKL7b)})_?Ma`KC0*zb}OLuV~=Q$zY+!&tI_V8QL{fQp8Y9t#~ zUZaIwatP(n$G4Y0GMQ<|-|otoH(#-ytAb@_=NNMEV#845G7<&|HxJ3sESuS5^$V|d z+iG8++8yFER85UzZfs-5ry4|Pm1EG!PZRGcAnb7pbV&+24^gJoX_(I8*Hn#a8qIw? zsTs4{#e-|nX8g|IAzlW<_7qhHjI+a>)ykjJskN?SWTfkTab=1uW>J?1Y}`%6JVia7 zR%Dp3;myEFP_^qEEUdc4`B4HllNG8sNFiM#F3T+<-_aTd#CD z>edO7uXf+q^~X@z0Au+Ii>V9J`1>>{0;FZto*+{|0|jyHTi#4hI=~@=`hi}5?AZVE zKbpZF9=5ZGKh~}Q{RF}ZeCZBf0L$+Q@N3e}l$R2a%DMtWd1Pj-u1ewVMP>CA@59#o zU1UE3#BUPk!7Ssd2MRd_Xb_j{jH+@s6>J2ulN&s&u%B>mKL}+2g5x&3S6drMcPj#3 z@!r@2{F6Z08r%^uU&&X0uzjh-aXhW)O3JZ)*DR_V1u15WRgo(Ax@v4J!yCUVX(sLO z(hW8275AY7Jk(NJMr#h;VZGivj5wzdL6GBM!ySc%ddjMCZHz9%#K-zXVxufda#p`X z&I!j_tBK{zSvL@-;{o&u3yy(P5iB^ekJ&h8UaeMGO?kn>$$P=HKXM6ncmOjzv=1LY znivp$*u+Y*s%q3}pr^mC4VndH$@guVb)Kw&$sY1x>#07ADD1UHv8Tsp=~q-gqkodk zOG&wSruOZ4@`MxmWi>mgc~igsJA`Rk_Bg6RKc>{kBf%;(>9Ug4l~t~To``Sff?WAk zmg8S?x0@34y&l#(ofEKEiI=@2wCY^y43OmKv$6p08b$f*(GMc}8jGVF2ZYxB5;WwV zqMoBhSKk5&`>@2L`PFCe?~8dt=kHfoES(ZMI?>E=&#<6rP5OoVL5JAQ(lPxxpLBvE z&M#+-Us~b8O((Aplia2HmrpQyUWB>yfNfhl7H)_aLCafYy^AT)S820E?h|;;$9X(^ zEBEZ#3o{qQI-hjDRkW^id5O_QxD*^y(>WJ^{-w^>l!WOaV%+?+O`GiP)eO6g*Vim& zSN6Rnv=^F?R@XJ^`_Fe0s77a9j=HYkDd`lKYGHXEIwLi7@5kDhtc@<;h;Wij9WQip z#PjiD(N$VZqSrJT$(q z!GXDeH&3CF#*n1B>!GTG#5R|ER1Sg;td=$qLp7og3DQz?DPP<43)NWWsqRBR1C)$K z+T!Y(wAq5ph5Y36g|DXAsX1Y~c7%J?Xuli*gCV802snD`!G$Q8wSriC0)4QN5#WV1 zj5U)gK+rsyCMl(;2u!vk?$FIeDCx1_$_#4Il8Gsyc_z;hdPLgLVFXTra9hpG?9V@C z^D&V6Q^&o%#>eJa-mmQ~cQb#Q)?Rt<&;t^=Z)?Qfunhx^{3w-e45@n?Qo@k@YINbk9aoCUbP3PxxrA8 zD>zGkiw%+cp>C7nsG8`Bv#|E+yPkrB(L4vma&DTv$Ih1n8 zX{On<$+T%B=3_UM*1mO*^pDD%G1s}DG4^%5O~C(FHcT5z+7L73zB7E?SK@u|&}_!X z>6U@7rl&O&J<{RwUhhk6N*iL#J)S?}md*UeyOHNHUtV2Hj3=BQg5qtgMRQ6#6%{M* zuJ+j zA^L;6uO>s*$yTsy&F=cfXk8-4>yON}`vy9z8~$+tB~)Gzl+a z_BS@d(#D+0pDyZ+9uG^uphw&X#NyoyZ4^`)sbLy5@>8x}<@J;IErjAqhw?UFe|^@I zsgTv~c5SM7tvCaDsg8Fr@%3U|D^;!R;m*}Kc1&zSjs?SO({~U0X3$@gNQ_h*6Jc%; z8F9Ui`PC}nW%UY;vpoyUkFKpMYUjdEA|_$Ind2q#NmKV+EgMR+E$r^F8VkNhwG_Q5 zy1Y3NUR}odimJN^Xs^_I@!-r^_k=cfs@apSGizTgq`tnQsXRbEbv8B6gXbipXYPyQ zWf*m9VP)rR_e&M}MB&OF=TapkcV{{Zf@7Xb9G4L{2xk|N@-)^E*~&r7oIK%fr+3_E zrPXr}mUol_uQj?ARgoYSZ%T|>Fg-OVmo%@yn=gK*jfJeqIdRG$ zW7W1sQ3M4D04NyprKn9gw=9wGtH@zT zrgqrMq9q8T|NYVxXPJswh20(6N4Jl2|KUbZI@#e#;V*St^+~X_mrmxtBOM&PTR^k( z{pofxdtoR3&};fH*ZzP(Tc3D7to;;5kM}~|1y@%xKf=-Q{V%TSQ zDaIDR5BHv7Juz zyY;HuKFWmkLx)m3?6pX98x4UI`w4wBU*2=ABBxgJJ%Z*9|0}-T(Z?Rq5m!yY>T#d| zyGVL43&roNA((g^>psF-m!az0c+KgK>Eg##?0`aOZF?w<-q)}`s2jB3ZWM$|qTSj^ zy8vR%47eRyUY`@_@l~Vjg6g017OMvlo@;)w3wuINc^PMG`SEf^!LY0w#V?ue%+;Ao zT$M5C@m;9=i1SbmYto9Mc~M!o#M-JQ8hNFAx@He){~+OV7d?r{Rf1g+*?RwA?+9u{ zV`i|rcts*a;S@Ws(d#1pRf|cF1?K(6>_(z|ic84u;ipqIh2QF%i%Sz5W7?yvyJ~Oc zUg%9s&}axvof;}$;Z{J{Fh&*WVHIE1((fA&-3v}ngis|h3VCrpGqXfBs8nvwRR{-= z9=6z@Zm}<@ug_Sl0D~+4p#p>c3%8p;#?G(ICItp?Ry^omavyK^^S>h={{~|)j@1-` zvxR;KM|8)4kiqYlwh<3iIS9kdMRMX7jT|||?Yom;PBisGmt$J+Cm^CzDE+I zKY^kuMi9Yg_^6SrO3#t4dG1%Q3pb)W#IY)s*_#{}*UU^LGWCjHs)USdA8fsIH6`tZ z;Rf!(O@|D)hPBg?i;k4fz6LhYyplLh5}m!!n%8xXujTU02nYTGI+JvPz$kl4A5(P2 zyT=SZ+>eId(UjmMzJi@;z6lqpw4)6@7SR4KEr*l#kkW3Tc#R%$SXrhKa2S3Ga^zbI znpIG*qfl>bkaxiuZfbUHEjGCq`B|)53(t2Bn8U=x3Fq~_Vc!zaAX=3JPBz2TpNuz^ zA0;p+4!IXuDY_@#L)KfkQf|f8{1|LE$3^-$>!W2Q{F@SJ!k2{ft{Ho3s8y9v?;*&i zGSRBMONAV}PdOmo$`jjNd@O=mQ9pFGeKu98)c>t^Ra>E$M!$P~!IZ3RV)7mOS=u*D zrF-ZZ?mC^_r3_Wry`}%2B#O%bjQYcavu=&eiSG6T^G7dh)un`ept?vz9L|xi_hRX) zyqSA3!&FuYYew$yo8V@ak`n8uIsLZfkO9CE+MeprJD zv(fas+*6-7O7DdQ9+F<3@tRtf?Oxw~NC`n}-|$ExRWl$6=u>*{xd-1*V(aw~#bOH| zK3;!)n0v72w7JK9jvz^jRT<^{rW)B#=jhOS>$3aiNtW*>4H|=U7CZ#)mH}0w^2j@; z=!iUqKZfSdA^P+EU)_ic=mTWGKXXa@z~7!dK> z&I2RoaDt3oKkr@E2Uy6tXBm^;%uc}30jP)#MC6X&%#dJ9X(5YJVXc zN7S1ke_})vlCrPbu&0P^@xV-k_2%OPc8-U?8Yi;{$keW)bCN?0g4_tI$YmIEbYLdI z`r;+K^@2f{WjNY$iC@SX@1lpKo9(mBW_=KDc)jsn_6ze=KT4SI?L3v@`hcfvgdhip z=#QJ5{c8=_xJw!1S9)&6;G-o*ouHS@SqM2zUeX@s#rR^+^!@u;=H^ybW30;rt9n07 zJV|@#?Nc^`_+Z_=9F_i|NcvQT@6tfUp@M5CH=^d92lMkbPNxLjcc@(Aa!6I?eDd<; zS!EdrNAYs^%$F^L9Kk|kHvUz?FG518*7-d4fvl@sm>zI|EDN03{*2f`IxqBszBV zwyxfOnne{leL~05l8Mypgb-L+SJ0Y-ciZs)&OjkEK9qs+0SpWy`GYCZLHyU90r`-h zmb^Ht{7d|QW``EG45Ir#X<;|qo!F;Ro&c8O&T7aIs;V4Ib<;@$Pl|r&u3M!;f~A9G z`PXOOq}l-8VOzdTZB0sRYmf^@Pt32Gcm|mLYhqKd&3mPQDa1&=6f+I|b>$?02%LDx zY)D{?e@5VbzDEMX%qyx*v<80%6(+xe7CQECSqm-PR33TiT&c}xal`$P8^+IdeGQ=+ z$W@80oTa{HFuhLeOgy~h$+nDkO{uVd^Wn<{NA0K@&emp&FL&#XJqs9qJyXSktqG`P z$#*rn6q_I8XyJo8^Encx1m!0jxKVnCxW|Ha`^+GY)nJg!a&TC=`kP$G0`S5aN*oFZ zA22;yW45|?ii($zm`M*YuLBiTG<^@y-N>g{N4*;V%Kc+>Oz*POrp5IihDmo{Rg}%Q zPsNF?bpxSQsmymAn2Vi414srvhipP!&s)tT$Z0AF%}^$zp;4%i$M96m#~^9uyxE@4 z_g-iMDzTb_)bgZHm3=)}m}wqETkiB4Dgv9Y(c7e@b2Qzuu{jW5X>Z){M%d)|hU;;%LGA573~ zF2}!+L*FDp983kb0R8%R-spDe?Q1=N4Dy3Y+N&xz4F;v~4#yL~O%R>w9FX%INu%Nk zVMucS3`i>03goPHd$KZ;YLJRO$bJrwNr#A3J^0pueB{R4a9~{p_FBvX**qbzFM?Dq zg7b8hU7bRx8h>uktyjC2E_hn@91dO!W5lZ~S&M$_5@Js84mg`Hqk?*lJRCII%X(hm zgKp$V_R*6y7X`wSldNNF#QU7m7l|Hng~&5-W2Hb|#ycyElSPDseFlk^<+6LTb5~}o zv_~#lAjAgdJNP(-o3V`R3`ja@K{hJ*0c)E2WTwNG;AG9b3Mn=lSrOr$AwjF}kTROx z&?ouNaiT&C+L+daG)LV%3-vfz9V~nfGX^`Pe5|j`gr;HX@-yo%?#HE(Zv&c>(P{!x}FkR8s$)eZ;5{`d$A=&Lw{au|x(`pmIhCgE3ZNH-2AjpSZWJbOpCxAd;fvPbO z>rt)$WV-e(I z_>^arjMS|lzf%5h;7e;IH9}8Sw`ChII6*7kpaQ{|x&IwxPTM)BwO%tHTL{hS$@b5B z*%1*E-3mSLnlqwTk^HX1`YnPf{^H|XnlhgO3g|ugW`D3$FiWi9Gp^D|r?6`R^;ogB zPfYlK`B3bgwm&q#eP{(711n_)#VCoqnQrTz1#G#Teth=7IVW z!bHqeTdapTHm1Rg)BL7+4^wmQx_44@A#r@eY^bW@?ePBl=sj8SO8ri!Zsw=MD@bF2 zkh8p=|0(E00#m$rYlSRSeC%WM`QVgcZlFkw>0SWa!3l<<$f}%I_*dlB+3JTC1ksYH z<{K%fZ>m>rg04N`3n=asiq;2!)1d9vF1N?ke;&Y~JnO&}`(g}|b33cz-I3#_C&=5a z>N~%-TUY69)6&z@q`>A!WgINZ9(PG=L=FQ_4+DqoL1Nz@~eEPSrG_|sN8bF96t9}!f0CfVFw1)oN zL;hR={?q%vA$>@5{l|<-PY6~ zlkCVJc8CIy^t5yai2Jr03uwLHTk_ZE=VWBYM}dVf1p%I!-S1x9fAZBJJMNzNFLw{^ zu;bMIbp-%J+bSGsttWm6F+}i?#R34c9KsJOyoblC^#ouJY&&&Vw?6OcVCCn7wEosV zJNsho&p=UTd&q734$)8QP)&eHx@%d1ecB3=2FT!K-H2U)EO81*O(b+3tlb%Fzi;h_ z|8YrWT&5hvA;^YGin^5>{7b&udod9_F=mzsge1CZ#@FC^x_RmjRop`CS-?{O zi8x2LT?=4ee>xI((f_=(T_JL%c0K86mphOW334e^$P64Xm9}4L=~;Hd(Vuz{{##Eu zuMEqbwK`BNfefrUS_Jpg%yltFJMrt^*E{{>C~ENx>G0;3ex%6!;c=p&S8p0#$h@-wd6 zV#k#|2R-k}%=YHgBJXJI>U}?oCurHw)FN#Jnxz9yTPRBG-tr0aoPC9dj|b-=Mh1); z9;+cOZ3LuL&a#hD?H=!<(Bb1(P^VI7K4KLAVDya0^Yr^`hnU=6%R+|ZQ#yH{jITj7Z7`^OE#v+Q( z2MAJKPEw0Ln=*6v#7JnKXIx6IAlIWvaUB+UK z<#k(yE)WJE>QJ#q5mzZSDY^b%bb~ZOJdvu#ZS)c3H7}{T!Re=&wdQV>H z?d?ZLldU<_-h4WJS~4unDgPz4Z^M}nNxn|B=P*xW@xg0;IOY(sd1G}Bf%fDMGw)9& zJV$$_r&@%STT0wSGL6;~{Y}r|Q*(t@Slj$^?`NfS_Rd+JRxVNYW@D^zyGK_dm{F}i z_o{0qTd-nS*G|x7`2x1LJ})?1QPbmP&$`JPCayHPxk$_O43=)(IU6UX*>iu;x#Lrd zW+BZ~bifqNM9waj{wEF_n47qWu_J_YSfR<+yX1wYEQ0#@YxzDkzu8R>c|Wp-KA%BT zEPbi2cu~rW7p(!kl+Q9YMr=gBouA4-H_99&hGum=SZ|EEQCw0{ImqSEn<}vHwXei& zvxtz@1;N+(cZvpCKD;PZZ?<49$Ws^Ne60_BltMAG3EBhJj-)*)+sZpX^isAHfb4pq zB;_!GPMDcifn*o{*}j5*I;$YbNmUt8J85tly=)|9zk;8|1%uW;fNKoM#)9A7*dQkc z8i*#4g&6_-&?4@2Juxia@YE&tLd>u(Z7*%fz?veEITI z(+ap4X`r5i3cE;r4E!twG4~5%tga!R0=l|mLpc*|Y!Z1obly375k=wcg!VwQtxF!Z zU7}ya1mXT`G0sPhM_wmB&_QP5rS7j|l8o2VRJ^-b6(I1mn_rh-}OFV&AxiER&#Q^Mkvc14bdqR zzb?)}Rq7sUVSY8><`EUpZig=)_mh#RopH?>rn0Nl$s)*h3QJp1Jw;2{i5Zvzw8``^E#auxGn=@*ODpFDX-CfJTI@S0(PB=D0((^|g6DYK zk(pROz)qnS@t-L5H4mC(bYN&Gp$UmT<}?~Au%XU-K&>ms!p7!fS@rC|{d3pP^Q2^| zENsG~mWXQwUuF4mF#&*ef*^`024C+=X&-W>w4Qc3UV7u!`@Pch7i7CvHPI*!a>7M% z8pu0rtRV%WZxNGZWg4+4jgrF7lRjEFvC+M3%06dLTJ9O(r%Ldqe7~um_w2^EKvzjo z&PA125OcCj^loT+)To;$#0dN_TGUNShT?Y~LS2=Hd9LcinmFHLU%zz6t%lbHft~L| z!YCJ=TU;cXB=zBNA-%YC@x7g;HmI~rm^Gtlp4TKSMA|SD53P6U8_GTr^x>1tVv$AM zD_!>Y5|p!WSz}T=oWaAy?)>Xf4gUg-(J#SXOC!?ZS5osk*zX|+^_o>yp?!k2E(ub2 z)m-a7x1yK=-?AX?*%P6q44Y41xQ|sLM@8l@jt=i7oG9&B<-5(g7Mam=K$h=WT!OOR zr_@Kd(EvbK7z8%RiBx5 zd2Cn`lk??C>G```Z>c0KB&b++-#fbxt&UEpv4HtY4h4z?NW=5{O5$Y8w~}8w_09OU zL<@eQccpqkv&mYS`zeY>qO6_{i%wM2B|Puv20eM5nRXC)7kPuXM)hy+*o>5v074g@lp8 z$LrR5ZiwYld80yy(Dsy?qQypyLjps1ba`TDkef2`F&EB%=qYzaFzdo->9YkQKn|kMTs<5q*)6C{E2y$3qeF4@2&*aHIy! zGcgz%$pvxUpp-cFg?oo$kw@sKUWNKKdwOdP%X7vxSleV(%(J;>2UJBSscki~E8jbqb@3Jo||1v8Ssh zJW*Kxx&$sD3OZEZu-T8;^iTLyswNs9|CpBFl4Zna17mkx_ThNWyvS$@9@XrOv(mRu ztr|F!%6Ih)4M;ZQ!ZoQYqq+RoLh!!-7jbVM4)xpq4-=&bW#6Z=6Dn)SDA}`zvP~s> zD8g9B60&EBB1_g3hU`lwJIOY-jBG=TeMXjvG52$g>i&E__x=6-zQ=Pse>{I^j;`6> zbG@(Yyw2ryzFzaiWs=G?vsW9@r|!^q;Cv)$ZmH!)0J$c%Gfk*s?0GL?6J6mq4MwQs z$u%Jv2h*BnQY6}|AHIy`3&YLmBp8ja(`QyC`5HZqT970NJJK@V^21xnx1bUd12jj< zHYH%!Txr^Cg3~HJyt`?pC!ZYUZ^-@#P`#sB0{FrL{zXh@>}TCVNOp?XswMPr zkg`j&9DciE&YgL}Xwd=zYcZrgl5f`Pa7le-%(G+TxFovio;%VC54+C|B z<*_ld2;b+fd2gczImSb~JZn5W7twWWe0e!|h@DbEzugtLu-ut$FI!e@1%F<-++7n#{PBvZIF!pi-^{*qWG2L;GX*{$6+2xhV1_}LI&uo%LN~( z;)v28q#y+;j%28V{tiPu@U1CiP+&wf#aES60h&W9+#up`A9JHUqIA19)-Bb3(`et| zP!pd!Ca;rEp#BNmpe5!3Ee|hV&->HP#`gSHnTM{g3Y0V`&z6>~_0Z*_fMK*29knJ0 zUw8)UU7%}$lbi)#Hhp}y3u|V+c#Xns!`FF>Fil1Hy_%Dzw+D=Mb{hpr4x=Llk3QT! zI##qQ)?+<2p|P(#0TKsvBxOQ!`}Bos+<27oIhl+%kEh10->Pbi_UuoF8ho?)X4FIN z(d34I)8ybGEh4CYK5?-HnhBptP?2#7VkdOu6ZFyJ2Fd!3d(h`+hw7iX-(fg!=GZB)7`d+368lL{H$@0myWmqzF*>b+YL9r7Aui47G$KM*G~8~yIo&1ra@7E8n^5&IPt0G;afQpN~2)W>Zx@$Mjg zqM!S5!V|bl8%E8g;p>%#Q185xPwI?P6&5zTN`V*u7{;_PzPSfGL6GAPR9P}i@N|w> zeu9W<_UW3B9$u=xcya0}vH$xQjv{rlr@843`r2|$PuKhqQOepya9ynolTkO7Ub|zI z3!er;^(k6I{n8^_RnC^MILq zRKd=r?G*Fu04YLC5mDo&Yb!lH6_brTfy8dj03{wKmC$#!8l$CK zt75~3Rj%9P=SyDs&&HEB%RFp1Vr{ICnwF)Vt#K+uV!qb{cEte>&@}7`Hse9#4#*UM za)fxSLTXP$xVgRWyc2D>Ze-!|H-l(EegSIygepQ;3mw1CUFlv|H_D`ojMHMM_R=Z84fyLp(x~EE zCfCGL5VP6Hdkn%4_XV~5{0iduxSf{bKo-r1S^R?!i~Htm>0jLX`wE*i1NU^7hMZ>T z+-7}@WEY0oxN6PXjwtL>xQNe=Wk_c4L2Z;@U;o_ZqZqc2K1ze{fKL}-_0;BR+9gy5 zfJSE!q|yv?I+05zN}pkn5r3}hV;3X`iDbW0nb0=#X?v(d4r(vg9W`X)Hk8o%8qrXg zv)lMIJZ(EdvQUgmUGu5WR;^ka!WPK#9wNk$n$G~eM?I^3Bx*r(2(*C-g7C6oyy;nf1C093HLeRhUKsL=m2WEd4{8N5#e2E{YkJ z`R%Z>ywb=mmgRF2u3jw)iY$V$D@ulo8ceiw`1inUCY1ualz{jlUhXA6q0OgRa42W6 znk{|2H)2|KFOLcLQBD4mrdy`?IMik4r?4D!i4+YdHKSMOkZ@Ihf{9N&d z9tZSe5sow4M*)bO-dBDFwi95q6$Wkr@+JWkI0U3MKxRsl)a}xPxbrWHMr=X}Wbcxv z=Kdt-5^6JVIBjp5iUJ{%DVJp}9!;A^6j-2xMo}@1X(v(}8WTZ4iw%%d%}5(r&vtxP_dmjk^=4X=fBdeBHrf z-c~c&sPa?b=@U@+i~w~LKvx4g=l_W#%q2rq0kKLPxC32(OH=z>gWPpA6#I#cqWVSN zki&QnA{K=dC{h`F(&25T_=(x*Lr0 z`4*CTcGn*_voB7cLN^(@3pZ87ugdt|avVRb=6jpidTjGUaNwIkge{5*esdLBrBI~{ zL~&}Cf`)}|<~OL`*OJsWW;EsT>F|dD#;BkCBzo}urYKUd_%|p9+Uf&h^dJ!@#K|Vb z2yy@7w*EJ*CG+dzh{`kH@O^tt8t>k2fJb zJza;IdQ38vpI=@Sw3tpv*~~L$m5wxWVb>wO{uc9HZiWeQcgoyEJO@OovK}p8>cVg4 zo%W2J&UqHkz<^IcTI@>Au6Jcp&9hd$x%KvwbIlT0pvr9M`-h)P6zm3q7+U!|8PVu3x+td8?CMTbvuP^173XC`JS*)*X9y!uE#y*~G zs#_}^iq!X9w%#b#zwS7@-lLe3n}XV&(Oht#^^CKq%kb1-75I#RT@Spe?h9ytW5yut zO~SlHCD$#5^SoPGzR?CoOGU4sk5lB z`z)&WtEM+jI?;s4(#3glTl8Oi+lZo;Ah;sy+Dfj97-ro_$#V~Mdb8XYl$k(l({Jj0 z*2HOMYQ`;Vx2JXx)RZBK^^U^xLn7 zT57|hA9ysC41e(^xHa&?Es`@0rr#M47Z3+@UUPj7D}tx#X6ujMn?qGRyQ3v+_yMs? z+6rpoo)_+5gan+&?|xpipvJ6+3s%N8&s9fae@G@BobERK59qg@~| zRpbKWP+jQNfgnB;RGSf2o z`j>CDSNMAo55EVv>}gFn@=4a=X2OZ`zSvb?ZssnBjZ?QM`0tyYRei5~%Wpi9G?*nx z2*)C+Is?>a{Sl0g6;u4BZT$kE;z2>#$bS*qcKzX$r{(^0k`y6y&wS8f|DooVr7 zO}W?W?&;SjsVFz{^DX1FYX@A5#piVMl5I5?bt)k5vX`=)8X2D*^_9kI;<*%i}SQDF~o z8f_MaC(b6c>@=u^^Uw=#Kb^S4yKyMz=an7d;>DY&>c}uzC_C|DNLAxaDTm>;u`jdX zx=KQ0#;QCo+y+=*xb)bw@LSOGq*~7Cx%w2BDkdyHe0nv{Rla_b$PQ*#caCxI$Ymfp zO^b7nHPf6kSY@1BXMbO)M6a*z68cX50Z-1pX;_fgyFDeMPqL{V&HGAt5PYp(H6VOCvYlGz-_n+nzR$B035*y*ZgxXA#giAXp-TGHm%Bil{eYlW-KGPn~z>v zTgXag=VyIE$ClTP(|#Dr9PimuNWe^5WmxdB<7{4Z>SmgKvA47FsK-3t?)?R1#vvnD ziK27wq&2Ip%Wqodi?4_D5cLh_VUnbN{7k?NLIEUd>+TF&NKu*yd!vq=)3(%dhT-=3m_p2y*j792hbn+W&ej@}T*_z3%{-(TlqAQk4BHf8Y|p*zPdR@>?nOyT-otd4@t0fH0hRbLlD=mf z!V0k1EdNAGz=YwtPW{;0w+Penlx2(l$B+gqi zVJ!2UUsd{wS?At$?e5T2@;#O9{O$cvDl_ZK8<`imWBYs0?clz*DyCZ-wLeI>)%pwX zKs5SDAU8WD`$K-E2=E)BVi3!~Z5t}Am4RWX+F>VG`jd0uW?Ei1Q#xdcNvX9p^Yo{G z`g;Z&6j^)&+0SPf2s=DW-gWJXSfyX$>93U$e1+WfHnI=>oPH>Z+;X|~mU!1?Li^R? z!`=0l#dyfvx0ohwyqG&U9Y&ms`F|o06+Qc|vuY=D(N5*f0J;GciK;To#>N>-a9flo zUm4A8sy?nAHM7?gd_DULlq;+qS|va_h8HR*TyExSVk__Ob@!(b6Ev7pxKFto@eN(a zJsD^@Ki(cEZ93cpl}jj`)bnL%l@Bq=)r8alLWDfCgo!Sawp`l3D6BzzGgNAU;!>9P0i{>2 z((qVU*)gW;aX~*;ZhhkW#=4LVh)M?LG6JD3X^O0EpZD`~d+J`Qt|wXREZj=AI;jtL z;8+9D8>%q6EAWre4;{rn$X-lum`n@QYPu1Fg<&kvU523Lg)=wG7SUj8J54i~zI*%n zamx2d2Eqhx=BE2e3wUpvQPa-Y-tob!(n2O{4M_Zh7OHfl`<|YSz#%k(R^RK1+_O|; zmfEtZjL1v%yJMdF3wP#+cyBbZ-up%uo^=Rq7#jN&{lLY0*d^$z`e*|jvHdLM9(%9$ z8cTyP)lMRD{)fQm04DUb+`GA|OI)HX^HXEhD$H;y%Q0oU2lEK#$?E(K=}YgLVybse z#!VWC9l}ZXIgo~-8|UE;`&}S#94k*A*}*+6?p1CzdX!S2pM5rc)LMOO*>-BPk>l)U zhNYJ6_1bvWI9saV99z~OT`jzPqs@}e?d?b3{`e@SNgL3YL%K@x+Vnya{nbgcB-2?4 zbtKUm$%L3h>~r7HUP{R5ae%(E*WcZc>dWj}rFZ9{OUCMuTGTzp*iKsX9L^GVtNTd zy-qn~1B72ds_B3&gq)=Y&B0`15k3T7cXm1%uQ*?7BZgPey}m@%!EJU=aMT?9BiV*KP6pd1{Y|pB`!}kdOC}6+GT$ zW6{GH-L#}#p6u(nZvt;?px3>TOEm}pa~78_A*ZkmfUwPP5gFtw8=*{FPahFmo|9E2 zEFM=v^)C_Lu=Mb2=fM}$ljINIym9x0U+5*#H!vfmNwqJq7(|r-j8*13E;6)gRl~nj zd+cJCR%oT9XT^2q@p=Uc^QHg@?kK@^mRCvRPN{cJugCIGftw{?ThW~Lk%T!t=C)FP zCkvJALYeHql2zj~GuAow)h?epY>WFtTjyT(Qm?J8P5w~6Cv>D#aiil-QJ>KpadCmB zcUM4v9>;blm~=3c|A8xGF$t#tTzQUwz14hMot)z<0qm_v#Nr3+yRs&eZIJHx+feYO zw=YE502U0y;!$MC*p0s+WB#`Q*Hj{6A#E7|oDX;ua~NHVj1Rg=L%~}VL)0lm$jVHT zqK=yYAC?mIFeNt_5c8=6=8Ja0CVD>cI`X zKX-FltgIL`=sPt|63=QJa%dh3+3NE%6_oHaD;3)kH1n@auXe(U>6Gl;27ttpYw8EU zGr)VwmJ~beNU8Uall)g7V>lc^?rr>ix*v>D_=mHkJkOz$og;w?dh+j5PxS%K!|K)? ziOl`bK>%6MJE;eQpwz=MeED~G0+lwoZewguUYuko$m(C9SMm53(3J^rl15s$$#~db ze^nITzbBgJ7QpC%-guE5V{{#8vQJ8T2_VPOBD{|Z@^eEX9D$}6un?=-1T+AS>6n9m zME^(PpMzHXhCk$a!bf7pk}7UQVi>4HTv9`Kg|`9D8`^w43lh!Hx3OvY;-9n_85xaNh}V`w z+)=~=UWFI&si^e88w~_Y7>oq{dxX4zO*~cU(p|wS)9VAuZ-?{qWSCWtI!UX8mPatd zxl4%rY2>ot^g00*O8+NG}s(w;W{iHY;3%X zHl{>GgA+#;)G3r0TGYwM4l5Q?<3r4t1{uHGuE4L?-}YLa9tDH$#Eh?h4L+1;0Sgll z{c@P-NeG?-(-`teBwr@wc=aoQs7JvTO_w@ff{IAeKBB<9EbKAa=!i1DlVG91h?3*S+J$BF(I@)e{vshmVdujpES^%nQ%Dpr&ok)T8xaNNB*v?XXH_m9sMp73J)3C?EF$N4XcJuwQDy5XKA_}NfA zplF1G^pEEM{(^se!z$~4UBUnNZ}|WJ6U@_2s|pMAydWc;e-*WIQxlyWDT;tz*@0Bu z$RCR!`)B`~*N|;Q_Qo~YKQ@AF5uyR9H!WW3-xk51jKu>vX$&4ZLHFBA*rN`{D7KfZ z3=bM6=|J8am}Rnb8aX`;wiVz&DfOiPoMcazAAUZrkawoc@zAJ*!+x?6h_D|a2#la` zVYIEMNeVsr`$npAv6vwyWFfmpHGi)ssZU#f`yYkl1KsF>J)u0%zX8e~zt>lK!agtJ z$Kw0zdMO-@#}E@Rw>Yc^22z!J%(*}&qUL1l*)x6YPL z_-GIb+ZJ7h9q_hKgLC%LAZ5Caw6@NCxly`CexG>!d!WAxsw-pwzVUS*y0eKVSwe+Z zJ~mbm?7wD5y-PCDqdJ9LH0a79e@045RmuT%^1%`$vxqT+QeVn1-XtyydJ1`&Kf1(! zd8mIp)nAXgMLVr~*om!=nSvRM7EKb)Yc- z`dai;stzu^3c#Pi73BuA{c*cej5M9WMhCu`D0qsz^Yi=2ougGLe-U4T{u4k4VIc~h zBRdx0mXLc;bSO-Kgu={EWmqKi_g1!dQU$cE+1MH(Ipifzh`Y0qM1cj@q%B!kUjO#jbK~zn4 zgImGx@n%a+N+qu|;B!?OgWm%Dt}5@`_>tgW0-Y8*-n?N1$92uIXj@txMhj7^JK#nK z180fmR&KPKYAd;~26=M760tA?&OtiI>|LKqq$`q%JTkrYXXK@uaK&m$%4$`$JjYbk zJ5i@X;K{4YR8z5_-3n7>QW!cwqW)KiP-@ZqCzrsrq(J@4uKtb{G`B(eBk2H&G6AS+ z$kHR^=(X`b2g}KH@c;Jt7yrsC95T-eu-VU1royoe8`R^+AG2SHF-**>0r>Q+xqLf{ z+FquM6n8yTdB$Ike}i_tGif2~vbjjH!x5b-LW7%71~|0Mhs!nD#T;)@lAhAL^Q8Md zQ;Fu(g72MR6;eGuBEe-&R9S!L&3JnI;N4NZJd^Z07-x2$gSxu{ZI% zU=xlx!!Q`u;qE6>zD#ZPi672QoOyHn9DgFF`PzP3kO%}4)Y2@0b}M5FG-$Hs zP9Jxndv=3oL+-={1tsU56$89w>&y|vZ4^B`xl^%dTP22f|1-VUwy#6myU#B*K0VFI zE}#g^P=`aKQBG2{gx=Ypf_dm6q9$5dsYuhi27yalR(!b1!a$p5TKhx;-BU$;tst<| zf@C95?wP_f4EJ-YkcXD|lwr{}JmdH5>#E1@;u^o3T)uF`HHOoa;hC|#=ZMES56T^S zRvb+m0^^Tm>_Ej>tUWCbuowzrU9t!&zqdkUEOtCu-W-18ICs7Nm&AH2Nqt|s(8HGa z+SSYRaW315c=r&=W|2i0)#g5*X^f7wfolAg?7eSalbp0}w9JMcr9MX`z(uiJd7ZGF z;{@fZLPjv(bnZhgnMom7KYHJtcszcQUdRvUcZi1Ki55~Fne62r*Q|m@xVj$=xTLt? zlf_++d%eW&%{Dg3Qr~d$qi%kI@_Ww2`&5*xHtsm+_H#lAz6dLWTSBovghpbjn~~dg z1Rum4p$l`$oKwB=$=VN{H7ilu#22>)X{J_7ZU`0grH_yGzUtw8diLXMp}QQt)csnV zJoE1?k^QCe?@=e#P_ua|r-xTng!f~tthra?N)WA3ccEowfvY!t1#4nOO@9RK^uF-4 zfUu%m_l3yeej}uJaeBoZ7tn3PsB%H);5)AZ) zhq17jYNv-r_ig)1T$#n5CqiOtHWluL|EyCuRF7oB5G~MTiIa(CJwm-#?$U1GyN|1` zC61+!8sZ*zKitTpT~g%{p+2{n!#IQyPb4(rX&E1Z)Fe@No-V_oF;e4g((=mp`*va~ zrUUnqjs!0M7#kB z(4p8@$Zggp`0QPQBij&F4js^=@O3Np-NRXlK-G*bRI!#CRk9JdBqW8opf7~TT zW>xnSWeKbEt0voab%d~agJ?xqXu-JbbI>tO57Ufxe|w|&7JtOFpqlcdu($~Y&fVa^ z;s=VD$zJJ0X0no3w%&6B83soDk2Zk7l8<bZ3Nkdi49<2f%^JO|}zPACX?FdT5q zllQGM*WrY4ZtLc(YfJ&ymrVukDU4|f7YAf0x2#7;$bo(;v)$ag@>G)O?ES?vS+ONRv?ZFjO?ZxX=mcehmBVich09i9>Wyf(g-Sr8 zR`mtGWv9)5l+v?-D%D)cKlI#Tuk%D}{P;@a-f6hGd3O6b!nzfnqoM*gn%Vi3H{Zg! ziSy+7xJtQ0xQ~~GIdd56zazttqzgf4NF^*vnfG;o)u^%(7Ih4sVU%EyG_DtJ?0@9J ziz8aLKZAP~x*5>YgCn~;oe2p+7xrH_3xmn=EnR$IdDHUhHnR)cU}w=>MRXB*I9Wa7=xW(|700^mn2?BkW3S{$ z6vslSTr3a6Wn@E>;<)SFzRb+j5zJn1P*+YisToweg2B_{wwkWE(u`OGdx85p8=p!H1Oc0CTRX16e>kiY~w&* z?Exh}@ZYLZmVP-J2NF^xFit|R2El*M+Mqq_bhEX*`tWU+n8qTF2V4wA&Y@l3DTT{R zUlY1Wv_|v2FoHVqB(77>Y+M`0;c_DG%G%l&*ZJCtsFB9T3QF?_Hribu9H|x)9)U^_ zCw`#KV|kipY}gu|n^~Iw{8s9rB#L9kNx9{-VZ$^sS&W2<*#QWAvVHo*V0CsaJf`hl z@#~5GR7X<(jVDKTR1>+n4bU;VSH;Pmq&ATm(%^v=`zqJbL<)rAD$wV>EX9 zqR|mTm_~=%Efi#hc6>i+{q`{8-V7#aZI0OJ?|Uazvqj0&=88UzaIl{t&F7IiB!pnH zX9ZA(bEP&;e8=#2mbv0eD{b_c7kA8B&KOgA3}1~2)>=;|>|jx=&*vE<<=Jic2ZIzq zso?~D_>!j29It#!S-M9%n&Q3l4W-A_$JaT9i<~L;&lDgZAspQ?;~g0HE=GF7h4Ieq z>?6wj#g2nZiSV~OYFy9jlhlu&!2ogRSN-VEx5ARU3-)uNP6Ll2v=4#!AJ-wGD*9~I z43j$l5|s7qrd@N*sf(WmC(CTVcqcV&&CW8bLQDc467X{%P~T}C1A*M*m1_TGrN-|i zOqcjRe(n|sm@V(7gm3jIP%xL(Sty6&7r3b=+6!_e|iEWSb`U9EWcT zw8Po9U2rn8(Zn|P9rlu{>Hm;l2wAw=bGtFFK;Pg@u*&SHY$jB#dQZO~(6wlCK`PVh zLi_VeM{b0q*TXWPwS{r>uDY#Is%oW~C~v+~q3btlYa;@sb;DK$ymeVdPPY6Exsunj z_rmLv3mWBQfU)d^R9Yj0gLs`$CrTmh>D{S;mRP}^$QE7QTMOcXvD{%$_0FU7S@jeU zx4m&rd>FPc8D}upUoo%eF}=O4gU-7t6lrkP_mgxux7yp{FOg=fcvK5w61zPOjb^jx zik1ixTO_d#_E#4YQU^2xg>*l=nnhT88=i>t;tuYzss0LMX>#4uAzs~24ltOX;rI-9 zFeG?kbrlw3v}+I1O{wrJ`g#hT`I#2W>dRR^fenW02Asq$Lm_Y_b-YkE5AM^_SmJaG z?}yhPUM+DPq1B?HdBMC@ib6L{ZiDDOIgp%~zr_vRXQ{@!gB|o%f7z zlqwHsz1Y?786TOx%7h2FjM-NyE30A@7gBD&25*l_A%>6N< z79p|}9<$nf9?d^u(QCX%Uln8&&Fc6s3h#=?lhnySL$?}1)l|G4=z?kV9jR8Li;9*lf=Xpayv@hPlPr3^SP3?L z_{#oX$%<;@#fkfYE?C4w@AhN-dIyRlK$QS?+lk~G&5@ey7{&zd)|p$GbgqZ9I#pl4 zXXBuzxbJ&$egH9>ErCs-BtWsKcuAr&nxNZ;smT*6nyW&eo+`G=r=4+R=bT}`{pFQ- z2SrfQ5bQ-dqH_Q<-vv4D9;D#7oAziEMP(^4UO~9I`t(aWM(Ffop)=pYm49~YAEUT0 z{*utKXHUG+j9XvX!9poqE7CgOzJ+~vJ$^y(W3JZxM!*EW{ncs9uf_}vj#TTf#b*)m z*?i5)q=9BGoPG?bPk;AQd5P6C`?janx$BnVUivWOmPv24!ZoM*V0&SYiqn2sdV$bA z*=0Y#B7A5Elh)oRj%_#o40%fk(|H^Hkz>2eHi@I>`u&OSeqT*UZq3Kr8e|Ou}8cdVH)q% z7j{XzvRBc#7JUB219ZK$0oEb*tQ(1(CcM1=&UY@q+Lis<24&l&oW!)FR6VJJ<=Vm= z9Qo3p#cDEM-&|(P{o1-lH)(HgB-`}n3{(D_}(Czu_lY`nlxfrHijW}zMgNg7RECJ4vl%C!e-uX&BRIus0U)<$*bpYIGU$! zm$&ft+B7*{O}zRxV6Dyj>#Ym-X;>lxy{1E8mk3z*XvEO~PH4iG0iPuyUP4l~X)Hhy zqs+FAaG>T4uX?P!aMxJtf^zWc!tv3Vbtb}c7X+fLH%Cv?0mH?^Js9bo8eaGqeY$+9 zomo%fLo!F=QNebn%(?d`XGP9xsrs*%+Ad`Ykb1|f0;Tt)j2lTn_hL1*W^c9BTXLoU z=TEAKHVg)=AZ7Mw%OF-o1(zMY1IP5Ljx-4ddKIes+ZD8(8?y`h`ooP${fxjZQyulN zCm%TuvADtkZ%h3=-9WHKTvJTZ4MKk>_J?uBjiInadqLCV@81T0gNpL~%wa_2HoFpD z77#Uu18j2%o^22gub;B}9*aemLTH)P+Tjw`N|}DW!j0EQulCwKPZQX1hD5=5$_(0F zVXbVWbK}aVopRI<8N70AKACM%|E@lf`VpJ1)9ol$oiE;Nce=P$IgbRMw_31KD$l^O zc7;8X=QoMGre|6x)~p>>;Mf}0=)F=r?{qqUM*{2`uSSnMggcY#Cjc>kZak-yx%sR7o{G7gF)QNn& z$i{QDRtxo+H%Gp}%O+bdTb>V{CH+AAZtSIm} z#2V6g`OZ78$d=l-y7HT8J?y*WMKhvT&Xz1X^DOsCZQrS5Nlmvd{OB{s8MOB?cd1a} z#$#N+xXSC{4>vJSXck{k7yWUaB{sO-{o>whgBq89uLK?!pKPumA-KNF6bVv}Gp%{) z?(pT*gfjABk_xUd-b{u=AS9lm2T7~KJtmtyF~4n@tdF;UnR9jc`}Q(@L21cLJ4c?< zPP?f7FBHM~D;F;sKTw5un#q2PYVN|mQ&J;+82P$q*HjC^7*4KLhV_cc=?VpzyUvJo zDOrMT>527@!$ovyqnk`$FJs2?w|^>z*G9S_`Q~X3FYK{4YH~9Z6=1h-qz|;anxY)t zBQ1_(2Q1E&y0>An@*cPdmTRPHVJ3TQ-%&qSeM+*5Z;Fi9xCYZt}uMS=r9<1OHGGyHG<@9l(zNK{O%g%}KHM z9GcWESuD5Fru}wwEx4R>g~72r9-UzE)29mok@QMYCZ_`wSDPLE7SgRs6lxYTQmVmulAAJ+1W_iQaK3?j)%`}!H zRbHcwJ{B#1Y7HC1s5ae1gQwFcg!9!3WgZ?8&tFR80zEIz>elF1ReS3ak0NE}uodwl zWb#8O`_eQG!5YtPG9*dr2n6k%V!H0~z1mP8iOrMUb@w$zK1#>N;=^o)@$MbFsBoe# z4n2IWaZk3n0xMY2Y#;&Ye>C}V@51}1KNI!MY2vF64`^?{uJtR(%Dz-0Z+_oW&qw&F zI}ci{Y9Q1*NBFA(^{n@iW>3N#UI%E;j zJ7RB@m;3WW``!cdnY43k-vcu=38p%>|k6utk)drnoFa2R`R=~yLDpi=eoF$5$9Mh zU!#m~evX40W>}#mw{8T9sNGR_rbci zFP_FrBC#3%2J|Kf68gL2@xO{5GXJ9JRB;A1!C#EZ2MD!Sg=>Y$m(5tyGwKGN=&5Kma?pn7 za+4B^Df7V6`&H$C!#;!mDK)-p>bIzi$@Mv&1azky)@TX6zy^3Lg?NB+Ij{W58A~-D+VCS^de0ko3G*Hf^jD_{aKOuN8xbn;|Dv!N z${~al6P2sbGL0j3)}EJ#-Jmr^aR$3J4y_{mdqtcrT^yy*1VkmGV?Dmdf6f)`+qrI> zi`ClOF*ljzD6dHosxL+i)V^IGwD|dp*I{I#@Shd~+t=`HA$vM-sV+q`WDzuis5EzX zki}Y8crxpH{Pml1M?bLk7VqsR@HLIcqV5hvXY&x9=3-`|EYjL0xg}_~l~Ah)+Qo4o zm_*&OvEUSBYOFfTMEQ-!e4;9aas%0sFyW5nE7-f{gko*xflnIXJ||mr+NrZ(JeN4uR{@IVF}@6!T`wgI~HQlStgnRU7rtZ#T{QPI-&+|b>gvpvu3 z%oZv*P5A{JShv*yJiDtiVM4wW!P?9LXSM7C8P3*WT>jT>sgt_L;<=-q*Pc6lS%i`< zRum#BH#_Ue<-U@D54%nnz%^po<5f;OWl4+#K&6`tOA4m|!yA8fyaKCb@!5g$lbVD;&`arQh2u%=Dbfz)lw11{ z-Izi9$@K$w%IzL%PnhA5Dnx*&y#G{1&=FB(TGXD+P8c`9d)7wY9d?%#IC6|MxYTx1 zjYpiTwVvafaFT|e9b9UgYy9?pT7U}<)$Z4y##dD0-1qvX&Emk>!S?rg8W2BzuWKhL zXLpcqzvnQ9RUK-<_FqCBW&kB1ANO_K=$G}j)rgS zjS2M~4`qJWM_-753aUwP{vm(`6=9+EqBxYPbyxx@ts&kr0$XYeY^nb*rL|U2kNroD z_5(KfQbQ&g5eKTbzzF}NdRqlFfV!av@VmdNx9A}NoM9nXiOCp83qw&*%sr^%2BhJo zwHqx=> z+q2ox5C)CF?A%tTI*yzq+?t7*5%otVjLP4Z6PrpDdCovrasgzMCX0TdR_428#lZD z3ufo6eNK43M-C|hvzQ$-Lmebdm~1-b^r4vYn%SGW9@68(i%TL@^Qfro z@}Ru3ppQY+1h2VlHcgtxLB%##PPwqS>6@RAefR;p1B4~}fbUF$H}60kCwk6h z$0JUZ5G|!1uQazD`o8^!#Y{Y+H?HC;XDzF8&R&jv(*=W1n)6dTEmO90g#~yY-6q`A z5?#LV{H~J`7mq$Uw|6axk&x^hFP8U3*P zY*Eta^IV6kv^uF2iHV)6yXGIJ&+f;QF2HH9wb9C^I0)80ZB9?V!yI;RWOM4yCx!Sc zy3(f{h0MM@i)SIW+c%GcZ3!}SdkTaE?9NJA5X8lXzjiFZlgG$P__5^EoT&9AJ*-;A zjoc0MFE%unG4<|Yz+q{woC7)HZ&+wplU*6cU>?G&>nT)J;^Zn&A?h{BhF*$&&!BZG z5PNYMW^SYk$?}U_VaE4b6HMj?j=I2@tMv4{+8Q#O(#_&ujcwn@bTyvc5UhPJ&=kzN z!Jhr>>$~aRZ96zTMJ05u4k7IEn*lTiQ}&0(P=FA+MngdIDUcw>VM#}DJ5Mht1b5fSC6u2q^g`RwBl zA+fvnphw+e6RY3#Wx1!)+^d3bM78vE-+XGI3>cx4=RvDD(lGVMtYBK0RMmGG#z^xk zmZ(+jeBGF*Gu+arv?g>b%mkS71UzAuL#lk2@yl&6R-#tT7>FMRg4e5}avZ%R#JkMW zm@kX79kvcg{B-xk1Wok&{REUf09M%}XqHpWcT3BPUN1MqIGbx1sot)v5xAvJo8=wy zdA|I8hyS{T8o37yz)t`G#LAmiN0k0_48Z3CMG>GyPbrJ#BPa9P{+{7m9R>JvDE=9O z;U`e(`i(TB_J1IgV$5}fdi*iGl4L+PH?tq@H%=A^28yqQe~b+BPZk5X$t6ed73x&w z02NXu4e8`#9uDxD3p8olZHG7x%ZmHd)Wz~Jm7IU#e{zJcopvLRzYWDpHqroj} zk4plMn2xdpkzKgSn@M)nsLix-Izz{b##cg6OsVBY?#F^xjx}yA;z|pG8Jd~*!DtH?lr|F#(I37=Z~hb?U98=0)O<3BUKCO z!bp<@O~l;^zP#6tGmi3@6tT;W#OEx6%{U5sChZmls#KQR1n-}rwrN~UYIj~TEs^Q#t)em+`1w4fiuP7tYV@n&w8*5 z?~)BGn)txg=Tb>R*)n@Wct!Z9NAY%sFE{dzpYOkLsqcXe4S@~|p<6^8nyqUb7EF7e z{;(!JA?<92y?6DAuw&;p=4+jI9xpIylqsI;?o$j_@x~>DHOmuV9V)_jmR(5O zzJtZsT9C^kQ7w>J-SN?5iA41<*Q?aY10O&3&BQ0068yoQqzkw*LWIpa5ifz7%`s}W z<&y7y3zcd|>!Re&;tW{aCMGwY-fh7!n7vDnHmBllJ?Ap$c#e_T`o`4q*_(x~a%}Na z9@6VZXX_1g_r5T2i9$-kN8w3<2q!9s;6ozh* z?uHo}ff*DSWn_Q1`+4?$_xJAi4~~OlaZ`7!`&!pJ&)><3=|vjGwQIi&=yd8!w?4Sa zgN=n&c1{*lYlD>^E6=;5afO-rEyDMPzn04WND*bEPXq2t{hD!f(y)q%O+I%esF0y$ zTC_y$#@CQ_Oa$AQi=eDaTOL);~_jFS<=-*jD7xLq#CNM}>A z_+VS(rK9Do<&55y5R0y9L*u1M)U{`Q$%QRjA>yEymK&95o zHV>7TdNf$A>AQZNYOLxNv@*@Oz(2IRxNYl}w1!m#UNzzzm|Sa2n9ugQwhmgh&%wb% zg(nBGwz3|-AeYDdOjlm`u8?H5y*hWS0R(fsy&ta}ii$O0xiXoZOfEe?0|zj@gVsLn zPE;2WVHD|;bum-Uw75SWVjwCUlSV(Mf_KCc_YJc_dNbD?H;^^bp1n{!-IOzX#Ocl` z#c0o5;(=@XJP`3-A2kc?7O3#Ul+lI>r&!sipzWHZ+E}60-%e*_> z50bGHAtByzKr;t37J=%APlnW9 z^{d8nuzsEw6*;?@q_G9HyPUKIo3AI2%G}cA_SMjm$MH#hIpcs;z*HkAZ^KmPu@<4* zVJe*DM~E8`I|C?z;UJiH$XT4vF?Oturw3m|zZI`@Yu--Zv~l_*d^RkIFuTpgP!XY- zA}@)nrHjv(PE;*J^$O*Oiar6 z6UeYNX&DWP`KA~~L=q%UmH6F^SW_^}4)uHc4Nqr|SwqKD&&%_78Ak2Ve0*|!zPB_( z*Kbp0b12jSXKN!HDT7{oc`AdCVAn8+?zfu`&=t^U%}Me})p65wOL^5%_Xzx1XM}+C z!{#kz-Fo*3>HtIPBSq&8S@h%N6=lig;H1hWHClBxy{}~5lf2S8rgRxGOn9AjiDC@* zJ~yQ?KqIzWGAb}GO&&0fry#Jmg_(zDSqF`|`sZ606O^nT!^}@+K2HL&CbElqD$Mb~ zjVzkn`c7^QXyNkm{4ktN>i+wmnv-wO6}fZFH=|b>WAF=`k6Aqe=hgz1Hxb?aDNchd zm2tp!ih4o;VL+k#tCax$e`m0B@L$_UjEYYKLVvHn0?C#@!2URB67WC)IjVV7H*af?S1E9~ z%gSYp@XW(I5u(C7`ANo`UbBxv* z9VS>*LLCE;>5W=c;VdF8(O;Buthc^+*uQ0gO8zos_Zp0&m8_>trf~ONLH;{FQk-Af z=(2iJfd1$JASi)Fo;2F^&9*LFoN_e4KtN9(@A`36JOx+5>e$-~b(eGG2oX_v;4i}g z)^&ZPR_(SVdyNM}JAFkB`BCk@Oy<~K>c`)}ePSv`zJ@Zmo5uExNb%H_vskP=y3sg# zXvhObJ)NQ=eLH18wk6esA9o1(d4yFDomP9N18sp2i1vny)??xxqrTZwEL@P;DT0{D z`;BK;wxb`M2szy@QWA6~ZJQ)t8pXM?9-4yNm0|L}TQ<*n-^}`yuIIDPov$lbM+<)& z!(rVGP_Ru6SuF4@qE_}nigSQAI8g+Ogyp$$_V;8(EgWe%gbEv ziAhx6f+Kqz7^ncj3K`ZF?U4BG#D5h7v*ZfZfR=2Yh!ZyHC|}};xWp?hkqeEoJ&RN+ zDd~csASVhUz;BT+KI#(3_#jpzXdW-54{voo()X(z-)*eQX0iBQs-*Wbk&Xa}pJZ1_ zL(?JT2SBw?EKx3Lv0JEun4Jal(GOwo>YB}CzUUkrJXn1f;9@d%CvYE!MvbF<+kwHu zUT*r?{12P@mcw&+M%jkVaE`XKPaF>FOMj*3Vtuv>3+(YC93O?hJ@88_f&TJc9W*aY z&Q4|f#+i0)EfYT$SMO7D>sK6#7-iBQ<5WN&PFV@su?*`5*;xj0rzs+9s<*t#VW zaMnlZk3MxkemZb3dI!F(DyDL%SR7*+K8f^3&Ifcd4snuykO@=q$P}kK&YP*Iui|@T z7SChU@@@C=fOC4fg_2*?if0j9b2lTjs9%vNYIjN|p}F(c{cu(wEj7W#YI+HmKw{{? zZpe2y;tTQCUgZ6WOt{mK10CG06dKpNK^lp1(nGs8Kb2kkRam*NTbaNkSW5wanz0)w z-X;{lt**RSg|WEyyS_CC&AL{-cE=N=%we;UTzhV$`W2Vy9^clE^F=X*aQ#YHTGN** zJ~xwCee>F_l5ZP2OaLtD`}eu!aR?rRoDYh#R7VUEoQmeCToTp3MP+d~2S95tHRd zS8@^;e&5PGoNJzjI_0Nj*3eIr{U+<*7S2SSJEIrJ{JNNy%~k{Dk{RtuCY4FgyV+rS zn3yeLAQ27|?Dd0_7FslL=kk^u3czmd4Rc`f_cFCoB@}#vGH;(SbD4E=A}o@dx)Z}m zQHF`{rZhK7w>wTO%!+ial68H^r%K{kw1~!5g(h|dxO|l&L+RrVCp_+PW8oih9hBF@8AjJ^>im|{5WhLt zBKC$CIyO{934!UBSsj?~9h(;37Xv@m}z|TgZ?DExRx>Fr(^Fg&`kdaG70ds}WkfG{9je zqw@v5b}M4NMx`%bdQa;7T_jJWO7zD$(EuAkh)CU2cvAsEhONyAhr;b2E>C+o5FaeQ zcHu8ppf-EYDCA#WfCQVVXrR6FmWQaKJ0k|Z&h4LQzD8w@CAd($(jt)T`%zy}g`e;d zvyIv}U5HI{`Lnv%@Z8fNo;|t9cWaZvk1n(fTa7|8jR(%7&Z6NSs5l1)9O zZF=5F%$c*^d+6EFfFB}rF1=66tVN&jw14g>|F4e20>W|S$>Qp6+5izAz~DslpY;>$(I@HC%lOpwjIzbHF= z{YhVIG}5pm)pCrt^6MZIy^FKR5|^)^eF~ZtImdwyke^QiZxmKhTOQXT;LmLOe2Mk!!IIRH!)TglX6=B0i*E)u$LN7^l#-T_8+{qi zO^)N1(f*N_crB=y@;s+25ZyWqVS-64_#aQwZ9?f~Iy&{p7*X@(t3aXA&o&dkV3xf@ z!Yj7xs!qKn$AoA%utBJ%$wHFt1yP(aAN(z!7|e>)BNt@dU7+uznO_K(=v0_I0>zlT zW=(JcPVBgzLOschs(Y>WBTOg))J=P^u*e~ax@qZR&oY=UhwxmUYj7@OOMbe~4@_ru zC~2KO7Ot7Tn2cAUZ=3_>wRm}Z`lSAa99nw=NQuo_Qj$qhTXl{u=~oHX^o;MRg7~u@ zJr<*5*UtBAJL3|>U{$yFks}| z$pvv<{_AK6V4iLOE=%V6pGN%OhoAm&Yu%lh0;FNDjP-v3E>zt=ZS_B#22UX$H4-zK z6F0;8!V?ilZN2@36WF)qyGSgF8o>}oRTxM6OYn}_m!D4CXD4Jpo8(3YQ(p#6yfSU= zhA`SrFifh?i9dOoNKN&Ifb#Pe`t7w5(X7#N9(bt1uS~Pbdi4G3gIv0$vM+9l0wOfa z(x1%|?-3-kSF%&s*XOB_VRfff1j6SyX)o%@M+1Ug3mK=kIK4i$HTdyN!gQ8>Smacm zes|))vV5+FLD3PBaPW4LIO9<`Zr-ZxJkQLg+VmOU&D=*1Uz#iU>$mt#ay)6we1I}n zbJ{@^qXyH-zns8C5ii1&{g-4??&zNE%z9h?PAP9+%HV9;zSE;rcGs-SmTOoN8g~i5 z248*-+TLV^iBA7G;n@o{n^AsJOWZ7~`|jB%C)%nO@-GhP6ZY&} zb1vYXkxQcf9?!#*SkipHNoL-AI2`}L(Gl*)nxPv9>0rRt+V&*Tp*487J4snJs3s{z zr|p_oYhjRmne`Q!^zMVq@sGERvy2Z^IRes%H?a?=Z}^AH>PokMONsh#x|4`Jw_DcX zTX|^G_(_L@(WGO^mom7R${y{O2uK<+5TnKBj-mz+MVMA-ewygc`BZ?C5rb-^Y&H3q z52-%$I%}1SmlIcV;!h1^OE=oj^mhT=>cjDhcC6!6v#3bp!KI<L=VitpMYB#iq!#n;HzeT zkUfoz_HC%v!TL1C@e*zRg`#&Jt;T#kuUz#(vAB_385tAAfQW4Gvno5x4AtEq4DjvL ztAA(i8~Nt6d|Vy#Q*ImDqZ#%(wW9p*D^n6CrCYPMAn-I`N@pYoXNiHz1e->KWQCZZii40+XB1-~P;aXsWXs zxguc)<)bT4D|$n}-<=UG=d-x6?CdwTW}&-mHS?;*5&%bVumpDsL9WeSm#l zYj^}I6~wZUP;2Av=c)L(eD5IDn>0A6(D&5&CDB;*s7ZNH1U*iexF&SrGn#c+mF$Kd zQ#sfLG`wHB4LlnnK9%{Ytdh0z;Aog&d{4Q<*!P38Nx0Fl`=OmcKZV9{NadSQ1R!#^ zZb=EiGjQ_TeqH%UQn`kiFCfDwSuuN3Rj^EnBMQK-0AvgPTbb#A<560TQ0|ZEvm_sw zCIOTS4q()z_ql!wyFVF0at!3?^(5T|yd6}GoS3iTaS?wTJ=HkY0MGWD;y}R53)nih zuu8yv`*+{{a@L*~G5J%O>fc8L!C@_b2ZsSzA^?#Am^IxJ!WZf6SFs9HWwT&pd$+@F zAYfKn;i22LwUZCdRtU0L9Rb!xQ1#u1^oMNJBky4?ijUfByRxeeeGi|n)Rx-HjqEl< zZ(zdNgf1Q=1lVQzd2AR4R0Gfo?NT#JJdHgVFMkcDmWcBUT!9FVx$oi?jK^$0#y!wE zwv<<;QsKrvK+DGgDh-7W)NW<0KkM9F&f){JC zL`eaV+nx^8&*sIt<~gzvHYqA!gWU^F$=DuMpGS2N=4WM1T7PF3M_eV=e>hsjCv*j_V8)=+fkWe2{EzN9P6|N6S6;d900j+fuFB_%@*<$H z<;~pzCn=h{s0YdWg|FO2A7Nn6izr~h*P76RlRVU6#$jW5`&(v*Z?7LtdNtsyT%@2^ z`&H?_b>Tu^ST4dKpbxV1O$Jp55z9X69$qPTX+_J7_f8+By0}rKCcRD5?!-`u$6ur+ zjeb~vkf)mRt~qwSmOTZiRl9TSqKxo~`o%~NCxqXiEBiK+H)*<<95CWm3vdAsrqp}0 zrnBQC`NlNu>HJy`TDDtuTbTW(z!U1v@Bh+qX`jv$4qll-=*SH#RyncK_c$_9KM(FD z@UdriLgx=igii!n$IMmB57_AEqGny`jqlX%hZS&+;k%oQd_ppw$q3^lH^H!P~SseDYNJubxAq2Gso zrM>WnFmh5XW1fAPmIa5ftB@rfA1^y?w#a5YlH@XCDiZxAapik%h4XKrQZ_OR(l4)d+ zmp(N9w7R4gKSpj8tMr;X@Y#6uU>e!XYfmqjE4rd_Ta^$4L?7zGuSM5SJIbvl1zx3f zF;_m!e%=%HS(lkD8KvoKu`)cc$p-U87sbF75H%38gAl$v@R)-^`Cu@&c5+kBw2<(N zJ0u=#4aO;@Zxeql_H8C@C{1C-%Q0zBPz6nj47HV;N{5i=HYPT~W=Sdl%*8)2`XQ)| zVR)k0@fYMJj3S~K`xr1EpWbN=6%G34;F>Du#1kxeJ8Y{8pAQ`k{ZaXJX&O_Gth!-= z0f_WOz828rw=wMk@T;va$d`{U$Ec;x>9OSXzQG-UY2R`YsleHaVO zz%HBu$5sQvL38Sys{kR3Ws-tPa@UYM(l-mHDk|pt}5PK%RWN zPG%gfzcWOyBicngJRa3tR5%o-ztvS4di}r>IxgCXhm zfqk-8=DPaJaCBEI(vZkg5}FAL?F`kJKa;WP7_)?zwY&4JeEN1!{;K-!J)(QH;sh94 zKx`U8qr!E(C~c$bh9FB)s5N15;vudbFxbN@#T=Zmc86ED2?}i_0`i@1$^GM8IH;Vpu4gw_698kr|>|q+w{>`9ilxM+gsAM^HV%` zSIh%a1?Rc)?nTlxO6ycA;A)#FtcxRaQ&@Xit-s|iwuVH!#Oi_>-c{SVOL@du&G3`f z3v2k$HbH2zM4VPg;&^de7Ar8D+ucb`X9-29A}HRzR+!0m!?Z{9{wcxbi5sVa>xDuL z?%68})E#MUJ+UW=M_$U(b7NBo zlb=k9d<}atWZhDY?$AeVWQ4JIrIa7idag__CG$(Q=7MORkr-#^JTQH6PB{eEH^ux8 z+oqz0`rdFR1$HUxpZKoo&H3BOKhUUn(D;St_xPI5AUkvJZ?EwV0!KRgH-J3A^>Zmq z3~io(Sodb#q=P0hHnz<-{T|?N?-4z@JSFwzH5%~WI7|Ipr#-+)pdte8?Hp62mafw9P=@4EaGpLM;M3~p^>UfR&r+Mqxwh>>O%8~!Xj{y>Udxp_ zy|c7k9dB@OPH>YJei}tYKuOHgp_<@8Eqg|FGszN>@If4&;%n*O8d`rxZZyx5!6Cic zUZd4pdg$&>#I4hMxCh$&>PcIxqI|NKs27hVb29p6v`(2ADL@I(`ZMuIE_6{~9nlVb z*Q$6&?UON28)kMsQ6I^HefE!xZN^yz->^u1q^%y&xP!PwNU^sNfqFaK=Tr~vj zhs}^Ju7xfyMc4fD{wmKP@wVNs-kC*$IJamebl1OA3iscLKxgeR+t&inCj<2eg&!SN zaAT*+joOr#KEpvWAs?QL>?t7K;%fw6(?w;)lHaQj?#6_o@%uM#V+|0T#OqMsFx7$S zRnghw2R?q4qvWm<)5LQhMzmFIWMKPyd!G#$0Zj;f{kP44E#VWV$3+7Pb>nr`M+8bX z@KR&>^q<9dKTdvO65Ps4-g2?S|2zZDw?r3a)jg&XfpIk!ABt&F9+Rk*hW zCrvzT-3qaH(1wEmNexQT#btY(v$$ZM!;&1>#b6+&s$|xdvO{CQcch&o_0C+zUYl3QjPD8_4J>c4Po&58g*w)OkleHzBA}*PXx?N4-Xh@bPHofXnE+w^^}8F>C-w;S z+fQuZq?PU&uWn}rb9Y^8hEM^gBTfpdhZuve^HI)A830m>wBFE-rZ`YF1ZA4S zesCZ8-7qe83Bn#?`q4zXAJO2J7u#uM(3}m`4G&}6aX)XblX8%Zs#uTJj*~FNXAeh$ zFpS(9ezslIzpmd~D2d6faWDjlb|*BCTnNjy28y$R+#tBLc*VEY$=L9;i1(GL9+W=F zJQYI%Wql&4hI+*xUVzNMGjnhe4*y$Mo{fn3-$So( z;tJhMn}JYixSD0S}mrVc_8I2sN+J*H8JBW{lfkj(8?1fabt?m zFM3;5@D^YsL^0cQNE-tuhb;5xz3JLfJjkb^xbeUM0$d&~#vndqfguAlbg+v6(*Z(K zkwG0X-FltVPSX%C(X)3gpIapLCU4j5OO@8Kzo6(z#=||dpPPGy`<;EiZ!%tSKT1Ve zyZT@OeXJr1psqWVY})eVj6t-);Z72}R-`%4-*BZ%CZ2VvVyNuUt^M^B9wDM>tB?;^ z^-}vu#saob?&u?-J6?@Af$tl76z*-&*>?hee{zUn%FS)8HX3~TVp|$%2&_s^WJOkw zjI1Xs*NII8EUXe^6JC1urZ?hQf|V_}sd^^aprt+CG$#RNvp+WJJnekH|7!hSd=}ghyjkQ+VHQ2k2C>V7$sPh3>YXRvO^HcLv z|L1tov^+zRNGgIw`K>T{gleD81?R29E-K*1D%r2C9ddBr%*8S}rg~UAh#lSeeQI0~ z>twNpa{$^9ET+nbbE49QtR4CdV_3{fD?jA5Bp@p0nhnL=vrXvu$C$O)qN!fTd1h*T zb9!MPhwLUi`n5jcoO3v{pyCi8!rvth9`e%zyw5o-&RtSo ziQoN67{a!&Imn3%%|*!f5`8kHI{c-bKxfo6Uos7@n;RO31h9?K88nGqTK0&h6c?6} zFTR5sB8H=tLxcJ2V_{Cfi;Tv0sAGd&z%=@BK1^?DC(7nmMYYSj!$rl|(jZ9XlJ$^F z;n9&Kj_WhLNTn<_4m-bcXwi`UeT*9-bdr3$c(^jco&CYg-uZ@=sPag(cH7Xd zWdmoA4g28@ABv;lefW$$BU;KTxRAXGHBnlzZuhfF&Ol`0Tf)Z{WOmAg8kbY1l$v?_DJBXU<)lnwl(Q~z9`$iC=miE_u6@Bot zKE(ZpA3wR*WyDP5G|;4n!*Fz03kf}^h~i++FoppFap9cH?)Ho==(sqzxNpUNpWpax z-B8jVr}!u#8fmSi=bxxa_PL@gH|Q{ah%3xHq$;%?RByPSMXqvNW>)4zTHdd_i8$Xkm(TjzE2D01Vd(l-*6Wv=!=$xZnmSs)(;AAICf~9f z(R)iXM=PAkVwkFO8?cMj)t6v2;Q=Mkic@wqvio{upbnu-;w|@O0cWk``1vS{5&o_x z_M3oC8@@@=#d8orPc&EsV=UcVTNl<)js7NUm)SRnM>egT@0ht$wbbv4eM{uDcvwSH z(bmwKv$y#zmsYEm|MT;kC)6Z%{V!e&Y1^9qFd~Y5PHWzF3gMIt^E5Cha~+h4Kz_4%jG&pfNqd> zkO!}1QdZlyjP#Q%f@nzxtwd`aR9XUX7ex$AE*~T0G8m(3VG6ZJE3lS!*9`V=RLY;# z(-buabOsZ8D0^@V*0xlKIPlThvDn}e=;i~|V12R^nD#!a#gy~sSk`o!)?pbcAsX89 z0=-5tbpn&Tgl2K3_#cz3o^sWttCMUs;@J=|O(L@fFB`;P8-EZ63DBeg*vUpiv*hWKtpXB3ua3jk69_ zoHd$tYsxpuGHY&6DOhwoHy04I$n-smOMSaoGaFe0IUveuI1Q8hfEAu`?P#E} zwuQcvWrch+Bz^kmkb`3QwW-^fGrrF?X0P*rs>c|J34fN8_#6?5^G}*F=ifBrf8gA_ zJr!R4qwaAA##a6ij{i9ZAdaieqx1xELHz)w4~VtmZ~yn#gx7=zRR2BJ{`c`~LO!~{ zeIeul_%`qgbpH^M|8VP?eE(pOGXcKd_lOr=H1lxbYOF@CY16exNNYl!EScPQ8tf9w zy=;eF0dKuliWkFrKk@|e`>?vG?!l1ZK_C~hFfZa07}${`vOHw5uiqCR3rq{K*?rO( z_;b2o34agC$MRi2?PWM1q!r$}&!~e@EyZyA-w%D%G4B?8KE4CDbiSqg)O|#x(=N}1 z?>-)_E~f(^+!pfLcA$r9U%r~d+EieC;OfQDeBF<;V&0COu?({u8&(OK4>MjU(&wj% zNW1u?`YNOmV+@B?0r+V%T+#zIGdbL@oEI~{mr*-hZEX{5Y$&Y1v;JYpr=j4L4Csyp zo15&9?Ay3cbYgA7#AcM@!6&B-{Ir9hA|X!v=Z~V!_wOOgVfkSwzhMS@I!wm)SVU}| zRz0G?+d$6}790@dxd}@3-Q#l{o}}fG8mjVPTA@VUtFL!2rXm4$J5*^sy6J9cJ+D|G zxs?a@r0^M8W;iA-!~ewDMETa3+i2FFk;U%u$T%k(jA9|23ub|-xRPuJrb2_Qj{CH; zW>&n<#4Ryz$3_k;xj}y(i-5uWycqI&!*`HPh z-|(^+q%?f`j&#`2dbZIqMt}tQbpV!qoej!)pvPS-Q%{?uipkX2_V7OxgU|i+rugbj z=N%L_FK};|9W)D+V(Il=lo-TVp98J30t7XyGmm{zJ6f$zl_pkqVsqj%VLOn_^b_$= z%W|HQmsNd(i2i_s6XSpYaV8esYEJkuo);8{QmvKMofgoY}T{}a2=?URvpebO8 z=*Bp1Vk{h%9ne-0eT=`ee&=7tcO#F#T`PA>N910$t#?2=!|3((kPY9vYEV~$h4|== zBIJEQ`V2(?R(7?<>5CNd^iX^+7SE<*JF7V3b!drtJ9K&6`sGs@}M)n5sa90Yf&ZO8FE$(Od$>l`&v6(T99H zi<9!4`lgEuY5Kyf-;0}J7ZI3j^!i+9eJHMHGq$}bu8R*8&ex1`z}^K^ADfmUQcxGj z_8V_;<-Vcg(N?!;TE^Wkat_K-QT7+WB=M#seddq#;O{77*B1uGgm*HtWMZS1NPwLg zJtY1DTzjzS9O^&`XcC_+bpI$B0vMnggWcy))&Ciayx;aupM*sMv_{~v7ZZA*Eb!qE zr$0Ls3@u6o_Rmo&9HE%M_G#|o4YvSBhui<0rHnQIK*h6r1nb!V6BxJW1|iUiwilFb z*xID>lsD0MkNLP%J2m@C`}ObyxzCxm^5|_Kpk?I4N$4Pl01%a4gbz)+p{v5WZO_*7 zchy#FyLr_FIQc#@8f~XvAvbAqhKHJ_EOSJr`Io?R2RY;x>$(IzSIm>bE)9SugdLc3 zO$H>p^MfRS=j&@(N)lVD+E9=fbnB;Sp(3Wf@@7I&eu(vqd9_zICkyg^6B=|awxJIi zk$ac>^_5_{-q`1;1%XKEHryu%>b$)GjD~a z6EtGjoP!UoozTQ;>FoqZVLxj4pTCb^8>7yEePYU#2p(bkbx~ia?6X}EF%Z!r5MAeg z#(vb!V~=@!bF>cr_Mz-r<^jlHs{ReWw*{Z1Xbu5Ixt1mv6Y&ro6yJ3Z`|RTs46t4k z7vxP=z}ss(%PcGm#g}@BtM4{>mNN6z(7mN-8&@$>NrE>g{+bE(ScFOQY8_`vl{qG- zY7;d(LaBhg&T!^KslyGn*{*(qnKygEcQS)3(<9}_L_W;uEaAAkev2dAn}38NV{JV=VxS;H77FDbUD)ZOWc*RqG|rKOeX9PPc0;j4@YPDN>2=QNai4XzAYi zSE{a^@&g-G7*VvqfDjU_g|Rvc=cQhTPn;vv-&R zBqzOlHyKRwou}M_y#QL}rEH3!Lwl2zW-Z)ja8LPpN|j46zfm!^bno81L9Wj~2stjk zVi^0|&tjXdlfps=TkXZOcQ@(hPqI6k8Y+lJ9umi|?d-cNBl8fjFoY^Sv=R|h54_Y# zKt32N@Q=xpFdJ(dol|cUx%6Rj3BKYX=ETn;M?Hj0PYS6g5R1d@VVeV@8okvNBl99n z2LTFqTpaiY`BD>c*E<58yi$rCQ$gkR1%M?Jlcw>zLKy@`h(24iF5K+PxwC2R&ahmb zcEU)&oDpv3=u7elPvUu}@GD3f;|&WsC`OtQB|^zx$xUaoqP9Dp5w6At+QL0 zM8j2m1lt&L-TYqhj7(B>?O=9>emsj!ZOey8tkGOV5XD)pC zV(eOU;DR)iyS2tALL#0imr@=)S*rU1x(G0N;AW-JpuzgBTB5!yn!}k*>%%EK`^^oe zObgX+^AsMV0}1Lz+O;}D2KGtsk}g-Y{E#6V;K+0vSQWk=@rHS9t=3qU%zJ@pWxJzY zQ%bT%!@i`NyNe*JBZ}_o|dsExxcbu@n%KaN_ z6%vIlO5CRfLq~e(!!7|&dos)mwAAnbV=Pj4)<0fZqbxR2=6q}EP)4naut7v^_ftg3 zEuS8QjB1j-GEhy3207JiY8BC0IJ8a?4#=S+)BN~+wGdJrkH>=jzI@-!^iw-c3!8-- zwf@>@tyFnawN#Pgv>p*mlne4@m!~7!-zS?+y+z>+T}4#HZAFk;LKEGxOlQvOKYZiL zeI)sn{nSl|_qjfyAd8wV0g4i6{JFdbZQMcU)T4*{@dsKJ?OIa6?jmBMwctD*`V;cZ zv_d%w_=|!bB#>P>Ao9a5RO5Hd4V!%tum(>#t;34S6;y0$KnUcV?N_s7>;<#G(3*Ou zcaR5EK2_ihn6{^?0c|K_tkBv$5}+=Lv_Tf}r25?%x7k(w*kvzoLUN9>pF1+HBHT>rm@PTUN< zAGNqO=jjCs9zOAnMMM({o0#BKWa+ns%#s1&5|2c6$MG9#|Bo)Q$8HQa!21N!M*un- z(2d3)=>A{k3DAFy2;vF}Z8ZIHYW$V=1O`NVe}sD^I&=R5vB#AIWO9CfalJp$1c^Ah zY}%{}2c>tqEYWxC-d(f9w9#STNyOnmq~rUwugEf+FB+WJ)ucSg<70;>I?+1IR9Z~3 z-&2ZWFC1vX$8MxCkV@;MG`fSafRZF$o+0Po+fYuwbR6g#CR(4-{Y0D|_+l3;)^QF? zM$}i;)afv2{kYaGSf#dP*^4FMebJ9Jx#y9B{)Bk6`dK~q_(CR!f*A){7GGfTiuLm5 z&sGDN0D71hLzylYZ(FeP>&eEv7l``)^=CS;`$%zOYi)s4xZ?X7a3}B0OtT}wrT4c7 z+i_pZQ|af}kg`$1C`LofG+KHf9OzGP9fyuft+0Ys)f(npQo$JnQigK8eoakjtpxit zluLZL8{}wsH+eV%_Th}1gW~t?pY06jZfsMNXR-7pPReD9#L@(2WvN`8I`8pgldGchw~n`;MjQl+ROi7 zv+`)+c;MNdaCBRM5w8343#WVRf_P2CJ%w@@Tm)bQi1F#KTsgRsJ_>Yjo8A4$u#!wP z?xn%Qdq<}qrN<8@csbwFKGa$7h+LKN88a?*Bx`5eSfk7PV{Gh7Q(-t03r8FAl!Qz_$emSz#`#KH{o9&77A#N`+n|d4B(hah5uJFwK8tx5gn|5tM-E@YDWrPI>+_UmeQMVoXXuxDrCi5PBMisdl z5<7x(@k@AEwtj`?2e1;tv<~EivtI+7W}eSI%yVO}-7b?;q%@R@&nMw`t<6>bDH}n#I}mM%H0$GUUPBv-7iCK9F~%K>GV7{F4IkU* zM8Wm-NlE%v6{a~`Rcn2+0^(7lL(hdLZn$9v=;4?%Su20l*va~X5L5er@r$8G`nOqC z2B)IW1aF=i4^K=-tK%$kBF=7G0+6!(Z&&q)AI2+>JgPQq-Y|l^Lb>{R8+yy65ANz& z>uHbz`3E>Gf7p}hQy$U)ck<7t`FTM60N_+_0{rTN6{qFTN%ctnxe@09WE9RLR73#B zde49GBxO&U5%&LkP=@?+QJM z`;O#yXDPNFU*V{!8}cQL@D?QHhbo=z;blumzr#_`^q~)v7tj-w4Wf^-+_?*;D3k(t zz1ej@gB&;kV19g~vaR((D_Bk5?R8Rg{ z)Rkh+XP)jvxP$lAqMuGV&L(jpA#Y~!!-kze{`U?vF#1JV$g)^O%+$KaasB9a^Yq}? z%zMq>Up@`;Ch2h1wtI6~?_p*+Z)SS8s>etI=f`{Nz0L}#n|{8@f-I|nz~e}K(cK@% zWHG`za7>YY}C3$u4YVTIzV1d*8`$pkR$elfn6xg}k zWWqQ+z`WRI;-7Ls)wTU9*eeKhEr8I54!KEw$tNW+5Bxcwz{@E(sP6;74coO|ppH#Z z83l|QXSrG1)k2nL@^apze2*?!S9EF!wB4@ zFZ^BLw0P?DPFung{4-qkBk`fd30L@0Ho-QrZ@{VIhmB`m&9Nf7qGZ~=uh?dKRR3H3GsjID6 zqnO&ToM_bGB4V~MQQx8IJ<)NC??-VC-teLg2Phl{?YTY@4Hhz|X_(GDjPk!U&ySxU z{`$^^;IpGuE38UFSjhsTgti&tU_(7lsM&;S8#JJXYAe@&hUm8O+z1D9N<-4F1l&hb zg7Ajl;4rDGAO0GQ{cz>s6b25t%wh#)x4v@gw9;PE8rd zb{s;sBFPY z5t|6r-2DgPMp&B7#4}g~>evKrp1oxQ(mip?da&TtOqM16k&5kZZ{O3$mix(Dp@~1N zAI*?)1l7Y7=42tMT@2_v15cR(ag90uj(EGJ$1}G2UI(UxE2hMpd>@sQ@h6lAyNI!- z4S+~QJXSHVdHXe1$OjSxsW_%|V@cEiHV`>PVJh|G*hjUYt>?VYEl*WNa5` zXnvYIQyCBVupX8;q$;#Ik?*9kR>d7|=9JTetd%@}ONG4~=5Rwt^>U?k#j9&_`@S4v z;%89TPY!)3@(J~Fv^@6&0`8PLzrXlDSJeNXSCsw#zoI$+c||J(OOU(T2>|F$AW7sR z4M=N1!SmRM7KaCouD+0A`&BWq3srTPMfn;$7^R@4bMk0j{}4 z0>2*yMrbGQn@%_XbZgZ)osmtRB%kJsBTpeLT47f zn>Tc!>yyMq&mi6{a5>{(LynPWoJs^b$Q}Xy1U}?FjzgA{6!oK9!T_8tW^juWqg#qP z!wPV)ad2fC^2-#RfFhckiq>c#Yc?;2qz%y8Luqjv#Pg}y2vv3{NNpakCZGL@MQh7j z;jr0;H90D$R1Z%Uwuqtb=wG=5X-rThvzaqduajDSkb$pK?DIuyKV7zQT0#@D;Pm33 zALSaqFXa{WraZp7eEAWbXTMw@QyS&%EpzTL3D&w-7A>T<+Uk}qjGy)PJ$j?_iS9zv z@9;k16RBUTCJnD#mGX7XEjJIo=*Wz#+Rw@+5p{o@1B-hZ7qIMY1Z46%KdU3dk2H;@ zdPgK5nFHUb6z^$C5)Au!v-KGV?)YcUZ$^NfI7SuvDP6x z%DC(n=jA9w*s51a5EU7GVxOg}<9@gI43O)@-i|G3$+}&K$09c~t+yqh2ZUfr-xyq3 z`~&A>O9GIq2U5ITTa3S1cDs@M&q1GcI?*>LP01e2)vuC1p3JS!#b!pu@ZWxg1Ee7r zQ-q%Vl|DpTc>Q;1X>^C7pb)SY!v#Q*{Qs;Xp}GGJi3I;&gD${=>+kQm{fb5K?@-Rn zNc?ebfJ*t#=qw&ziJ5$p9QnMpHXjf3916D+FBt~gzbNdD&T)11(?~{7xU%w;{~0A z2%go# z&pF@o$(FG*+4l~#lBT~lHp&DAjGasK)LwqF9el9PYc=*PC>zoK9KG2h-0y7FmEYY# zd}fgg9aNhxccJR>IgWLoo4=TRb#Jg&>=dE~-^Ryrb;w;YW0F_dYFjzL$

(PL-5stAM^IFn~Ax?Sy%7uSV1+5?mS3 zvX>?q*riEE*Z6l0;$;Erf3A_oIRUJsN*^#WS>d4II4>wRSBua7@`f77Kj1H5IDZEG z+p7WLvlLM=e3qFYfA<=VT|9mHn(IlS-`YjvY^A~IaYv1LD43U)Zh^!&3w*7Ra?@^GhGf_HxCE7bc)z~2E~-z4A;TF@9tc!a&<@c{hwUBkW zijzd)H)mD2kETs?TKQR*S3O%!rJ?z~Zf6`Q0iW=2Fpk)SDAlzD!wEd>axAGf2MPod z+6|-Q7LGY>O;)6rgiwB2sSzb-f@qqonf&J`89>lLqNx_I;r`Q}+aq#&eLkK>?=ey< zKQ~v&3^j{zDOnSK^{Y5i_(@IV#SF-Wj$LGasL*=Q&wAhT*0-)*Sd zDsi)7MB$F{eyr%WjiZtD0I-ai2z>DRZpAE@DxmCE_xvu07)*9`G~xmWH5vuOD#9p| zJQ_LJ*UC{vOF3w9Rm*#DCL%K^AGTXE5C+mYKCzxnGH$;LMigf*f1Ic`i7zq?6Sno! z9RXB+8H#nv(@@u(%`4UtWL-f!dcf_Abp0fYiovM$u(AWwV;t+^_U0du$5TI|PC}F$ zv^F5YcR6E?zQ!pmG!ph?2mX+ST!T^_F{aDC3cf3Z%)?~h&MPcBaf2$k0|w=iCeM9z{TtS$Ku z@o1cjf7-MWR`FIjX~FTx9Ht0TWZgns{VwSZ}PP=86f zaScKXVY|6@ubTTjoovy+-*9jAvD>w9OjWGS18qeNMHR)GAhzp{d!_~Sce7uLp1uMYFC#=cv# z2y3=3nW$}3{3_N1Tp;czA*-apk<2CI5Ug#@fhOE_UU1jpFsb+c|m03_7pe6L%*|EGRoE6A77&Jd_#@mXQhwR$IxETGdY|f(FAKb>Cy+ zm*7&ncv7#IRPvs5J=m{pVfd0Tz%;@?FX+#>Yrs}Z7b`YjDo+;f5qH&ao7_A?>w6Yg z(`}(o^I_8|q{@)3;OF+=1eF}v7{p2~zFPZpan(i4J_@2OJ@$;b7~HJpr z**9UMXw_U36d!%mI97Sav5LHhY~5|M>eb{|tY)?p4dslJewJ#FcjuGKi#5r#kPG33 zixGp4t7|BQj(I2d&LX8s7UP9^2lD>FFy0ejGJ@6giA zWKI{H_BD1haKA*;MH3FYn!XWh=bE=yl3s2g2G8EW75QjNx?#qcDuiZKPPOc7k2c2B z`&G?V9ZBlb>vgxyO%{F{5N4Khsu(yl-T3Of%*P-5V<%&_N0=FQuL)yN;Ftsr&}&Ca zyk=5sYn_trIhvM^P6PKp-;j7+Vhd)Kzs*3IrhfnfTZbe%_=l^UAg&= z_|5y0pL>`ebo~cY-knYmSkmhtBe3RjJ99A7!)2d5H zB72k3-Yxm@rPv3&ga!2koTSf*bd6ovVD~XW2v-Utm`q4_hD%)TVwQ}vZjbipOIMd< z9L^UCMEzA%&viL&an-!~^m(&Y4P;n{C}GqB%6Y<(RDH(65oN7+KYT;TODn3lJdKGa zYr4A3+?g*JP0JM2IZ~fhj5%2ujcC;+4PTi-&xUynt7@2!?|Df1nq)E*;*7*vXQde5 zTlwg2`H51T+2AfhGw4FF9pn05jVr5sIa>PunwfAk}|Yf=`ZD!G_ygb9)O_}P!fkIjbW*0K5tkwaxwjZ39$ z*I=!)erT)i=7y+tdQ9oVF<0Pj_)#ve(O6g0`k7U|l0nZY_HD-cg(#HG1Z^ABPHC~N zZ{b)w22=GQQPdQp8VK3*qS)`gDR?*#LX}WY-xdetypVWl3X{x!L!q;okjZaY-$^c zs#k0Uo9G)}VwDRMLtT~8_(*oN`c8&kY=KqPco$QPdzy=7=A8vnex6=N04l`lV!Tvm z{F#e`1Z&%DCiOE-nrDy|>zdaU;tM}?r*dnw8wqcmMK3TT${eX+_fcg1!VJ4UY~51( z40iC1q8w(4`i!=i4}GYD$kkMPy(Pd zkuxf*uQh1C^xQ1DhBZ|k)Od5`g}ENe;Bi0{90smg*PD`*$DQg()U;>6b}Psfd3;QV%d)U64snV)@CyK^7BKHf`V&??}Dib6LRSZbDu+4KJ-4p(CN?o7$w|3$>tc# z?9Q0)!(DniOEFS4a7W5q{K*$hj<+k(fsVd3c)>bWqC228*10e*$>SRNgbr=NFwV!q zDh-C!yBi0J8jP#$}I?w&wHZ9G4th$If5z{z%(LQhC;>8u&M$_u*A60QK6~ z_I#cbh@d$a@RxD9Jx7ax{)J_WZM~g;E^$cdz=GaCc`woA%USS$ST1Zp$aL#v$aFry zjAwHJr2yJwtN&_9;?2F1e~d>ODC6%-!7LN_Zv-^2^yOc73=* zSVM}Ht?ZHRV?(pIx1OmmTUWKE2S;0RM$Za6mH@!IW;s$?jkuj^rd4mn*-$ELEg$Ij|6!c)eJ&S5&!UCnG0+Uf%#v~1lnOj9nFwQ-LdW;ogsCsdRT zd#7$hZG>6Uy%Y_kysc!s=?=V4jy@9o%aO;d45)FmPz|T?vrWEz+>|oyC{IZ12HTZv z>-}x4sX<_O6g>6~wIm9mu5-k;_Y}{%bescAh za0c>!MB&BwS%C^xzrAAva>KtiYjMnfT3j&XQde|LII-qe0<{hYt;~q7+=I?0ze6cfw z78Mw;Ob@kAkPmfjiRUlDXsrAJ38OQ=Jt+iKM7IySGO}{*bM&WL?}(0Dt}4STRv74$ zqI;X3t1Rwd)vd6>MRd^@#kcBEV!a*VwMcoLsu_=;9=xeb-Fwx8j34#&ywzTJljjpK zTgToo3a}%2IjYfs*4qyh5|m*X8c?kLmFQP>&66-q*tTZl^>_V0AV#fxpYKn+hp}J+ z-Y>)O(UoC!L~2>IhX&JU4Vc)@|Q-(oJ}lZ@;{A1)w%~AP97YaISUD~6JpIv^V2_;dg*V(OzdVl)KQDqWLq?k z`Ec@yjy8vM^BIS+pNh;}ycDilTFuu3pri~aTcTdi-8Pm*Wmfj(5brKgYZ8MxT7!mP z*1fHhjy}@$A5B`=HS%nJa^7_zwhn?EnhT=#S1Y^c&4eGKE;>%{nvyY@c#EecDXo5k z_hu>UH=C@zR<~ukIKG${>4o^cDr1nxiTB3K(ufs{prLq>7n4i+$+K7sOO^K$eNo_& z45Upay#FW{JDU#zLhzFbFN?>EN%#jOU-$$IQE-QR& zsZGzzppM_qwp;YZWmlo67z>&#w}WaLqCC^kbp12O^|SiMOG);Yzn04VZrK$_eYT>) zvnRU|IUk|7zP?os@o}a>`~g8tv$Dn#h?&F@W9mmcYnsdag&QZjwhN8w%fhz~Q?D!c z#E`3#ds$_KsnBZ9(|#ULNob{DCnmkm=Jd!-j&)JR{Qd|3$!yk%&1_^X8>K1q5!?#5 zKMJ`=Q~TafywCFVtS?&pehP`-;?YkwK8lz( zF-M6yUcc%7gaOnK-X*W$Z{1mF73S`5xkt_)oo9 zr*sI-?dWTDbTZ^X!yxFmr?!bR2Cei-ZPcV)6WWS)^j-z|8D`+at6L9{Vs5yk&V=nj z0rrHhmySy=zK%-~7#?s-YJr8bz(H3d7e`cEV;F)5S}DE&<~aaJ!1ws`0=D5JZ4G!} zR=8aBpFR($bJskVGMbG2rURNE6_cDLBNgVfGcCjV7Xr0gbl^$6fxm%1PtdhVfj^a-d9hQqAYKQ!X>Zx(Maumx*`SF7j?u5s`7WRS z@E`O&sgi{S^=^C&Kh0EgwoItC@W05`n7=E9siS`c?~addo(W7oLm|7>TlAf^=dHeo z%+jJ2@3S2)OaY86yLhdV&VN#dZTzE$nUefuIWi&B&P#k^^(wNZc+KLmjxNq-0FD0` zD+#)xGfS;ALtS?Xx~x56qN0gg_qHT}q`QB^DfN0&tSC=9)73f^yyD3-3Gd=F|M4UI z#*ueq-Qrae45RxGzh&0^*lbTI*o@zUaiM1AF(!_vEe(`JoV51ZqDY@-eYffqjIo__ zdb)y>jh)<>v*lS(9|13+^V7G4zi_&n&DzsWbv5l>UMbkGm>9C(W*K=ev;a|Ae~Xn< z=<8yaw^nta6e}(E0n?&?UMO15IQ@fY(~osk6f-b97JDTD!Ek5r1I<>2g3n2q|2-%G zx87#6puTr7`*SweB#Pt@5+|4&$)>*Kjz|_&2!Ma$e69hCK`ZdPWJ%m*LuYbG-IsAN z`@J&vCZ%m1okYu(I!h56X2yv72Nbw@D)q|DPl5xBW*9zIHXgOfu<)qWRMivLAAc;k z6>d31^B619=Pj0g47+Hc>7>0q?~P7n4Lm@yaNFA#fzK;nZ6F_S0e8wL^$7s*$}toG z8iKFo{VR5W-kfx~l-e{VU~aw79yK=!ZP)vDPWaNq@|SJ}PC zm+Jm6ZPj2Y=l?8a;KezQQrE&QR27I6exGmP&HyPU1mYyqEAwiXCtY4*%oMRPUlX{F zbe}Nt;R^n_{oH6fKd(=-dg1N+f+Q=e)mdPcj+FAwxS~;d90cn#ccYptTx3-y-y65E z7#N}O7R@xT_fnH%g2$JXtU~p)xm7p=A-frnBS{ejY5+_h17v!R&t7VLSFU%MOMj}4 z-;YhBxuYfNb8TJ3R)$7leO`&p9ULAAYs&DM5T<<9B8HVJuBV-3TDD$Pg$qi%yLZr; zCLhLRt>7Y)Ot5QLI1g+bRCsBi;{ogn8dH#3n>QNVb}XSbM+L)N)Wp2tqC`9a${%p4 zwG?oe6#3=~xP#`@b#x|df`Y^l7MHg63m~L=YDNW~))^0EI*1Fb?pR;`5pYWQE$5eK zO)<;(ZLq~9PtJs>=xBjGp#n>R{Z7H6GImT?ri$F2Gq`uHcUWv|JwEw69r_#VO-a~WL&dX|t zZ+xxvV2V+^h1=}6ZEBx7MD6?biJu+Lw`B&{gUMEkyzPv_BMF`P9lDu%3c+Un+n--t zdniDJRZGjz$-{zGR>=f2uSUhzikb)#nWA0l+37^=FWBl;o;AEmq}j0NjG?;Y#OFL* zi^QivSWwB_h3X%Db$EB(OHx_ju(GS(WLpOuI7K zx6`5G`g0h~fSpu0;*7n>vDDtJz~Pr?n_rRPBT5S9c&X0o#TjKySSV$G5joc0_x|JA zWRozCcbHgvVa2Qu4pvHV=i^~VkHfG?ySXrZ;!xGVAr%NtM#CywOtWi5tKAY!%UIn$ z(XOV`Z0hsf`UrydAZ0R6f!Gz~LX$qc`}vp11GRlpsDbL(1>NwfZmF5QWKJA=yM*NW zES07kd*l=A#pmj;#6R7?2jwXDvY^5V=vIVr5cD#X*M%;~?!I!zs`NmgJW4Wco2+Fc z#$@)Q4NaNUkZSVPF-XB8eB#jc1nE8t1O-0FyUpgDD`yWUVearV{rH8i3*UUAGpvzq zJ9*8@k!4I9HcSj=Mvq?JUnfN81P9Pb^P|$bJeRQ0g}*?lM6>f^kD#p|Rlb8%v6o5I zODrmZpb%L4wM2p|k7b!$et--&^13VIvYRX@fjz(SZ#WD7 zY?z%vghbfdtoyw0d(odo8N7^0J@rB6=xhh1%}@H08;`DDp~Q9fNL#faHNwbtOX`Og zD_*afEYGhCrM4o7qz zK{1vb?&WB&%7-f%2s#q~JdqGyA<{iQ8$RIlbbk2}_Ii zo|-SHZzpmH+`%1KzcymR5>%noDT9xR^C+?Nzdj{%yS}b)fOd0DscOtA|6@~3a#qzB z9QD#f0@s___rBrG*;UG-JNhfJ&m?C^Qz~}Dh>l`xGMF6sqdIC7zs_@R(l0y+$KQ;9 zzgF%BvWYFEbLcOK1D-ijiAR)*=L?itzwBE&xMkmdyYtaki0sV~KVYXDzAtu2*KHB` z%eJ=?&filth#;erYZjc+hYKZM4%4+fV?4)505IDVF?s$6&=Da~VdKLJ}py-^^Mowv)-7NPI6RsTnVcxw5mTpC z1N^N!I?yX)_sA57xxAkR)ZuquEf}AM>5<3jCp@ihbW{rsRNlJET>a+w!JdymsLH^n zyEG+vXHXQ+>(*!Ov>#_DoNey&Brut1FiS{@wBDDx7HQddm!gIu){bFWhw&d|873>p=eB zXEfk&`H1;@&Mx(u0jR^j^_r#gU=0X6{(Y?^0)(#!!HqMc=9`{33~ZLIVVnb58w9>NFHkkh={aW zrt%*lUBVxOS*(zeJ9JPLPN!JSlvi-7-REl@ZXp*Md#X5ev0Js;a#0F19@~d$0FQCA zVkQ~fg9&+K2foCzlerE}*L&H6m8irSWI^UX^2-7lgktN2(*+&vhsrRfb$h0%Pnaz) z&wqYdBG9{jSM3KrQzreR?s9vU=%TiDpV>HBCk?k1*?vt5;0^UlO|oA(nXs9bzFTUK z8f0!zz=K6dD6baq@EwG1u`OXYv{Iqt*6`|1rA;u0#*@-ixw}3EyZ7EQ_V&O%VnRc( z^-W4x7Sm&~x#;1m%&R*sMhr1UQH3H@Ety2~{cg zDn6!U#%ZwoQQ_FXd?mn6IFb$G&uZu>LsQj^_6qu8I+(w6a$E!7P2CAp;CD`YdX1#4 z`ZncNj&era4NsqdPr~!y`0D75v05FCEJW$ol-@6FFR`wGl3u8li$Ce`;b=`=!uq^b zwHELA`6y@p5M)ouknb+toRDGo1}|(6Dp6rAE#-qcA(3e=G?tEW;wm>)1d1J^tu;K@ z2ri2aql`6?YQ>kcDIMw}rBF7susQ`M^wD_k>&VW$^8}mdKo8Z>n@$1mPt^M%9W|~! zu7nqTIkc@rAD~`)?Kfs^+x#TEku#%P5fs>Lx1RAfT|3`oJ@S3GFCyEwjM8#_-Aon^ zV?z^8u0GT6AgmJ)V=C9xZX|gjFhrKY{y2&EgFLQnbol)I^_HAywz}EaGpbWHyJb=s z&&sYWE!?3Qdq=G*R@PG3@=+qK8iWRomIFVWokp@XG8=a z8E9a72Ph{TE2f~8mGRt4E(~i5YV!C4x<6LTHA0q1h94?9*d-%cg*sn)hiyMW8>tM- zPmz@ljAwbY@sdoM*dY7jOAgx+fB`#>vBG-|JZ*JboLzP%`uc|{u5NiT$%AHZfGa)W zZ^B5N-h-K->EbS~of%@P9E-Lugr{q-9+`>APjV|NPv^sBo_zlxqlIS{H>7gCNtVRF z=Ry+AvnS??+BK|eKy3j?$i>-w0)BwtTk(e)K+|W53x2JvU5^oGYioj^-92xRzR%Vx zOpLhwV>h*8`)potezb${!-fXv{}*SXJ(NB&OH$|l;&|ArttJ3G<#>iOVYLb zl#Tf7j)isOn1`9-3N&PIbxL|Cgg{0|nHVy_A+7Jaw}lEvw8h1ixxk!JYO6}G+|OAV zSXoSzCIn`+ni?Np=QfC0wjFw@Opz^1xPE4Idh1LPy!Q5*OI91X9q>9f6VejXvR5-le`iTfyxQwh_YlHmZwbG_|M6QUh>@PT}}DW{6o zL69lNt+()^|3eWO{XNI=Lv!a95;ug$ACMkR`mfN2?u>ihYy)^|GPTv6AY% zqT~N8IsL{EuM`v_FRvcprQr5g3%T@Z|9z|H0E2`U_9e#f@4qa;`@n9USpMYCh34sH z(xik|_zAjaU0EO@)0{L`VOX+ALrM_uh!1dlxndgBHnJ9vUpP??QX|0k(cQkpOS2*s z08ol62_O;$$wV+O;beGl-SGrT62Of@1g?6y*g-%}CJ5dXkvZF6sA$0dKt=ySO66Er z`0Vmh`kz#i{Z3L(-IYBS-~BUp{rP{d6^JZ=Bnx|DFrjH-``KUKfz(tKe*eI$m%bM8 zpG(@MGXaoPT2LY#&%i2##*0`DAYF1W=V-0_IL@J!W|-yQE?``{eAvP(T)eZL`8C46Zu|j(<^=~Z z;#+N6gx52^x)8t=KlMWcfB7flwvN{>-z#gAYzt`|Z3&+=E_*l~h}sC2w!F};4T`p` zKUyZ-_9sNg0o#*Z##A=@1iNI7v4hU*&TNT#TYXmJ56(Xu;gG&t7IGlmwel6r(a+Hv z!AAJ4`Nv|h_z7cn7RgZjACSh8g3486FGjpH58tQWGMTnwOF1ojUX*g>T=g69B@>F| zFQ^yMv3v^x-m+cvORkYQ%=xkU4GDFb3P#D;T3Is8H?NQ@?+)~OT;QU>RY$l)^F{fY z(hSVnmD7ZBQfKn$)IFw}6g4x0uDUbO$>}!HJSa$~Iq51~ffNudcd5*>Zpo(CV_oeE zCE>tIWtW4zML|lF!(b_=GXA5IUeWCgCXQ6V)`^$KHGPrWH>rfS-pDG0w!V3KU5PM1 zw!6Ynr1MuN27fgGPV2~LJKIrnW@PV@=FiiTc*5$O5}966=+-taJ>1U;D~*o)b*M6a zbdJ~SRNKPjEIG`O?u=wN%-0z*daVa}R>k=%QA0njn^BZQ@8Bqt`eG(PL~GtD1R3{H zso6yq>UpEWM4FccS1L3vt*%^6JMK6UBSQKPeSNmiCpyWZ1#a)-PJ28v-j@F~=)rdh ztH1-`c*bX_ynmkQZX%^2i}1{+CTwWP7Mj+#uR6S4@J4Sm_w(9R(o&m*0{DRZ0U;?* z+-9X3JF2Tn|1PLMs1Zit%3m=HJ$?@#YNfoXyfQPAmACy%+KXb&(0H!@g~)<>{aBf$ zk?F?z6MTXG66x$85;*I1V}LKjc##%6AK!UJ$0#^-kZbCaP;4U<03W?usfOt&f*V$T z!XS}rL4wlSRy56hx!x+rMrjVeNLu~)XFFCY#yVD6WY{ixCxNb4-G-y^JXN+jnTHEj z=G-a0Mqje;eosp0ln#D$YEoIH+Cbaxe&ga{B!Pi?TuLA(_Dw|%I_w|RI*bXgLFD#? zeYDxsjHs@L6ltf50(;&Xz@qpdO+^{ts}5_jO98MUU3!Qv#lUq4|3588-hV(Wi|YP> z6w>M;QxMtw|Co{fTzqV*e%x+9Z@9zZn!0mBpI1pw%86|S~|w^IK;x#O*QsBV`G+m!WqohHD5 z^i~k}z?YIO+@OByKA$#&DWI}e|A2Iog_8(}yH}E{ zyg;a#$r?JAE5nsMyj-E<-u_x)a+WFi3o5e0Z2Y<(E1gG6ct8LRzXi!WG}4vY?&CMQ zhXi6qVg&j#I*%YmYR*paAeO=X#_sh}emq46K(o*sSVUF8oV`y~V0R{6WSB7aE8%ie z6_yhw=7LTY6UG9EWJl8YalSXsI_@My@Qt0@Grm&L)5;~rH4lNuPd4rt|D0T%jce!L z$*J{*+3DNtn?{K=wxp90gd0{S$5Qe*{Pr^VB zqRE7)6w-6HPcq3q!9?+)f&L+ZjbGSp0wK=u!E4nY)~lQZ9qeT3#lA^<6k)9TmWV<8 zhrYA5wKMne>|?c4->?|3TA>H*v1U!qjn?ds=n<*!+(?zM!O2 zYXm-2b2z(msdV#Qp_rl=Hm@W{1@s4y*>@2B$e|Vv!;2zjz1}{)I)3D#!qw5D*6@lN zON)W)MdMb{Bg2&pN6tliZRBrr+0IH6tBKvf_C71;r(@um|)E>Bds2V zPns6AO$~aMBgSBob(7bl)SGAP7R|TzK%@9}#&=sFj6)2zg7wIwT=caym=qSsVe3<6qPVNye2}wYMU(m#6l5eIvDLt_ zL%C7NiX=s+hA+M^H@H=q%*ntC5f_1idiDPj4)Y(Vi7F*3*SYwybuiI2L!LkMwD$YL z9yM`nw09qG&5du!Z^)YUw6lNVLxxVjpiXk3aT~AoYvU)ssYmiPb@D6v> z%}u2mDg|$vjy_E)Oj3_eVZ*C4yylsR0tY3vjwzSAH%P8S1J{hF;e)QXbiMTL4nqQ= z;K7as(r_~#{ZqP7=_}G|%+IeDwYE)9YKCy;EGnEfF|2FcNYbfDvHVP<}68YC%$@2ob z7fU;vUp(HZ56nEGY1|sf*8oqxN+vZ=}ibHG#p53d&NIj}W1Dp2_9SHS<3Yk12 zZ)IPJl_#tz{TZq9Q)cB$_DJWDPn{4PU4+>{R8j`mTVR-0hqC!lOF1tDYa9$1dpvKO zx2o%FRzDnAdO}PXmXJq$zmp@w_O+~ixny_uGU<;jCi%8m=T_?y@pPJ2=MB^Op=;^K zqR+@pRmf8BTkCzcoHG{aclTAANNla?vY`LrTlrcSDgr8rEeIW|4OKqaADHZ@1>kP9 zx|9kJaXd(5cmIOq*rd18g2`Zd5djf&#x^BIP-3MOXGJ4Q76)gWTkaHUSp3FU?kHM+ z!hX=rapP|1(?gD#_jwspk<=49+8G9rg5fKZ_K@220zq{i(eS6eC1q2|3u97PcPpvS z0r&>sbXNeUcK|pY49j4tV?EGe`LCH=?@xf;JuFf+u^W)EU|zONbB1sUkq8R%$M&`T zUA3tBPg;O^#OiDN+BJ3N)`VH`JkYs@6{WFG#Wj+FGsbnynL;4+(qR;uaT=53cY-@cXZU|={L>`8FabL*! z>G)9$?a2OK_TzHa!SF@{o3xEVtdGQNEW6*ro-(Fdg=!~R+&t6GS_g>L*n%knq)x*BUX|KnrQ~MfEFbTw0Z0TKr@tNM3=;gpJ~<}IUSAwzQJ71kHY0rqiWK|n^R$=Ls1{aTo3Z6kR>XygYqp0s*e z%R7`bqx2&R&nFz{z+QA8)px2@cS`ePH1G#sg<#Y9RE${qv#qg2Swh6Jay2;JgT0`i zuh&J;nz89O#b03)Ei%gVJ{iE5=?c8}2Sg3Cz2a=ur;Rb5G*=NzSC7CJYyC{p&Fxhn zQcX~3-T5WIivBLm5cJTrs%A_^1cY4$lqcSAM_>r+DP{E| zu{nD@ffQXPI%5o*S(1Y8ohcP6xm3Mp)%oetXz}Pyg$XDoh;KlG|B+8h1I{JHAU?5i zXoH6+*{%TG9e@7m4Ip!VR8MiS{mF{=OGjy zD?gr}FO}qV%NkN~!e!&zkuEa@(MT}C^A8l#O2QGA-cLbqUsoqJt8b?Abe43X-Zf{) zO?)Z0q?tgO5RhIH7tShF-}*7CKP{Ck-_ge}YvC6|x|vJi8v^I78FAujiGh{pSnQG0 z_vDYWCPXXabTN3FOJ~ONPby|_e)BrK+hSjjyO2P05PZ0KKJ?aw{(4o1y&(6bJcet< zk%)j2KC#?$!4i8XUkf|Pm&DoJLElFh>9OfR=XtE+wK5jZ6rnFGmWWCvO8ShICTk4& z!A6K#dx&lbA8E+95L|jwtwE(e-m1XL^9<;|@pQ_{2(G!8vQncJ)7M-{=XRR~KetX) z<{vs&5Noo%N&obu0;{thpE*{E<}f^#rv(Ml6g*I~8VTi_m>@UYr7+z%-bFTkExtb> zrd^`_7tIEflB{u#tBZ{NYE=_-)j8knpVq-~EIG(Zz=9`09pKSx5X ze(GJ~j=Q8Pc;r(Yl@X(fkjc}Zoh-@d?cPB)zhr2#uwc4mDp*%7oe*|SG$!!&quajPPo!?}kAR|VA5>a* z<*Vkj9=sXRMFYi#dhc?^d984pch^h!iEq2y_&%=;d6uI(53Ku_VHjYnS3$5anNN6- z+_fTcdtnto)1yE7QG{cl81i0vi!D@&82EJL0_A$Wd#H z+~9aRZItjDe{RO55Y!!HWTy+(XRc|`j$9mSW_%ync3F8-rcB_N@?Z`+-kj6Yca@rW zSEXq7mLsB#zv}{bvd)zp)1aR5+1c_oCT!Er*|{P@`*h69Neh4|kQbvgy)aKq0o-D5 zc@21Wm@I0VvPV@ib~UHnPr|-aT#seY9Eu5<3+BVxH7Xc(V+`*h!ReA=nDhV4-jL6bct{0f`4vURXM z@1>j?xa(B$s560-0)T9a_qI*1oi`CG{2N~{Wl3yDF%`>Trq?D|EMY+2W2m2 z5b1gs6zGL-{`~?{Ac6WzT2axe`=_+xUu$pDfznGs zeo4RZ4_(}avKgVpGgr^ux+!{PFM%1MQY%O4Y-6*S80PwoUqYD0K%T&Pl}h!(d4=94 z1RYzgV#^1k_MCSxZgeNwq^PHEx^~>&ia0!G!kqRN{nUZF^=B3=v7|~E-?TInDs8nP z{7I~{n@%bs)ky@2 z*&Ushh?WKr@1Qhc@m0ZmtiIP@kH&l$q8PZcj=b~DF+U&>p!DnL;xn${l{T7I?c=z* z0LM(xQ_T$7IDe5GpMBm!PbpTW!puJ)xEB-VwGES*5p~;kEP)2KItG+3+UwWwbuSn%j|$upAumjM z_M|mXH7K6caiT(uL}@Lp*A>Uy2xDbF)+)8i)bv9wOWxw6FMjq$`Fil3>ydISpD0^I z)?9I}Y_<{gFOz_%(>rLTAh9?kTU9`paH&km`cGC{xR>!Tep<`(uNN(z&zHUEw>avG zpKHsgqbj>Q=;TqsYV(5F+dqbJ5UlwK5WCtVK4CE*wH?cG2w|{W7kG76_TVri<763f zrXKUKq5$3&TDW>G_9G`||* zEv!YtlKFUL!B}Wn&g)qqb!BO1Sj-`Y2AEb&9XTuZ>($kvS<|^QkGBQ4jG}Mw5H7sp zZ;Z*Uew%S}OM#9hF8jL@2U1W{6fI=WC8lQ!2ON)>j<7xt^yVq2QAQ8nPTU$DDCz ziv666jrWyXcbifAsIt6Gjrz({6sBL|)yT@0qXv zeBJg&vDMSp5Ts04ROI3)2`aAZbeRUjwq_7sCnZ&WbJv`sBel=*wejtj3G}vc&wnOM z2GhSdn6e;-u_0t5vq{h+ZHUF&``y{s(X{D6-hL!LJ6`+Jq?=ffNoGP)dS5i~L?^5b z47yW$KZa$NfDWi7_@Zz1O8lbnmuiI@tkGo!!DR1CeeD{5YR0s)z(g=!j)mX+*}xyJ z&2G?I)SpUnseeY`#&ifo2IT8BTPy6XkmdRTA)ZwKD~ zatHrn*THsDtzqqaB$VO&L*xaOF?1q<6>=N zA#jZURc9z-{!he0ar+;TxX=C_6Q#37(CfZ`vP8gZBLdq6PkAp+VaV+3y z!}WLC-hVD-KpxohivAN2fUP}~3dCZ7U-@&YzI@NF{(OEwD4Q3l3gJ(STD1tyMYEl8 zQeb57$C;1pBNZmE(HZULNVx1>h)rqBQQM5U55`!Mn43vXXc!Zi!}+&@)X_cbbTQBR zR?75Lw4STRsa7me0N+!M9lpZkNT{eQb%s%0++IbjiicfQZP@1l4t(yq_ql`y8LV0F z`|VklavVKa-j>{}9wuxIQ#h>%j7tf(yh}rzO*3xor=6qp(Q7(h>Ai>vyF?_Nol)cY z3emx8O+QTu$kxU&21&sIRQBQ-`ca%(biaz^L@>ow{WKV>W z8g1U70HjLVa=`7N0L%`?dIi`(%5>Krh>dlk$JLZA$A9;0UNUE(?b29|`qKJvDJcdJ zO6o4?FaH0tZn;X0zo!l0rncJuGjhKD+Mlk7P~eIH*a>F?(3Mm*^ss@xWMGTPZ^M}Z z0&-t%6$SoikU0^5NmF)#95A3AkL0B{2=t$Pfk|{_fiYQJHMQ4oM$ZbhMtHpE%9sks z9d7Ya{MH}6Mtrw2VA#shoD9tMATX=wo&9kXlGdx~W7?3%!80ga+zA`U+9-^kVzeAv zdiFlhbx4`1fjY$4GkL2@=$mf`+cWjQ6rzj`4dwj3uYh~}-pH4h^vt|KNaaAe`f>WQb1C~_0IZV@(0xLLy2An+yK4jE~c1 zXg9rKqz5X^Tz8Yriq-~pL}psZNG^s0NY1#*;UFS6ER|tz{m^M8^#I{$BC)j@3|C>~ zHAxz>E70GHX{W$wBDhNm9ej?HT|5K@8oml&YtDchPQMmpfM%&F_8Rc^l=NSlXk?Ry zN_Aa`T~_x+(dnmwR_AT8?b?>o0BWeunpSJ%w6^r?hVq3c5464$k`vdks&JPzbfa?} zEO7i;p!1Kb(QIKeW{MUmgoaCIJrZz}H;c+c7xdfAeOIZ5wR2jySCPiQwz>j?B)Peml1oQQCnrRrXF!(ob)FupQIS zbIV;b)i8Hg&)I7=q||3To~gYn+unC8*1Q21>10MLar%Zxu4B&$l*|VD<#fy9!?;$3 z$#ychVl&11GC%O7__CG+ym&9VU0X5Z8^IE}k`TR?QD}>z`{W6N$52*w{JyKD5z@0& zQ$ocp-Neo}4q7Cg;o``r4LB~mM68NNR4GnY+B=mPFe%1_pC#UKPiF4SRJvPTY zX)sPMUA3FJCqFD=Of(qeP=4*nK1)aX#=~_H82UrQJ^Q%i-3ta_T9QB&p9ee39;q6P zG|#@;huudA$M{r4(PbhmNT6w&Ge>35jqRM+1!?^_qd$We__r5Ky6-x(Y1DqKjk^?9 zUkkg}Tc7t{kAa5=LRy!T50zI&U6hyac+zGo9M?&jS`2PVc~5XUGS<__H!5g{LSF2u zJ)QfR*|Wr3gE`HG$)Ptw{PMdl*ic#1VTt|uOQVjPX0kJ)2Z&>tCqF8JmyMG^-j>?x zJ+`h4+75&sYRae9V2LBSygrK{mOg!4U&1ye3IdrAWZGotJdTlF4G%ra!Cl6v4l0LD zb~kjbm>PC{kG${|(Ky6ubP-x4=!5p*T4OR-T!W>}ioc2bWJeD*Q(Ll#`#R1?IY!xa zh0fLVULSZI3MsKT_AMLOp##xNfEatP#+5H9A|hOHj`xJ{7n8p8+@6)KuDW)XMpvs|XPA1Y>gN+`%f}>dnX%p`zI@~mn}cHelwljR z4`a^2UWB?J8wQJa=tRukb6|Ngn5AQ{I1|d5k~HS6TD<}TEE(Om9GdbolL|)}(pMEp zFP@6Ug^8*howP-#@J{SECEtk&A3Q*}Mjx8Wb}omtGXom$+36^aP=c*&#P>6)_x!uM za^;>?6ujhR1ull4AWm0EMWPcOmtaIFt4=^(e;0BQ6%;b8_-;wA;^pi2!dEt#^r5B7 zy60p3|Hs&S$5Z|P|Nlo;R?6PWmc3<5b`r`S$vjS0_Ffs;D?%KZAxUOt93zBmS%)Jt z;~c`tdVL>yjraJxFPGo%k5iW(jxL<@d_K|rfRHUW6g0ZImZkQ5|@BBXL{)z&ZR6Q$k2+Wyl&zbsrLqnp8HTqPDtjItVu>h1$-=fH-lhUF>?LO1Z`USK+t{WnvH<<$RK(jmG?>C+eNxCac|z|&FO4ID>6IBJALE52A8oL-Pj z0?o z?%q+;cLx{uZOfdur$%2D$c?e&2nBrfcrg2@L^EH;*X^H-T_SxeH(T7hdU^CtJAE>V z^C>@91hb;4+sZH}D23um->0$z^jklRYMR@t$*Cf3(lyw9%$X>}B6`%Eka2QiqNko% zGk5Ev&*BM^0}m55owt*|ZTqNRP0+-3a=^psadUs*@NPL!2t%Qxi?UBD#zuG0v@v3I zdLxG_e4b9Fm=D?(xFtYW3#`W}o86p0vih{zZL-!%Hn|e?Xv`g2CP1**d-9rSD~8I~TP2kJ!H>6PgeA z8*Y6X^o>TqpRwSe;ZdHum$A)qE+w5TqT`!nc0i z^0FQ=cFHaZ@h(kDd{r&nv?MQr2k;unA4@18I3i+^k466T=|948`%Ry(jsv* z0VDY=e@n0h7d05t1%8gG19GCdKD)OYY!91!OCB1kJ|D*sdig=kAoL#QQ+XQ+x^*Gx zM8|h%Po4`keA|_HebPHqK}Dch+`!W1IV+>4s%?KnGJTM$-%v5Sq3fIrxVfK&4`?;U z?(KKY1XmQ1e4e56Vy&4qRT%fCv$KDm=JuKJ*1F?Y|L9)Vm^{%*1*Q!k><*O6L73XV zuH4|7T$R5k+xM93kfOqiqTtt^LA6cR-02>Y$Sm2TaiBzAH-dnisRp0+|DUXz(iYre43vsMXslt@5Nkv zK0@Qqjeti?9Xo{I6~-=bL=6DR?|uH4GqrsbI1Cj+0}b2*59w_}MJ%ks+k~Btqm!xv znU`6Ks-D$(R?8H>dhY}ulz{A_^ur8kRY70Ru z5DA{u+7wgJnRU>3`SRLH`^(5*)qUOG3Nq-#(yHLG1%_sA#%iEY>}0Gk)+tlRrDMx5 zsjPpU#;9`jtr9mk^k;GYn0Mp0%3;*x_D_>jYllUm-E+oIq*d<(OBoh4pOlh$QRgAr zu8Yone)quk#^=hUYk>}vf^rt_OYN7i_k?JjXbdHfUwm)KF5md7PI}UkA8(VAJI&7I zOe{S>IyDUXc)?e(gq^|kV=e5c-EXKMhn^Y#IPP(QKLo} z@)&_r^-`=EqK%kgIQIK!no(;_+eB*rLO+*t4rvF6;0Xe9?eedu8lS6F(c5Q-Wv{2_ zH~nFb$Irf23_lv<@0tI+B2`5eG3N$bOJXuXlsU3oi$<|NJGF7gP)un&&+Jwptj^72 zEU{YXs+na?TQh%=emPOya$x`+6sX6FP`pwVKM(KfoKXFtLHR`>E#X8#=frzwwr2}f zyru85_87`Dui2De4JE@2w|{sPyKdZp zA{h56s`NNg&W5J1l*$h`Xu9*ApxhJOW3Tw+RP|p`}jQ|A5TE(OJX%imoTF zX@~NW1cB`WEnjxe_kO+!BstLp7xD%0V|hval)c>s2!8BmWRtc-8vNkUxoYM%-3-Z) z${#IfNPoy-lPOwp260#fKeVexx74?hr}apg9?X90hLU0G-4-vsR*Jp70*b>+jl<7E z8j>-YUE5-xf@QI=szBkX0~dbE2ZDWK9-%(nl!9AVuZ)WIs6v9QR0Vl1JV8KAYsi{d zH;_c?%WX9&pnpry^Mq)pKr@WTgxww|^!hS;HT7x;&>!?vdK`vgSxc}3U6M^fd8Hm+ z?j(p3HB2YXE#fkq`B`bE({Y?_PdzOjNVk9+l3qc~*Je>M+zz7~C&=9}y8Pl1t9@8W z%J(OLA2&sqs&w>jvS&l+?c&FHVa-AlDk<}8De$3CuHu3hjyOuc|Ae}0>Vm5X;P_%y zh)sEbw>S1LvZEprka2Oaq%fcH z`BIdzoCU@F-D)jN>W$QSDvJBB=m{sAhsjR$o3NF@38gn4NbY(i7X_1zuyZf=Eum+F zF{az?vo&h;J@S4PUKH`-dNQ!zJC>B}3ivF0-5o>AF85wdFPtEsoE*CFDe+uhV_U=Z2n z<=CR@iaHZy#oKJ77pZ?!vq>bMdI$6Cxv2RIC7YQm+T~XteD|rkllUM{JXOjlA=J+9 z79Ptp1|^x;v*$I8ejBg$W4EBw z)|%Ru50{XvzBN55fx6W|llFKrP_d#dtB7&srCHncC?yxkYMU6F>p$>5 z_xXl8Q5#lt+?Q+Fo=?`x|YQa4=%d?dT{2gBwT z86fSdb|APsqfHn+l=g1=ZJAw}X|Esgm)GX(Z|vr9t}|Vmn-Gdo5RF*+0}|n-y%uFWvQ8)5jPVrg`Bkb*f^;ufbsx~a3l)hyQ>Oqyj zRP8#`{J!69R!!~7tukQ#Ak@Zm$$W2ee8rXYX1I8cK3UK{j`48s=|}l{GWX@LWM{*2 zu_4QYQYfQmPvq2#h+U)a<>h)e7lPl~yiC@XZF<(x&-TWt%~{(?IVjjZ@d z|8jq|Ge1jLM9{0Hg_dnk%DWM<1W%N^Q|}LL{d%fD!(406E7uTep-QvHVz0fO{3@J4 zX}K<7!STHdFd$m{JL3W}ZcZ6r40dEWU^iNk?7;OZwAySqz|oqBd9+5&=S$2}>hzyB z$V%Ml?e&sE7gg@YK^dSYxPUWU==XmQ36cMDWM7O8fl2$Hm7Z*Z01E+50k+60DgXn` z{R4s+KB1PVI^+3pp^y5nrIJYYS%87it}j_3E4?{7dQ|L2OQomT1V{!hn_7g{vt^)B z4E)pr;`(c&U~wr}61%s%G>nk9F&!&Nbbffpds5`f6@F+)H2o^(AG1i(9sj~P%Fm7|urIT|X$og!-`y5lkG6BeYZU-a{~>jR6qX4+`l za5=$quBq4WO}f*NBxYues%LYq@`z~b#&;7q7KxM6%Mxa)*fkF>_%fqKNar3Fq+ZjF z4Hdf=)}e;UzTh*b8VgdZeSaMPk%wMm#ozY51xvLm;C*?W7;G{qMzx47Gg~d49b|?c zRJ$^4R$D%Qx0LOMr!;dJK{x+d)p_FGQmjFD9&reXLN!caGR23b zC^JqfovM$jLocvQ>5+H%rkmTGo;|I4*>>@)C1-J90LCc37{kGG73=Ah^^?qUO}iO3 zX#FHVgg2DMpqAfKjZoHm%MvF|UZPUbQAhS6HmC?T;~n?L0%lsdya8@h4<;Wgm-xmE z!pE~z>MI`|hAzERdx`_}cK+QD#}BlDoW2pr6c^uDH>hw-8R_%8--)OY6M^7t!Un{L z<5_N|8IH?|yv4H3s$z1WqfVweFDJVWW#S$C-{F6LCjO*&%uR%e0MDh0A8U;X=*FB6 zDDYscl1{lxv2hw>wcEG%&$vycbj$rbw3`-;7iO>J<2v4Q4^_|`jUyCQsEl&lu}*Zf zHF=z(QWT?Xs4se%{Y~lApdyt5d!WkLInOD3EjlUs3CBnspWmuBy~J8g;&$6WV>er6 zO{7K`zV2Wn;I1u8tmkoJTi>am?9io=f!F19P&olSRbJBH$0tV-_b415%XcJX<;K=e z825eZo4H(oSXD36H;P*b9vLh;mBL0=ZsyT6j>+jQ=14V<2i|h`E|R{M#qRiyTEp=> zWa;qxqwxHABLj3hO0oeQ+-~pv=xDHHL8CQnN9(kkKj+qQKGK$s4E9ZT56ALfmZp}8 z7L`4gnaaeTEDHcCCzoR$;h3BlS9=AF!LH}^6T>Z~SnE`Aw%hlg-PKI}#T|O7TH=di zNUcgxR8%)K$?TOJcf_M^%g0SZ5M22Z~kXR##<*j!Om@t zX{;T2ye3ImEGyYsdm&VTzUPCOc5ox-J5u;&|L8ImpJ{`O)3R~csmBDaBtO&nup@X+ z4MN-%_TVRShWkWH1v9=Njvha!n+#+owy@ebY=8x(b+Bc`YH-fvL|h%%zj;-8xRSzP$C0dn%@h*zhIREM;G%4%b!oe zYm*{(;9FBAGYf0?TJ=>-=dZ`~UT1&h5V55boHCW^D|G0GUJDBr!M3AxB6rFpn^B&| zy8=6$36+H;4J@WH2=XR{eJOtkmVrFJsiA9lhVszotDFg5#)2H$yV9!&=Ki_J)eUob zm5^!oA;(F$|24~Hw9vV}Dh6@h@jhg|+39eXF8K2~0VcCofE?AbTW0KAsO>l>5I<7f zI>zN)`+20A12?`xP>JP5M<5VThI^b!Vuvw|rkIk3N^JSmgNY(d?r7cSdJU?#FuEC* z`Il}H6)XTi1x`o!ayj~SnTvBX5-mRdAj1l2=PJ^vExq@}srM<`46-M4!0d#K2*y8k z$8KO|JCIk;C7?63&-ep5w7+eb%EfV{wQVD0?b0E}yd3Syp|`kNC1(%G(PUNOr{qh| zCQdb;$la(=sIg`@pZ)dW%N`*g@1C&1P3ilk%s)1WA!y{R8A_oUU3KV;agC9a;+x%E zPcc*Vp!>B*(O+E~*U|CtvGVq0n<&;AJ&W>c?M1jUg4oT3wdh^$=K36Cj4eP9JMT_) zUDXV`o-zOZK8bB|A%|y&#IXTtm}mnu1QrVOVf7-LrfRQAls#aA-k+o2ETqXZ=I$YELaEIH8b0>|EcGJ%G5=}MS@JpS`N){#6wA%1^Mze4D?yS3!NuF#*RH?Jx%ugx`lHKz9RR&v zx`lY_w7@!{=(ro5Y8GO%&Tub*bFd_g^K83p5hZyn9W1$&V{) zzo)4ofn)6T=nh+wO`c~}QSeeCfB&;~@h6tzUT z*4`cr;(M|B{pE*Gco4j42=`>B1-K>r1FvIfy98W&XeAj!+a@w>1PaW#zdg>ruQHVK zffdeg?82$yQ6H+HhCOp71f3-;cRKL>ru3b=fOR~{mN*}_N+hw~btcP!t z2h0^hE=g^Hi13b+u`(x1n$XPHfShNvj_20N(w?dopVE>t=@ui5C&4aR+F6Gcaz+pB z09K~{i1WLm*6`!a??Fwx$fy+ChY#^W2`+74e@e!e*gGWSfOvQYHdR$5@#RLKn53HG=B%OQ-nJ%e~t~n4w`q3|Cd*I)lUWjDs!D z=M-W-ZZ-Wj``YYww1g|Y9Bb^<2S%S1ryZ7}w{}qkQ?(>P+7G9^#oe74COUOEi06dk z+1SF>>1SX0e4lba=N$T?*FY|k?`&7L#7>!45Nfu)9;qJ0?@HzRG3Oe7f(E_EQJnY1 z04d^}+t*Y#vYj3T`{>T_eF6!LAyK;ry4V!#nEFJ?HK=gSHHNn$P335#mB65}$S3>9 zVAN!0<#;G1`7~>5HT>3%7fnsn_+}V9qv7MJEwRORsvqQ%?WlHU*fEJk4ic;CMSfLw zgYU-EQzfdaINT6P2&yfnt}eEJhR>JX_)#RDdrt&$IROOsE3pdK1?TN?tk>Tt!d%VK z3#Bq}_onOXf0YFqL;c3i7RA4!;QoabYyDq~7XfiRMY6xmz9E2s`0rj{;Q#$=2}20t z?7?hwNx2}F7<4Rvw?fN6Yti>;3#j`zRJ6CxbKijO*OLiU2t7x%!PgL|fUXYljJ8hE zH@T6dRPE1!)e7WKUl-~J88KH(c*4I44y!k0cg~_*yEJNf?XwLE%{I|yvq_R_DI=zg zo-d5}=Q#-T>gIlg;GXi82i}A)tyI3v=8oo6)fLjQGohs*B5`s1_87tWlH%!8k3rgN zE3DUo={AtGijK3cJhXK*Br@}{oiUJj^wa{?4TP5Oet^~}Z-be3f;OT- zuj|N-m(wPTGf`uXmm9BSDJ`6*kwri-X*r1utNsaoB~Z`zoV6Mw6ubICpyY?>ysm5U zTZ(%bFkhR5JuSZgAg6+?HnYaTYv2f7(o-^QBum!fpy1>!@#pV25+3tciHpdp@?Fvp z<>lB_lnj`9?PP^_daE)}F|O0}RZ~eZjH+`-UTf|Ci#oiIq~_)VJwE(liOO`i`E;n< zC97x4w?_}06%8@2&jHY^%q&NfvP(+1Q6uKj!#g5sqAFL7!+UYFt!=~(-GR=g3g?sy zy>`l;&7AoBthm&4ejz9dKW_Ntr>qprL{jBbfWWPt9gag%i&sIQQ%H0|TZA+1b1n?^ z`@ys3nidgt7(7KA=el>Y%(kpI^EZ|Rbo)|~6^S_OR)7@j&m z({EN6RM_qqR_UPO(a6pPmYUJ;6Lrj9+fy(nhT00wFTET)-VYyg=k)_WpWx+RR=vo* zJ0Y;7=$Fn}-+Y6dJw-er6o@0D<#%F*c z>}=&msYg6Hl_;CD?686Q*!iZK%J3I&+iTk3eR$vDI4;E$?xt=kNF!IFisp*I{6IBy z#U(L)d)E$Is?--?)fyEn_HeZipW)4Rr%~FsC?3v=53LC2>krf^>{vp6t2!(n6L8rP za<23u`COR2=^>Y+THHqcD%JSSucyHN%8`9+&~0_;C-N$`(~5Fq;Rc5GnODG@RI=N; z|U!7-PWMde(I(EuzN5z zQKr2Yfl8%*=}OBMq;beSRlm4lgKpXQ!{M%qwa(iv9H2j@C^LJ_FD1GE z*{k|dq8S1=9gi6h8weQq`WVG$+igtiZBl4>gus?(tC;#!i?&2OP0Uxq{X2^GIfEWR z@#|tGgv*rnv;0(Qj2%;-f=ZD`9!6yrYZj@n9PDM&`cBu7eH3~*m8|p~W6UCnr3Yf$ z?g~^6nU5vf=Q+c?v!H~~m3ei_`OcyjiMAQH*xia=Q8Y*0yL>S(7m!Q={snNmw_gd= z{e8Q0gWH{3XVR`Ik4)fEs%MQEg2^@2U|>FUR+>;^;Z*f3H<-cK2M8Auw}0*uK)olA z7#`H%q%7QAi zv?T7i1J`Ym)gbEx(q)0{=vu5xuR-njsToR_UO9yDTW+9=2Wc2v2mS+9DjN zTcz&PYaw<>h_CA>0Pd3$kc$2(%gkV5pjFe{29q`v^AHL1U|-&q;!ZQ~-DY0yt8DRG zR3!a|g_&al4UZJj$xhim(Mz$=(>GTHj)*uCZfEs32>b!zZ=~LUlLXVwo^pR3O0uZ} zVX--M8rwG}-|2@`-wut8VtZ5SNLZ2KLSVf3u$6UbA-B-?sEb8NbLX9xvpq?GN$}TA zn{YanCyCdosP!#get(sw5ybBc?|?Id!}F;=|dClVC`id|?G#yO4h z_m<`S`k#w6_KZ)z4Ky$hNdW)~`BK-6Ro7$)o?|?s%fBR@d^%0h_DW+!>2Vw3z*y`t zdAHRSz=5Ih&FP#uJ)-sYPl|T2ZLCWLyr}L}F{TvFkN0dIDPMoI@?f<_P6{22=nTF! z8K_?J!*u+r)0q6khHVcP+A z0TeJ^cZGZ!M^dH%Qrpe^|%Q<5`da`}#qgTjPOJCRn4-T2TM~MD0 zrhzrrIy+^yYg}|L-m~$fS9oU}w$_SFvCRle?(r6Flq&`7oHf<3wslg7Ejc|^j6UJT zW;N9^BYrnmcr#w zT=%o=%|#w-%0rgh_?ZKAu=i=h{?O*DZKSYq6U1x`_sUk`*}rc*~qjd z!7vXLE2Y!a$UqsGIH{J-Ze$)B727uWKBj<3ZV&0c{ML7CnGV1}OBaQ3*TGH4XOq7! z35Kv^m6=iYLL6WGptc?Z%}muceOxU^%iP15fW;3<5GpC+ZdjJx}2IpHF{$#QH26 zD==U!k%&Y8lH&4d(;5VZ!-Do0D6qGaatAw?2 zOkRG$KWo-Psa~qe?&7;WqIWwYRCG26LN;CWFo^{FvB)umlD4;L=Q$enqnozLb>NHUcE9qLqQNA0_TsN34cL7za-^^6`#w1oh9YUF_8mKPniTpDt*Y{5+*nH)(rz}r zZIbj+ZYrQW_{%xV_`5)5x+ytzhx{L-+VVj7;p{r$`HL$#a1WoyLQ_#h>4fgUAeGNU z4a@y{-G}8;Hc6}UY)6K^G4$LvC57LWq373t23GjFx_GzsI~{B@&F zbulA>+9&~)rTS>pY~fq0>FLdx=(rtz7tii+pD6K$=gzEKTd$Sp5kM zBmynoi?Qj>Zb)poJ1eJGWUg5}>uYb2wlvPHf{W0-xfEX*UWCV~rrSD29fU5`Y{Vv% zVi!hN_cf=S7bP=w9#X>dn+Z1~cnj+6>Ci1X8NLba$>Lv462; zmrLZEErUi28FWg(Dt(sV-9O+g3%&+M6P}6`jv3xQUmY|4fVeh~zIz0vBZ!r2seYfH zFcO6jy2~^fbeQ(;RILpgw;ZuFudP5)Add(Po!ij$#a@=jd805vKTy)}zI(XJa6dgsMwp^ng%e{qbHg?cXT^s(FPH2S$qqLu18^ey79Gl}?iY%i-_m{Ab`?y1(WD_) zt((yyFt_SFBk;@0tddU&Z=qbnWag0wBr)uq6K!iL zkcaH1U0|O)W&I!2kMv=Ic<+Jtke>IZ`k*!i9(sd^RTlQ z?3vf;^qBZ7KlRDpMhoBb-S_o8^H%)EigG}xqcda1x|)JBD|%u*KQb^PMNLeFYBciS zc29H=XR*JAV5}_lvds}Ia)c<=yfTpAx!4ff6YV`v1M87dv?Ckmgs|ON?^LU!jRWTB965qDT{&jQ5V~+)kjiYO-E45;?>RC#BwNVcr zU0HOWY+VLoI* z9wk)yz17&bGgYDs|9+b=T_gwT_s}3-B$;@?33(in#?|GCi@>&!tN3AEV+M`!+QZt1 zkK@(o2o{&6<&55psP+z24kt>GM>Hi?0xNvOaeG(^MyDY<(5S-7Nr9%NmLQ@;L zC{OVxhx>hONhoHS*q^SE3uIXTDbt?Ui&YUv!K8Wel8)_lRi5NqA8j4zgw+|t_&ob$ zy}|neTW8?ir^rv)XZKaNaK&=JM6VqR7XASt0`csqu@mmC4?$JKCz5hi+>A$S35`QA zkKW0q%*azsbU=3{&}7y{#X;rsUK6A}J~EPAh1ou|AwK<1V9taazYhojI)yI-ONV^G z&gx~w5@?C-Wt>Jr{e5Ci=}R^yn<59Bi-MuF*6EVMCUV_#`KhAvPR4EA*aFumMf9@< zgcU_;VtY-sVWd3im^lYh-FT4n~62CBUXykUddW?V`-g#9|U zipy_|6xa>y-kaek>qWS*VmD|AOs-l0NQU_4_(8VD%{Abpo^=2mq-Z=~_h`p%| zeD`sQAfS?m066apa_eJNB3>_5rv8NAfu|Fx{^o4dttwI|+(Mv_x+hr}B03g$Fy0k@w#T7yi2BLB znAb*^wSU$0CcR9|$|t{D&{!#iYRg{0&*`@0`p||AD(ztB7{R_rz&L!b6L8vkA}BZHKFZUu$A}+4A-BGosKThx!@HnNXLrl_)s=gE6B6VxGWKxP zE*aLhXn!7tHa9obB`$l$ar13=;?^FM4dk2(XH3r=-c24esYo2i=z)^I4c-fbaC+-d zQ)o<*JKq3XPi)vcE$2%wYhwj6?1&Gl^7iV&{!)8FivS+Y2lKkaW+;q_tcLZv$ee~x z@;w~7#XTO}F?<0JD?kJ*Jq2)!NYwmI#+)E2<2vmD-zsWDJ2iTo&KgMYvoq)d5oRJf zNubW;C)VYLj+B1mXvv9Ic}51!%vQ7Uist$~^7qB=xy&-j#OMI4nSIUdo*+}J=mq(D zbQh&IRU4a(Ii*QY_2-e6(*E4<)5NdfLZPDW8V;=n6)BkLZe3C=WvBN!+k^zu##FHB zHxi8*zbcC!o5agsX;P*=9}+YUkk7!>EwqMayYeV8Az{w0_GSSVx?S0(2nYUnr8}bP z>@jhap`72+dM!a13e?>^_oypDOc$G5BBJjg z`oiSwt#BIbzAoCnc&UPM+Qlx@YFsVLX>zkzUxl$RbAVKk-9_!51}b?d;}zdd=l@eU76yrb)e5DS1vV*!WtYFDTteqB=9Y%cWEcwJl10} zeN+A6L~{Ow^jTc%ECG?Cpqq?$l}G-i7ce-cH%VM5>YN#FhM_)&9(&%c-n}`rr_oEq z2P%#53vnl2_DaycLjwEwnb_({0*6v6fvswf zUp~2Y@jKXE%ptr2U{=uS!gKG63uk66hWW~4Ot?kyF00t4tkgdx)Qg{_?M{l%XeV}& zQ*>Hjm^w9h14|pgFHVZik1~0ls3yP|T?tqtXW*p!e_wp?wJ5+2{5JU)<7P)H{9;@xQeHQxssQf$9H=88`GMCNpGr?_ z5xjtWp|$BbWm5Wdn>yn&;FswGf>VOO=h%P{Xz2(UlqxHI919rdAP@ow7K@xfdIf+Y z-N=6gtp#rw)tH89PE%MSK&NWxK=5mHE^;e)L8JvutURwmI;knfYL@XoJh3mfB~ZwE zsncX;v-8d-q4V5I-tNtq>?8VZ&-CqDi0lIfqRC9xRskwf5pB(EQ;?@zoKI$<@>lS9 zm7&DnA42}6L-?Eoe}x~!@(lh@+LOg!v?s)WAfW!4)r0t^`s;kKQYq^9YN-(h6JmUtUF%=VC0!dfHukQwsZv&Z7HH zW=l*02Tz)vd*)mB^Tu&(I@caxTqC#bF!0DN&mi5QeO7I=?(xUZQ}!9(mWbn_8F#`< zH?sEuZ8-B>?wmbjdZRpK9cDRfZiNtvwnD$N#xRAcVN;2GAb?RIxY&Qt=;BORuk(=rY5eSDyifmH=(feP4ueT32>y90kt-;}0{h zYU4@H*~?*6)$y-?MTm(xv_YtFB$Lq7r((4qF%OU8hbON8VPoqTw4pD-pjSP{VzZV z(ALnt;}+m{qOC$tT?;G(8e&dE$YXzAMmZXeT3nG0$Vz`VvNrUA`YGX&&E2DSk9!ws zW5t=7Px)R4YJD6%y{w{2lXLchs^KtM(gTmy?$ZNEa&dvTUMORj5|j9X7$&594a>n_ z_hXn?7v;>Z=WJ>rXwgipPUpvF*iTE!Xd{~2ydUcj@0IJIi|Jw|sZ2xZ#{RS-ThC~q zHCkXE-v-p2lB}YKu!=J7@Vodie?ZE6R)Nh9cN_Gf2tCh)y4rBtN9`@7?xJ5L3`W*x zzu%!y=XZ?T-}&YJ?B}gvr_$x&XZ{&=%i6B=zUtj%ORQ9!rH4UXqtlfK#oO;6Z>HV0 zAiCN0!p@7G!y_9qe3rW__h`l{hW`SAA3oGMiX~%krIK&nq%i$}TkjCAN!_n-Kza|= z?2(pc`O(tKuhjPH}KuR*qd=h_;xFsuwS_36y)ek)oP875}{TP!=}gUP>m-vrnY zHomrZbeivuEQZmoaaYokP=~-i3Ob*Ym}@(@H29=!Hrc4Sa^#uzqYf#`tq2{taiyh) z++ePVU@l-WArWvvb}w8SJLPraw-q)w!9H{1G7(nmY`YWSyLL+=1tk;_ERAVE$QvMn zIk3?-Ar*rG&VF_kX1<wIZFbuFdpFZUh8cvJs*7OOxv?{ z z2lM$>sbCr8WEO~m_>?Dvc7MJ+g_T)#%>*h0&GoIy31?<03Dmki(4QW7bE~QzAAIaY zoRe+Lkas>mC9LSrKF@k8IJ($Jn9ixWl6t$@M^a?f=GJx|4{mq^dJrBRtZ>SQ9{f`2 zW&PS$ZTEz?1ee0RP?@rCqeXR$8Mn^gsApbLXZ)NTJBl*Ve0V)n z`G9($;S+tGP;?v1*8?jWqz+2)ipFpu_FWHk_~A5ZIRUAK?b*V$BeD-RoXQ*bbi)(` zF=4R-DAumNK!c^m6%#Afv0BI4&%55q4+V2AX6}6MQ=P7yyVebH4<=1LUS96~bW(Y@ zx)r|Ez58mEHLD2u*i7McwRty*R1w5QiDxm+c30E|yfT?fOX_T5jb$YYSyFH&*BkX3 zs3P*sxt#*PJF#V*XeD5AyN@bGdY0oZr2myC-c?n5UzNqxrZ)bu7M*1zLqmd>jsS;+ zC`pj0yv)$ZD`&quT04U`tTH`rK`2ATG)Q;9P0>1I_zZJtbg${6D%ORhq=slHC>BHm zcw%law>=IRZzBZs#rCU#8WSy)C?u|_ul^1h`-ve*oQ|8*CC4u%!k4eA*M6rQpBSbM zq#Q57cpb%prXnL87j_)W=EQr*v%4kKPgcfJzhiV1`p8xNwi0BGnmbg1bn}=zzUtH( zRYgCD8m6#-RumJ)-HYlOrNhnayi=UR`TQ1?;pMRCSw^5WMpQAXjb!XK*1be$$y_{S zwjG^@pVKF%V0R+)ttl~BSFFMGIJPYWE%m8Aj+MODSxRNJO=yEJe_F)2vIy-64J=rR zr6*gn)R<4|Y?WFXT#KpGhEpxTD&@QrDh95+fXcs*EU0^KD2((JBlJq?cfHb_mI>~j;ej?_6zE?R74 zPrh6deCq&7SU0A=9{%t}m?2&`->^i|x1*kOxy0dnn5k}uDLHxI`FcabEQOdnq}1i0 zcw7D49?7NlQ{S7-fo@U1aSP@vbbc=mWpx%N9{*Q^$`gQb)scj!7C| zowx=)GL4?rS9&rnBKpfUo)q5X5j%nL0aGmVK>bNMo5kUndKMan%7n_?!56#H1cmVH zpC3l#yu`qTF%1o-R`h`kn5o#eE#v{TU1p3zbiOVD&*3JnSBM@p?H7t93$%T07A`&6 z9!(yyS?;M>2HJ=(3No1h@>zp^@zYZGg|ow3GF6Tk?Q5d6dj&Hdjm{kx=7qyu8ks7@ zJahn3sQog7JKj%c@Tn<)oKnG9Xfd?W=bYFkM6Zlx^%G~g9C=b`-h(G^S*5cry1qZc z#2fL1IJ7fcIhJjs=T{IZ05h2rT3U#)>Dtk^y7biL?L9>x{`i8;~%LeVL` zX=~Ro9j^|OH^S1ee` z&;E_oaK@z-2dvSHD8|vzMzU_;!{r4f_J#dcP*sZSBF_=PUqE+UktXCnn&USQ`~UA6 z=#78V*1R{!7HSmhTXTL7bXWgAPZ-rD`g;N+0f5oOf5L(`*IQ0A*n>(qj( zzPlhb7UD&f-c}oG3W|Aia4>2vgKUE6h5NE3O|kzQLXak4 z1Wr(~7e79IzO`l^2!7Mgj_n>}d z{5+`CjIe1k`aax5DctIwXXQc%`#_{^J5~xC;X{QIJs3ZX`D z)AO%L^4&ICsV+0;z&T&(yN}LCqxzQyq%-?EyV&;`*M6x{t-V#h^7~fS4*U>+BV!D{QS#^KLFR* z=5Q3NF%>lq^Y3VSuurOpj@_K?qzs)69(d&tKV^g7MOCgP)$!$M!=ujy(cuW%mx6Mt zFc99ipYY~!o=S83b$4S!#XWC^snkg)dOhAumlV5_ zy^9X3V9C9-_a4sZkgcx5l6jWQQECqGcZ`?))R6K$IqEHiK##hZ1qyevuvHbLR8$Zx0SAf_>~_UJ~TG_ zao#lM(YQ8^e(P@&`wc@2_!!i_E+@(pPn>tf<32Miz)T7WB;(_4n5DNjVt?_rKoz3V zK8u1?Wr%WE`?p}8)Nw92jB{nB>wd2fshy3IsoHeK*w<{OAWk%LIXXO;-`vv7*T#&t zVbX~O)w4PSlYNu^iqO&FPBQP-tXmMQxeDIK>YGiTXvHKD=uiZm@)yiqvXJ?q-jZbe z%*=_>{N9Lk!gk+2+a2Sh;Gcr1#W(EP$HZZgGRgX*jhz{(nM|Vq;(CET1os7VWN{IF z@kM)DA96u*b^E=KD^dM_x*q@8m>+tQI0br0 znnC(i7_ivNx>nK^yD=v6%xYgm>=xN5btaGb=-;PdZpvwjb;c!KO^t#1&QYRTf_|enhD=`*(mRRb$Z^)SO^+8#p{aiCn~O;TPmIrG`y=gQiM4$ZXBO zh}ilaPiuZ(#m{z&aXT|MEl=f zFy}v4$p2X>i4;h320eQ%@cX^^#8Z$k4)$#Tj$n>IsWerOI8$aV(k# zDcMvW+TGVF635$qf@E0;&~$34Ttt$AZ5cGcVrBGcKo(1y4p+8mYFHc?o9Z(_$S(39 z^z9=q_UntZYXiq0YBMSn@eJNP`8H6n&4V}i_XGdGK;BTxiC0Fk=hNWKNb?_)hEt8K zD)+o8RVjs^FSsI(^u3CJN11ol(Uj0y61o+*Oydf_*`4T}AGQevA%kYiofHfRS zj4wC?k_5pg0$*e!n)4LN__pOSS3QHR3|^DL9g~)2K68$+(AfqTs{Q6$+ZGdoJh-2O zsZR+N$%8q4tGe`&m8&_PTiMj1tb3$U4*T!!yZ8J`^yQA!>(%-`br^_oPTD$pgOwcm zp_)d;E?B+@wLLqe*4eV*fCTxdP46yVU`?3wI z>`k|1(-iy351j_)^VhF84jVM1D;YyBArY9SqaH4pO^HsvYBB$foFR%z#AoA#X78nF z*_ynGd2rp!ne{G^V6z-B&#(iZGR`U68sK-5wDQQWSy`AheHhwv(d8n-VHIXdG4&rt zz^McE(bTmpA~~AqsrcF-2jk+E+6d7BFX~r)=kIip9Cmvod^aG}kHr>RA9(^-eBKQ& zU^Ptw)0j+^@w#@rbs9?UG==tpZZC-wH8r|!ynJ=al##g4w6{7?4HE6)1Nc>)^HPWq zhYusCtmVC)81v633-}Eg{Jg|Ix1cz(F~i&vn=gxt2y=YXmg+0WyDKSs6=)i3>hd)H zfYh|8ICy?}9%3(Q<8`z0WIbnR4N@V@;cl5BtFOPE?@a@)LvNalWP>*sF?XO25#S5G z2xe>q*R95-jMNO!@73}GVJF(a9r?e6(ZAKwf7Xcql2B(Vd;Qi+3;*9>qpE*}9sPr1 zEBz|)d)HwL|9w|j)HeUMLnIU6-zXLz)4&%8eOIx&-ptvc@u~v21L9c)pzTbhua84i zRFLuphana0Kkx06vx!Q+3Kwx@WINIsXciaE0{>58+BQJZ16)i&{3xZyrPMSpkZcf*Ja!lNtI*VC|81uX50059-4U>`QkQ#7xm@RQX2iM~q_Do5#xlk;!4Q&#?^ z91sr$29GIkIzU0C1(JzJ>E>&%X^sDh=w)L;IjSyRI_)?>qdVLW_k zzgP#GlJkG*BN_^J*5~lwxg3vP79#)WR+-6Ery0fjx2*B{98i&iR~;zK%E%4|Uf{_a(A$bdEMU zM-dw*gSyCwi_N3!fv26Hkunk30t*d*KIEUP^`m$+WgodMQ8kQY%o6)~h~^n47nwoZ zwZa;gE&Y3|#D=`%JoMVbZOGHWbeZ~Y-Kucxp|Y=38B9mzx7O*$?l^=RFlXqJ&V zCnV?RnrqXa(UL3XEPVX?>zXy&ZC?fHcE@CQ!@;qI6fZuv(qUT~Yex2sE!ih7E_Z(U z&4TOSHM}zRx<`N#nGnPFzvd8p3@ni1wH zL4-dC{cQOhBAl`aqTPBuNt>U%4ldJ;*X!m6ANerh{OOM9`)iheayM02JaVGQ0&Q+7 zO&V=0xAj>%QW~B|XB8HlofqRyMfOAHA#A`jD3`{ z9m~vT%x|$bx&5I2yFV&V@#ebQKFKUx-ADc2(VtmBzVd7Hsr|2?{Hc6oqE}LH{glb= z$m!i3W*%I(h27s#ne+75w7&H2)1^iG?Y+PdT~zn+q{6hot(a9-P~A4}-o_1D5Vj0C zz%YLBEOf`-tv|nsD7~jxEI)fuv9a6vCs~QX#@R^@pKMC+J~mC6<3uNQ-&nlRBmW)v zc7?uTOlFB%AsfIk3}5Uxp)3<*HL?>mSRhMIQlK^5Ef6tYjgzqcrMNmcQpQf=K-X1_ z#No>9{U9VJ2nAtQL|po2h~On+Z%yi;4t?m*N?E-@n#R?!ijDwz!+WUtj+`Dwu3*ke zukaCBVtXNi4&Wz)Z2ALMM3Rp6-B`qa^C5W}bKO*vRSwx#Uv`i8iZ(yUh%LJerIUfc zj2(>a?8>|Ct1Eu`>t^Ln(~73P1AD4XJ}Ms+ch$-_=D+oJeDWaoNo756qW@lI%l^Eu z88gX7HZ-yHArTj=@!ng|=|ROU<#nV;AL^8e)NhHdEp*64;^orgl8{e(2$0JGG$FKx zcSb?pJ8BR8_Mp(OFlg0S?XjB&X%#L)_-rzi8##?k4NULW%{NAm&lLWZ_YX7 zx6YWMoKn$LE5jd^8};2IXu<1c?U%&$pBK2`+1-gyjaEi<88cwf3N^u|L#3!o4KATdrIR(eHW32aYgswH-Z2DN(DcL5ZW7?sUz^L8ES1B_{+QRtLEO}!$L>(2((LOW zJQTL;MnMw|G-~rNxIs^^LddF8A_&E7C&~ZBrG6+taXRoUPuPO5kx1jU50@OYyj%IVIz>qcu-nC_glcudYDME zM4ktDb05S3;s9Lz4Pys`+T7cz#G4J0Q3<!ZqEjBJ%E%)Ex`lR*_<5Jz>K$zbbX(x z*oD}%Yhc>B*t7rv+&yNnWLFJ-@*h10^3Va;&8iw7h3R{EZ4g zPQ%JL=Y?`3&6v8a`wq@)zP$O?d^4_3Tie3kVCyqIvFnp<_wH&*ooM*q_cv!K{KT34 ziS6rmCJcPoDJ8uLdQ{G(#_<0}L{*FM6V@|~pA==2(fUCgch{GpO4!YWZTk}xw(*EL zp2_l*Q-I5nC}}bpy_>!18xLME1p*3I5l{h8_A-pJf%9WV-yMAzFV@G1L`_m(_8v%T z?7mgrOd~T5F>>u3P1!)e!y`k4z3S-IcL?Ap)gyC&y;T6K$&W2x84UC|<85y`)6wP* zkPk>3ARjzpQba!>oY4^Q@z97xYko(PR?tf|!+1L*-kOR8Ae~*k4j_OLwrYTJ?QO^D z^x~sIFB5?6)&VDe`3){!tGamTcE&ZS5ko)v!U%x?$UEi^D4t6BA|!^#f?8MCb*_rr zw*Sn)u&7ZflsvVdQXoNvsg9BzmR^G8|GoK=W72YUv<~62)uL3bKKkzFN_buU7Z$)H z{YQ&)*gJ0|C*Q~tR5~Pjy-?r^7`_Plu6hth#(rRc09~NgHp2r@u8nDUxE`<=pAT9s z`;aeywot9r1PG$Y9yj34LZsD-)544<*I$?AbU!k}RFsL_J(%5tXIHO5D(5DF-97^$ zoKd!iw(98LhwG8D%xiE|ILM(kgi7^LT|$#UG}PB<4C3~xluYVMt^8p;g;qW^+W}dJJ;>CtVuo+vG2f- z(?Z-9UE5pkAMx6?rDA2sKc%O~gC!%{BZA&h{4;L(JU>VidYO20;*PnqP(n%!{$1LSqXZ-$jcG1cz6NN{+{7{aeVqVW#k>U`Iln^Fs zTJ$XLpH)3eQj)iwFbVBT5x*mJyz?P#0abd5Jk}&Zs&s@=z~sNV@+mm{PL;}B7`6j| z6V=Anu>C*~MgAF;89u_+Q2zvmjN0P*!ZtthlN_~0*w+4=y)gm;g221_@BjC-b>^?h zU2}eUhs-jyAq1ueGRuH-FwTNaEw3+ zYK6Kn#T5n_f%QoDkKr`L6Jv%-!i2EX0f89D=Z}F?_%GN0$Uj-EM#&hi1s2K%0nt2U zHuqhnxHpHuDnXv>`{kN^$_vIrP-VzMM-(_bVA1jl5!}`%JwEc#BtUF)4H(-cV0^=# zjJfiphF6<=^QQmazb*r-=l_;xaA<4Z)Dl<^VePSE5-lzlNlwME7C0NO^(Ua7Jq5y^O7f#Umh4d_5CgZUL8jjlSIH zWFPlPPucoI*7B@K(npiq4I2#lUxynhOW%^|HxSiU@4T#BsMyxZie^^n{W{ymKzw=M zk;5_X7R|XkcmC#5{ofW{ey6j0%|Ah<&)@65sxna6!Rz*EkkxxA$1Y`x3C`w~y#eB=La@58j zW}&Dr6?#p;+5C!sf%rA*|F9cB32ODXF1#i5rqtJrC3^;L4`|t9C(+^!I>vP*|s*TQK*Eo1&UnT(^UngH%quwB2JvH zavyd2HOyv%gUxE0p?tj-K0ca@+0LndEyb9z&H@DBQa91!SOV>5yB9 z*Lj+$9M6ZEe(np*;}R*}tu)vF>}**~WQiw1I2#WAe(v)i_5>-gaKsiw!gqDSyzg1W z!}+sN8BPcgA>_aZRoF|>&(ntJPHjjl2PT*lX*NgVI?)yKQen-)HrFmo{1b_54fv11 zrHuRy>W# zmj0YR-Z*l4`3IPLIzCJ!>_s`m6Fv&tR&Ob=52u`$)LF!COU)M3r5YDmQJjtN3I&w_ zsmB83Y_)`v-MX6^vl5!JJSw$j?!g06t4qzSq%2p>*l9_**)*>176zz{xx2y zd-_RDmzpovpM||kxy@zyAP3^m%K03$=fsVh!2c49oDl#CQ|;cJ#ZULupg~&_&4vPS zw!@-1Kn4_==O%C1|1L*u$lAqEid3Z;L)}r-&JiL@nAaDISIuTcNy{hD$}%{vRuWos zODe~e6iTefZ}W^Y0gw1SZLKTU$tq7a)_?@LlEK*tu5BWr4jS$R*ge$TcFI3rw0nA( zC4PHTiniSuF82>`gB#}B(2b~?w>!?UD7jo69q_+Fwnd#?tR_^Gl=?dAL)a?PHs3fF zbCi2Zz-NadU+U;U7IYv_0YA3Cy92nUhjO8-9zZ5Fi-qgW3*(Iw-4dZfz$i}wb5VhB zl`xCWrFuSeoXr2}21jSfl}^fyEE)$FBnI?@KeZ(_)(|)V*vitL8<-1Z#G}hHJ6CUs zl48+;p~L~|3ZR5EFYrSGEv*hnO^uzBhA{Qe8u1HPcheFfPr6gMIMjyY*^yRED4z&X z%FJEl2%*;Q7;zU6H8}ktesvacMFs5IuBw2-b)_mfvg6O~_q0vlxSOFc zDz+0Aryywp6^Xahng*eQxId{-hh{Tf3M;12Ag>%^FU1<(_#;3Y(6`3Y8lsPWAy?=e z>h9ddB0h&P3%t$%aHL%(d2)3=!Zbe>NJ`PktbO6#N_}pI(oq;1c2Wt7%z_QF#qd-E zP~XfwLn-PcD$xexV+eDy7gB9JV71Wk52`Cq5P~leqPyUy(L_#GS4*>Ra^GQ2*^U(0 zCQDCn$llK4+CVT9vU0rYaQOJsUb7E_WH`@_Bm<@)+L@I zFbEGW6?Pqn0td2}xB2A-X0K5689DSLFc;=|?rx~Hvk4*~ASYCBCJjwclbgZMa@De9 z5fNpG#lO;RA+aB@3p)!j6c7)hOu(UCW|%`7Io6Xd`7HI-@DQ0<;>_@QMdqmX zFQuZ$OzX58pp+X(*4UC@{Qx)x==Nx64&Zp0U37mX)4Et`dq8X z*f~8W-N=2vxEkjJAV#0TV|o9>C9z-7edo0k-hiy&z;cD)xJ1rN9uv4o4+dhf zj}Mhl5P~Ic!B5&C3v=U-i|~GHkPDcG%L8euDd_}`Z3pkIFCDJY1vJgj9`5#CqRpUID%(g)YmaJi-6ekO|jmam5Ezq}Ix^OA~4aEI0_KIh00 z01>THdl|}dKw3|4N(0O>ykOPB(~Mx?%!(p_2m=TKrT!3B!P9VfBLZ)nu@_8bgyNSM zm!VBJYE1X@)u6(3(_n0zpcL Date: Mon, 18 Nov 2024 16:43:17 -0800 Subject: [PATCH 06/10] integrate online and remote changes --- scenarios/evaluate/README.md | 2 +- .../Supported_Evaluation_Metrics/README.md | 2 +- .../Evaluate_Online/Evaluate_Online.ipynb | 248 ++++++++++++++++++ .../Evaluate_Online/README.md | 41 +++ .../Evaluate_Remotely/Evaluate_Remotely.ipynb | 222 ++++++++++++++++ .../Evaluate_Remotely/README.md | 41 +++ .../evaluate_test_data.jsonl | 3 + 7 files changed, 557 insertions(+), 2 deletions(-) create mode 100644 scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Online/Evaluate_Online.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Online/README.md create mode 100644 scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/Evaluate_Remotely.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/README.md create mode 100644 scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/evaluate_test_data.jsonl diff --git a/scenarios/evaluate/README.md b/scenarios/evaluate/README.md index 257ea7bb..d18e535e 100644 --- a/scenarios/evaluate/README.md +++ b/scenarios/evaluate/README.md @@ -29,7 +29,7 @@ In both cases, running evaluations requires specific tools, methods, and dataset - Pre-Built Evaluators: Azure AI provides a range of [generation safety](\Supported_Evaluation_Metrics\AI_Judge_Evaluators_Safety_Risks\) and [quality/NLP evaluators](\Supported_Evaluation_Metrics\AI_Judge_Evaluators_Quality\) ready for immediate use. - [Custom Evaluators](\Supported_Evaluation_Metrics\Custom_Evaluators\): Using the Azure AI Evaluation SDK, you can design and implement evaluators that align with the unique requirements of your application. -* Generating and Visualizing Evaluation Results: Azure AI Evaluation SDK enables you to evaluate the target functions (such as [endpoints of your AI application](\Supported_Evaluation_Targets\Evaluate_App_Endpoint\) or your [model endpoints](\Supported_Evaluation_Targets\Evaluate_Base_Model_Endpoint\) on your dataset with either built-in or custom evaluators. You can run evaluations remotely in the cloud or locally on your own machine. +* Generating and Visualizing Evaluation Results: Azure AI Evaluation SDK enables you to evaluate the target functions (such as [endpoints of your AI application](\Supported_Evaluation_Targets\Evaluate_App_Endpoint\) or your [model endpoints](\Supported_Evaluation_Targets\Evaluate_Base_Model_Endpoint\) on your dataset with either built-in or custom evaluators. You can run evaluations [remotely](\Supported_Evaluation_Targets\Evaluate_Remotely\) in the cloud or locally on your own machine. ### Objective diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/README.md b/scenarios/evaluate/Supported_Evaluation_Metrics/README.md index 96ca436d..9347a081 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/README.md +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/README.md @@ -26,4 +26,4 @@ Currently, Azure AI Evaluation SDK supports three types of evaluators: -You can run evaluators locally or remotely, log results in the cloud using the evaluation SDK, or integrate them into automated evaluations within the Azure AI Studio UI. +You can run evaluators locally or [remotely](..\Supported_Evaluation_Targets\Evaluate_Remotely\Evaluate_Remotely.ipynb), log results in the cloud using the evaluation SDK, or integrate them into automated evaluations within the Azure AI Studio UI. diff --git a/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Online/Evaluate_Online.ipynb b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Online/Evaluate_Online.ipynb new file mode 100644 index 00000000..995f64f1 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Online/Evaluate_Online.ipynb @@ -0,0 +1,248 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Online Evaluations: Evaluating in the Cloud on a Schedule\n", + "\n", + "## Objective\n", + "\n", + "This tutorial provides a step-by-step guide on how to evaluate data generated by LLMs online on a schedule. \n", + "\n", + "This tutorial uses the following Azure AI services:\n", + "\n", + "- [Azure AI Safety Evaluation](https://aka.ms/azureaistudiosafetyeval)\n", + "- [azure-ai-evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk)\n", + "\n", + "## Time\n", + "\n", + "You should expect to spend 30 minutes running this sample. \n", + "\n", + "## About this example\n", + "\n", + "This example demonstrates the online evaluation of a LLM. It is important to have access to AzureOpenAI credentials and an AzureAI project. This example demonstrates: \n", + "\n", + "- Recurring, Online Evaluation (to be used to monitor LLMs once they are deployed)\n", + "\n", + "## Before you begin\n", + "### Prerequesite\n", + "- Configure resources to support Online Evaluation as per [Online Evaluation documentation](https://aka.ms/GenAIMonitoringDoc)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -U azure-identity\n", + "%pip install -U azure-ai-project\n", + "%pip install -U azure-ai-evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.project import AIProjectClient\n", + "from azure.identity import DefaultAzureCredential\n", + "from azure.ai.project.models import (\n", + " ApplicationInsightsConfiguration,\n", + " EvaluatorConfiguration,\n", + " ConnectionType,\n", + " EvaluationSchedule,\n", + " RecurrenceTrigger,\n", + ")\n", + "from azure.ai.evaluation import F1ScoreEvaluator, ViolenceEvaluator" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Connect to your Azure Open AI deployment\n", + "To evaluate your LLM-generated data remotely in the cloud, we must connect to your Azure Open AI deployment. This deployment must be a GPT model which supports `chat completion`, such as `gpt-4`. To see the proper value for `conn_str`, navigate to the connection string at the \"Project Overview\" page for your Azure AI project. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "project_client = AIProjectClient.from_connection_string(\n", + " credential=DefaultAzureCredential(),\n", + " conn_str=\"\", # At the moment, it should be in the format \".api.azureml.ms;;;\" Ex: eastus2.api.azureml.ms;xxxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxxxxx;rg-sample;sample-project-eastus2\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Please see [Online Evaluation documentation](https://aka.ms/GenAIMonitoringDoc) for configuration of Application Insights. `service_name` is a unique name you provide to define your Generative AI application and identify it within your Application Insights resource. This property will be logged in the `traces` table in Application Insights and can be found in the `customDimensions[\"service.name\"]` field. `evaluation_name` is a unique name you provide for your Online Evaluation schedule. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Your Application Insights resource ID\n", + "# At the moment, it should be something in the format \"/subscriptions//resourceGroups//providers/Microsoft.Insights/components/\"\"\n", + "app_insights_resource_id = \"\"\n", + "\n", + "# Name of your generative AI application (will be available in trace data in Application Insights)\n", + "service_name = \"\"\n", + "\n", + "# Name of your online evaluation schedule\n", + "evaluation_name = \"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Below is the Kusto Query Language (KQL) query to query data from Application Insights resource. This query is compatible with data logged by the Azure AI Inferencing Tracing SDK (linked in [documentation](https://aka.ms/GenAIMonitoringDoc)). You can modify it depending on your data schema. The KQL query must output several columns: `operation_ID`, `operation_ParentID`, and `gen_ai_response_id`. You can choose which other columns to output as required by the evaluators you are using." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kusto_query = 'let gen_ai_spans=(dependencies | where isnotnull(customDimensions[\"gen_ai.system\"]) | extend response_id = tostring(customDimensions[\"gen_ai.response.id\"]) | project id, operation_Id, operation_ParentId, timestamp, response_id); let gen_ai_events=(traces | where message in (\"gen_ai.choice\", \"gen_ai.user.message\", \"gen_ai.system.message\") or tostring(customDimensions[\"event.name\"]) in (\"gen_ai.choice\", \"gen_ai.user.message\", \"gen_ai.system.message\") | project id= operation_ParentId, operation_Id, operation_ParentId, user_input = iff(message == \"gen_ai.user.message\" or tostring(customDimensions[\"event.name\"]) == \"gen_ai.user.message\", parse_json(iff(message == \"gen_ai.user.message\", tostring(customDimensions[\"gen_ai.event.content\"]), message)).content, \"\"), system = iff(message == \"gen_ai.system.message\" or tostring(customDimensions[\"event.name\"]) == \"gen_ai.system.message\", parse_json(iff(message == \"gen_ai.system.message\", tostring(customDimensions[\"gen_ai.event.content\"]), message)).content, \"\"), llm_response = iff(message == \"gen_ai.choice\", parse_json(tostring(parse_json(tostring(customDimensions[\"gen_ai.event.content\"])).message)).content, iff(tostring(customDimensions[\"event.name\"]) == \"gen_ai.choice\", parse_json(parse_json(message).message).content, \"\")) | summarize operation_ParentId = any(operation_ParentId), Input = maxif(user_input, user_input != \"\"), System = maxif(system, system != \"\"), Output = maxif(llm_response, llm_response != \"\") by operation_Id, id); gen_ai_spans | join kind=inner (gen_ai_events) on id, operation_Id | project Input, System, Output, operation_Id, operation_ParentId, gen_ai_response_id = response_id'\n", + "\n", + "# AzureMSIClientId is the clientID of the User-assigned managed identity created during set-up - see documentation for how to find it\n", + "properties = {\"AzureMSIClientId\": \"your_client_id\"}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Connect to your Application Insights resource\n", + "app_insights_config = ApplicationInsightsConfiguration(\n", + " resource_id=app_insights_resource_id, query=kusto_query, service_name=service_name\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Connect to your AOAI resource, you must use an AOAI GPT model\n", + "deployment_name = \"gpt-4\"\n", + "api_version = \"2024-06-01\"\n", + "default_connection = project_client.connections.get_default(connection_type=ConnectionType.AZURE_OPEN_AI)\n", + "model_config = default_connection.to_evaluator_model_config(deployment_name=deployment_name, api_version=api_version)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Configure Evaluators to Run\n", + "The code below demonstrates how to configure the evaluators you want to run. In this example, we use the `F1ScoreEvaluator`, `RelevanceEvaluator` and the `ViolenceEvaluator`, but all evaluators supported by [Azure AI Evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/concepts/evaluation-metrics-built-in?tabs=warning) are supported by Online Evaluation and can be configured here. You can either import the classes from the SDK and reference them with the `.id` property, or you can find the fully formed `id` of the evaluator in the AI Studio registry of evaluators, and use it here. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# id for each evaluator can be found in your AI Studio registry - please see documentation for more information\n", + "# init_params is the configuration for the model to use to perform the evaluation\n", + "# data_mapping is used to map the output columns of your query to the names required by the evaluator\n", + "evaluators = {\n", + " \"f1_score\": EvaluatorConfiguration(\n", + " id=F1ScoreEvaluator.id,\n", + " ),\n", + " \"relevance\": EvaluatorConfiguration(\n", + " id=\"azureml://registries/azureml-staging/models/Relevance-Evaluator/versions/4\",\n", + " init_params={\"model_config\": model_config},\n", + " data_mapping={\"query\": \"${data.Input}\", \"response\": \"${data.Output}\"},\n", + " ),\n", + " \"violence\": EvaluatorConfiguration(\n", + " id=ViolenceEvaluator.id,\n", + " init_params={\"azure_ai_project\": project_client.scope},\n", + " data_mapping={\"query\": \"${data.Input}\", \"response\": \"${data.Output}\"},\n", + " ),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Evaluate in the Cloud on a Schedule with Online Evaluation\n", + "\n", + "You can configure the `RecurrenceTrigger` based on the class definition [here](https://learn.microsoft.com/en-us/python/api/azure-ai-ml/azure.ai.ml.entities.recurrencetrigger?view=azure-python)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Frequency to run the schedule\n", + "recurrence_trigger = RecurrenceTrigger(frequency=\"day\", interval=1)\n", + "\n", + "# Configure the online evaluation schedule\n", + "evaluation_schedule = EvaluationSchedule(\n", + " data=app_insights_config,\n", + " evaluators=evaluators,\n", + " trigger=recurrence_trigger,\n", + " description=f\"{service_name} evaluation schedule\",\n", + " properties=properties,\n", + ")\n", + "\n", + "# Create the online evaluation schedule\n", + "created_evaluation_schedule = project_client.evaluations.create_or_replace_schedule(service_name, evaluation_schedule)\n", + "print(\n", + " f\"Successfully submitted the online evaluation schedule creation request - {created_evaluation_schedule.name}, currently in {created_evaluation_schedule.provisioning_state} state.\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Next steps \n", + "\n", + "Navigate to the \"Tracing\" tab in [Azure AI Studio](https://ai.azure.com/) to view your logged trace data alongside the evaluations produced by the Online Evaluation schedule. You can use the reference link provided in the \"Tracing\" tab to navigate to a comprehensive workbook in Application Insights for more details on how your application is performing. " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "azureai-samples313", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Online/README.md b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Online/README.md new file mode 100644 index 00000000..f7af45a1 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Online/README.md @@ -0,0 +1,41 @@ +--- +page_type: sample +languages: +- python +products: +- ai-services +- azure-openai +description: Evaluating online +--- + +## Evaluating in the cloud on a schedule + +### Overview + +This tutorial provides a step-by-step guide on how to evaluate generative AI or LLMs on a scheduling using online evaluation. + +### Objective + +The main objective of this tutorial is to help users understand the process of evaluating model remotely in the cloud by triggering an evaluation. This type of evaluation can be used for monitoring LLMs and Generative AI that has been deployed. By the end of this tutorial, you should be able to: + + - Learn about evaluations + - Evaluate LLM using various evaluators from Azure AI Evaluations SDK online in the cloud. + +### Note +All evaluators supported by [Azure AI Evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/concepts/evaluation-metrics-built-in?tabs=warning) are supported by Online Evaluation. For updated documentation, please see [Online Evaluation documentation](https://aka.ms/GenAIMonitoringDoc). + +#### Region Support for Evaluations + +| Region | Hate and Unfairness, Sexual, Violent, Self-Harm, XPIA | Groundedness Pro | Protected Material | +| - | - | - | - | +| UK South | Will be deprecated 12/1/24 | no | no | +| East US 2 | yes | yes | yes | +| Sweden Central | yes | yes | no | +| US North Central | yes | no | no | +| France Central | yes | no | no | +| Switzerland West | yes | no | no | + +### Programming Languages + - Python + +### Estimated Runtime: 30 mins \ No newline at end of file diff --git a/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/Evaluate_Remotely.ipynb b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/Evaluate_Remotely.ipynb new file mode 100644 index 00000000..2d37962a --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/Evaluate_Remotely.ipynb @@ -0,0 +1,222 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Remote Evaluations: Evaluating in the Cloud \n", + "\n", + "## Objective\n", + "\n", + "This tutorial provides a step-by-step guide on how to evaluate data generated by LLMs remotely in the cloud. \n", + "\n", + "This tutorial uses the following Azure AI services:\n", + "\n", + "- [Azure AI Safety Evaluation](https://aka.ms/azureaistudiosafetyeval)\n", + "- [azure-ai-evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk)\n", + "\n", + "## Time\n", + "\n", + "You should expect to spend 20 minutes running this sample. \n", + "\n", + "## About this example\n", + "\n", + "This example demonstrates the remote evaluation of query and response pairs that were generated by a LLM. It is important to have access to AzureOpenAI credentials and an AzureAI project. **To create data to use in your own evaluation, learn more [here](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/simulator-interaction-data)** . This example demonstrates: \n", + "\n", + "- Single-instance, triggered Remote Evaluation (to be used for pre-deployment evaluation of LLMs)\n", + "\n", + "## Before you begin\n", + "### Prerequesite\n", + "- [Have an online deployment on Azure Open AI studio supporting `chat completion` such as `gpt-4`](https://learn.microsoft.com/en-us/azure/machine-learning/concept-endpoints-online?view=azureml-api-2)\n", + "- You also might want to evaluate data generated by your LLM, to see how to generate data to be evaluated using the Azure AI Evaluation SDK, see our samples on simulation \n", + "\n", + "### Installation\n", + "\n", + "Install the following packages required to execute this notebook. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -U azure-identity\n", + "%pip install -U azure-ai-project\n", + "%pip install -U azure-ai-evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.project import AIProjectClient\n", + "from azure.identity import DefaultAzureCredential\n", + "from azure.ai.project.models import (\n", + " Evaluation,\n", + " Dataset,\n", + " EvaluatorConfiguration,\n", + " ConnectionType,\n", + ")\n", + "from azure.ai.evaluation import F1ScoreEvaluator, ViolenceEvaluator" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Connect to your Azure Open AI deployment\n", + "To evaluate your LLM-generated data remotely in the cloud, we must connect to your Azure Open AI deployment. This deployment must be a GPT model which supports `chat completion`, such as `gpt-4`. To see the proper value for `conn_str`, navigate to the connection string at the \"Project Overview\" page for your Azure AI project. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "project_client = AIProjectClient.from_connection_string(\n", + " credential=DefaultAzureCredential(),\n", + " conn_str=\"\", # At the moment, it should be in the format \".api.azureml.ms;;;\" Ex: eastus2.api.azureml.ms;xxxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxxxxx;rg-sample;sample-project-eastus2\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Connect to your AOAI resource, you must use an AOAI GPT model\n", + "deployment_name = \"gpt-4\"\n", + "api_version = \"2024-06-01\"\n", + "default_connection = project_client.connections.get_default(connection_type=ConnectionType.AZURE_OPEN_AI)\n", + "model_config = default_connection.to_evaluator_model_config(deployment_name=deployment_name, api_version=api_version)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data\n", + "The following code demonstrates how to upload the data for evaluation to your Azure AI project. Below we use `evaluate_test_data.jsonl` which exemplifies LLM-generated data in the query-response format expected by the Azure AI Evaluation SDK. For your use case, you should upload data in the same format, which can be generated using the [`Simulator`](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/simulator-interaction-data) from Azure AI Evaluation SDK. \n", + "\n", + "Alternatively, if you already have an existing dataset for evaluation, you can use that by finding the link to your dataset in your [registry](https://ml.azure.com/registries) or find the dataset ID." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# # Upload data for evaluation\n", + "data_id, _ = project_client.upload_file(\"./evaluate_test_data.jsonl\")\n", + "# data_id = \"azureml://registries//data//versions/\"\n", + "# To use an existing dataset, replace the above line with the following line\n", + "# data_id = \"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Configure Evaluators to Run\n", + "The code below demonstrates how to configure the evaluators you want to run. In this example, we use the `F1ScoreEvaluator`, `RelevanceEvaluator` and the `ViolenceEvaluator`, but all evaluators supported by [Azure AI Evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/concepts/evaluation-metrics-built-in?tabs=warning) are supported by Online Evaluation and can be configured here. You can either import the classes from the SDK and reference them with the `.id` property, or you can find the fully formed `id` of the evaluator in the AI Studio registry of evaluators, and use it here. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# id for each evaluator can be found in your AI Studio registry - please see documentation for more information\n", + "# init_params is the configuration for the model to use to perform the evaluation\n", + "# data_mapping is used to map the output columns of your query to the names required by the evaluator\n", + "evaluators = {\n", + " \"f1_score\": EvaluatorConfiguration(\n", + " id=F1ScoreEvaluator.id,\n", + " ),\n", + " \"relevance\": EvaluatorConfiguration(\n", + " id=\"azureml://registries/azureml-staging/models/Relevance-Evaluator/versions/4\",\n", + " init_params={\"model_config\": model_config},\n", + " data_mapping={\"query\": \"${data.Input}\", \"response\": \"${data.Output}\"},\n", + " ),\n", + " \"violence\": EvaluatorConfiguration(\n", + " id=ViolenceEvaluator.id,\n", + " init_params={\"azure_ai_project\": project_client.scope},\n", + " data_mapping={\"query\": \"${data.Input}\", \"response\": \"${data.Output}\"},\n", + " ),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Evaluate in the Cloud with Remote Evaluation\n", + "Below we demonstrate how to trigger a single-instance Remote Evaluation in the Cloud. This can be used for pre-deployment testing of an LLM. \n", + " \n", + "Here we pass in the `data_id` we would like to use for the evaluation. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "evaluation = Evaluation(\n", + " display_name=\"Remote Evaluation\",\n", + " description=\"Evaluation of dataset\",\n", + " data=Dataset(id=data_id),\n", + " evaluators=evaluators,\n", + ")\n", + "\n", + "# Create evaluation\n", + "evaluation_response = project_client.evaluations.create(\n", + " evaluation=evaluation,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Get evaluation\n", + "get_evaluation_response = project_client.evaluations.get(evaluation_response.id)\n", + "\n", + "print(\"----------------------------------------------------------------\")\n", + "print(\"Created evaluation, evaluation ID: \", get_evaluation_response.id)\n", + "print(\"Evaluation status: \", get_evaluation_response.status)\n", + "print(\"AI Foundry Portal URI: \", get_evaluation_response.properties[\"AiStudioEvaluationUri\"])\n", + "print(\"----------------------------------------------------------------\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "azureai-samples313", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/README.md b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/README.md new file mode 100644 index 00000000..190f49cc --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/README.md @@ -0,0 +1,41 @@ +--- +page_type: sample +languages: +- python +products: +- ai-services +- azure-openai +description: Evaluating remotely +--- + +## Evaluating in the cloud + +### Overview + +This tutorial provides a step-by-step guide on how to evaluate generative AI or LLMs remotely using a triggered evaluation. + +### Objective + +The main objective of this tutorial is to help users understand the process of evaluating model remotely in the cloud by triggering an evaluation. This type of evaluation can be used for pre-deployment testing. By the end of this tutorial, you should be able to: + + - Learn about evaluations + - Evaluate LLM using various evaluators from Azure AI Evaluations SDK remotely in the cloud. + +### Note +Remote evaluations do not support `Retrieval-Evaluator`, `ContentSafetyEvaluator`, and `QAEvaluator`. + +#### Region Support for Evaluations + +| Region | Hate and Unfairness, Sexual, Violent, Self-Harm, XPIA | Groundedness Pro | Protected Material | +| - | - | - | - | +| UK South | Will be deprecated 12/1/24 | no | no | +| East US 2 | yes | yes | yes | +| Sweden Central | yes | yes | no | +| US North Central | yes | no | no | +| France Central | yes | no | no | +| Switzerland West | yes | no | no | + +### Programming Languages + - Python + +### Estimated Runtime: 20 mins \ No newline at end of file diff --git a/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/evaluate_test_data.jsonl b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/evaluate_test_data.jsonl new file mode 100644 index 00000000..ddca8949 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/evaluate_test_data.jsonl @@ -0,0 +1,3 @@ +{"query":"How do you create a run?","context":"AML API only","response":"To create a run using the Azure Machine Learning API, you first need to create an Experiment. Once you have an experiment, you can create a Run object that is associated with that experiment. Here is some Python code that demonstrates this process:\n\n```\nfrom azureml.core import Experiment, Run\nfrom azureml.core.workspace import Workspace\n\n# Define workspace and experiment\nws = Workspace.from_config()\nexp = Experiment(workspace=ws, name='my_experiment')\n\n# Create a new run\nrun = exp.start_logging()\n```\n\nIn this code, the `from_config()` method reads the configuration file that you created when you set up your Azure Machine Learning workspace. The `Experiment` constructor creates an Experiment object that is associated with your workspace, and the `start_logging()` method creates a new Run object that is associated with the Experiment. Now you can use the `run` object to log metrics, upload files, and track other information related to your machine learning experiment.", "ground_truth":"Paris is the capital of France."} +{"query":"How do you log a model?","context":"Logging can be done using any OSS Sdk","response":"There are a few ways to log models in Azure Machine Learning. \n\nOne way is to use the `register_model()` method of the `Run` object. The `register_model()` method logs a model file in the Azure Machine Learning service workspace and makes it available for deployment. Here's an example:\n\n```python\nfrom azureml.core import Model\n\nmodel_path = '.\/outputs\/my_model.pkl'\nmodel = Model.register(workspace=ws, model_path=model_path, model_name='my_model')\n```\n\nThis code registers the model file located at `model_path` to the Azure Machine Learning service workspace with the name `my_model`. \n\nAnother way to log a model is to save it as an output of a `Run`. If your model generation code is part of a script or Jupyter notebook that runs as an Azure Machine Learning experiment, you can save the model file as an output of the `Run` object. Here's an example:\n\n```python\nfrom sklearn.linear_model import LogisticRegression\nfrom azureml.core.run import Run\n\n# Initialize a run object\nrun = Run.get_context()\n\n# Train your model\nX_train, y_train = ...\nlog_reg = LogisticRegression().fit(X_train, y_train)\n\n# Save the model to the Run object's outputs directory\nmodel_path = 'outputs\/model.pkl'\njoblib.dump(value=log_reg, filename=model_path)\n\n# Log the model as a run artifact\nrun.upload_file(name=model_path, path_or_stream=model_path)\n```\n\nIn this code, `Run.get_context()` retrieves the current run context object, which you can use to track metadata and metrics for the run. After training your model, you can use `joblib.dump()` to save the model to a file, and then log the file as an artifact of the run using `run.upload_file()`.","ground_truth":"Paris is the capital of France."} +{"query":"What is the capital of France?","context":"France is in Europe","response":"Paris is the capital of France.", "ground_truth":"Paris is the capital of France."} From 9743f19d5c55a43c61cb978e9d0ab00d789d1da0 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Mon, 18 Nov 2024 16:57:23 -0800 Subject: [PATCH 07/10] update hyperlinks --- scenarios/evaluate/README.md | 28 +++++++++---------- scenarios/evaluate/Simulators/README.md | 2 +- .../Supported_Evaluation_Metrics/README.md | 4 +-- .../Supported_Evaluation_Targets/README.md | 4 +-- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/scenarios/evaluate/README.md b/scenarios/evaluate/README.md index d18e535e..3c54883d 100644 --- a/scenarios/evaluate/README.md +++ b/scenarios/evaluate/README.md @@ -22,14 +22,14 @@ In both cases, running evaluations requires specific tools, methods, and dataset - Bring Your Own Data: Use datasets tailored to your application or domain. - Redteaming Queries: Design adversarial prompts to test robustness. - - ![Azure AI Simulators](./Simulators/): Leverage Azure AI's context-specific or adversarial dataset generators to create relevant test cases. + - [Azure AI Simulators](Simulators/): Leverage Azure AI's context-specific or adversarial dataset generators to create relevant test cases. * Selecting the Appropriate Evaluators or Building Custom Ones - - Pre-Built Evaluators: Azure AI provides a range of [generation safety](\Supported_Evaluation_Metrics\AI_Judge_Evaluators_Safety_Risks\) and [quality/NLP evaluators](\Supported_Evaluation_Metrics\AI_Judge_Evaluators_Quality\) ready for immediate use. - - [Custom Evaluators](\Supported_Evaluation_Metrics\Custom_Evaluators\): Using the Azure AI Evaluation SDK, you can design and implement evaluators that align with the unique requirements of your application. + - Pre-Built Evaluators: Azure AI provides a range of [generation safety](Supported_Evaluation_Metrics\AI_Judge_Evaluators_Safety_Risks\) and [quality/NLP evaluators](Supported_Evaluation_Metrics\AI_Judge_Evaluators_Quality\) ready for immediate use. + - [Custom Evaluators](Supported_Evaluation_Metrics\Custom_Evaluators\): Using the Azure AI Evaluation SDK, you can design and implement evaluators that align with the unique requirements of your application. -* Generating and Visualizing Evaluation Results: Azure AI Evaluation SDK enables you to evaluate the target functions (such as [endpoints of your AI application](\Supported_Evaluation_Targets\Evaluate_App_Endpoint\) or your [model endpoints](\Supported_Evaluation_Targets\Evaluate_Base_Model_Endpoint\) on your dataset with either built-in or custom evaluators. You can run evaluations [remotely](\Supported_Evaluation_Targets\Evaluate_Remotely\) in the cloud or locally on your own machine. +* Generating and Visualizing Evaluation Results: Azure AI Evaluation SDK enables you to evaluate the target functions (such as [endpoints of your AI application](Supported_Evaluation_Targets\Evaluate_App_Endpoint\) or your [model endpoints](Supported_Evaluation_Targets\Evaluate_Base_Model_Endpoint\) on your dataset with either built-in or custom evaluators. You can run evaluations [remotely](Supported_Evaluation_Targets\Evaluate_Remotely\) in the cloud or locally on your own machine. ### Objective @@ -43,17 +43,17 @@ The main objective of this tutorial is to help users understand the process of e | Sample name | adversarial | simulator | conversation starter | index | raw text | against model endpoint | against app | qualitative metrics | custom metrics | quantitative NLP metrics | |----------------------------------------|-------------|-----------|---------------------|-------|----------|-----------------------|-------------|---------------------|----------------|----------------------| -| [Simulate_Adversarial.ipynb](scenarios\evaluate\Simulators\Simulate_Adversarial_Data\simulate_adversarial.ipynb) | X | X | | | | X | | | | | -| [Simulate_From_Conversation_Starter.ipynb](scenarios\evaluate\Simulators\Simulate_Context-Relevant_Data\Simulate_From_Conversation_Starter\Simulate_From_Conversation_Starter.ipynb) | | X | X | | | X | | | | | -| [Simulate_From_Azure_Search_Index.ipynb](scenarios\evaluate\Simulators\Simulate_Context-Relevant_Data\Simulate_From_Azure_Search_Index\Simulate_From_Azure_Search_Index.ipynb) | | X | | X | | X | | | | | -| [Simulate_From_Input_Text.ipynb](scenarios\evaluate\Simulators\Simulate_Context-Relevant_Data\Simulate_From_Input_Text\Simulate_From_Input_Text.ipynb) | | X | | | X | X | | | | | -| [Evaluate_Base_Model_Endpoint.ipynb](scenarios\evaluate\Supported_Evaluation_Targets\Evaluate_Base_Model_Endpoint\Evaluate_Base_Model_Endpoint.ipynb) | | | | | | X | | X | | | +| [Simulate_Adversarial.ipynb](\Simulators\Simulate_Adversarial_Data\simulate_adversarial.ipynb) | X | X | | | | X | | | | | +| [Simulate_From_Conversation_Starter.ipynb](\Simulators\Simulate_Context-Relevant_Data\Simulate_From_Conversation_Starter\Simulate_From_Conversation_Starter.ipynb) | | X | X | | | X | | | | | +| [Simulate_From_Azure_Search_Index.ipynb](\Simulators\Simulate_Context-Relevant_Data\Simulate_From_Azure_Search_Index\Simulate_From_Azure_Search_Index.ipynb) | | X | | X | | X | | | | | +| [Simulate_From_Input_Text.ipynb](\Simulators\Simulate_Context-Relevant_Data\Simulate_From_Input_Text\Simulate_From_Input_Text.ipynb) | | X | | | X | X | | | | | +| [Evaluate_Base_Model_Endpoint.ipynb](\Supported_Evaluation_Targets\Evaluate_Base_Model_Endpoint\Evaluate_Base_Model_Endpoint.ipynb) | | | | | | X | | X | | | | [Evaluate_App_Endpoint.ipynb](scenarios\evaluate\Supported_Evaluation_Targets\Evaluate_App_Endpoint\Evaluate_App_Endpoint.ipynb) | | | | | | | X | X | | | -| [AI_Judge_evaluators_quality.ipynb](scenarios\evaluate\Supported_Evaluation_Metrics\AI_Judge_evaluators_quality\AI_Judge_evaluators_quality.ipynb) | | | | | | X | | X | | | -| [evaluate_custom.ipynb](scenarios\evaluate\Supported_Evaluation_Metrics\custom_evaluators\custom_evaluators.ipynb) | | | | | | X | | | X | | -| [NLP_evaluators.ipynb](scenarios\evaluate\Supported_Evaluation_Metrics\NLP_evaluators\NLP_evaluators.ipynb) | | | | | | X | | | | X | -| [AI_Judge_evaluators_safety_risks.ipynb](scenarios\evaluate\Supported_Evaluation_Metrics\AI_Judge_evaluators_safety_risks\AI_Judge_evaluators_safety_risks.ipynb) | X | | | | | X | | | | | -| [simulate_evaluate_groundedness.py](scenarios\evaluate\Simulators\simulate_evaluate_groundedness\simulate_evaluate_groundedness.ipynb) | | X | | | X | X | | X | | | +| [AI_Judge_Evaluators_Quality.ipynb](\Supported_Evaluation_Metrics\AI_Judge_evaluators_quality\AI_Judge_Evaluators_Quality.ipynb) | | | | | | X | | X | | | +| [Custum_Evaluators.ipynb](\Supported_Evaluation_Metrics\Custom_Evaluators\Custom_Evaluators.ipynb) | | | | | | X | | | X | | +| [NLP_Evaluators.ipynb](\Supported_Evaluation_Metrics\NLP_Evaluators\NLP_Evaluators.ipynb) | | | | | | X | | | | X | +| [AI_Judge_Evaluators_Safety_Risks.ipynb](\Supported_Evaluation_Metrics\AI_Judge_Evaluators_Safety_Risks\AI_Judge_Evaluators_Safety_Risks.ipynb) | X | | | | | X | | | | | +| [Simulate_Evaluate_Groundedness.py](\Simulators\Simulate_Evaluate_Groundedness\Simulate_Evaluate_Groundedness.ipynb) | | X | | | X | X | | X | | | diff --git a/scenarios/evaluate/Simulators/README.md b/scenarios/evaluate/Simulators/README.md index 3eb1273b..e0577dc4 100644 --- a/scenarios/evaluate/Simulators/README.md +++ b/scenarios/evaluate/Simulators/README.md @@ -20,5 +20,5 @@ Relevant, robust evaluation data is essential for effective evaluations. This da - Adversarial simulators, on the other hand, generate queries designed to challenge the AI system, mimicking potential security threats or attempting to provoke undesirable behaviors. This approach helps identify the model's limitations and prepares it to perform well in unexpected or hostile conditions. -Azure AI Studio provides tools for both topic-related and adversarial simulations, enabling comprehensive evaluation and enhancing confidence in deployment. For topic-related simulations, Azure AI enables you to simulate relevant conversations using [your data](\Simulate_Context-Relevant_Data\Simulate_From_Input_Text\Simulate_From_Input_Text.ipynb), [your Azure Search Index](\Simulate_Context-Relevant_Data\Simulate_From_Azure_Search_Index\Simulate_From_Azure_Search_Index.ipynb), or [your pre-defined conversation starters](\Simulate_Context-Relevant_Data\Simulate_From_Conversation_Starter\Simulate_From_Conversation_Starter.ipynb). +Azure AI Studio provides tools for both topic-related and adversarial simulations, enabling comprehensive evaluation and enhancing confidence in deployment. For topic-related simulations, Azure AI enables you to simulate relevant conversations using [your data](Simulate_Context-Relevant_Data/Simulate_From_Input_Text/Simulate_From_Input_Text.ipynb), [your Azure Search Index](Simulate_Context-Relevant_Data\Simulate_From_Azure_Search_Index\Simulate_From_Azure_Search_Index.ipynb), or [your pre-defined conversation starters](Simulate_Context-Relevant_Data\Simulate_From_Conversation_Starter\Simulate_From_Conversation_Starter.ipynb). diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/README.md b/scenarios/evaluate/Supported_Evaluation_Metrics/README.md index 9347a081..d364322c 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/README.md +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/README.md @@ -19,10 +19,10 @@ Currently, Azure AI Evaluation SDK supports three types of evaluators: * [Risk and safety evaluators](\AI_Judge_Evaluators_Safety_Risks\): Evaluating potential risks associated with AI-generated content is essential for safeguarding against content risks with varying degrees of severity. This includes evaluating an AI system's predisposition towards generating harmful or inappropriate content. -* Generation quality evaluators: This involves assessing metrics such as the groundedness, coherence and relevance of generated content using robust [AI-assisted](\AI_Judge_Evaluators_Quality\) and [NLP](scenarios\evaluate\Supported_Evaluation_Metrics\NLP_Evaluators\) metrics. +* Generation quality evaluators: This involves assessing metrics such as the groundedness, coherence and relevance of generated content using robust [AI-assisted](AI_Judge_Evaluators_Quality\) and [NLP](NLP_Evaluators\) metrics. -* [Custom evaluators](\Custom_Evaluators\): Tailored evaluation metrics can be designed to meet specific needs and goals, providing flexibility and precision in assessing unique aspects of AI-generated content. These custom evaluators allow for more detailed and specific analyses, addressing particular concerns or requirements that standard metrics may not cover. +* [Custom evaluators](Custom_Evaluators\): Tailored evaluation metrics can be designed to meet specific needs and goals, providing flexibility and precision in assessing unique aspects of AI-generated content. These custom evaluators allow for more detailed and specific analyses, addressing particular concerns or requirements that standard metrics may not cover. diff --git a/scenarios/evaluate/Supported_Evaluation_Targets/README.md b/scenarios/evaluate/Supported_Evaluation_Targets/README.md index 694da0ea..79382ede 100644 --- a/scenarios/evaluate/Supported_Evaluation_Targets/README.md +++ b/scenarios/evaluate/Supported_Evaluation_Targets/README.md @@ -23,13 +23,13 @@ Evaluation plays a critical role at three pivotal stages: The first stage of the AI lifecycle involves selecting an appropriate base model. With over 1,600 base models available in the Azure AI Studio's model catalog, it's crucial to choose one that best aligns with your specific use case. This involves comparing models based on quality and safety metrics like groundedness, relevance, and safety scores, as well as considering cost, computational efficiency, and latency. -Follow [this tutorial](\Evaluate_Base_Model_Endpoint\Evaluate_Base_Model_Endpoint.ipynb) to evaluate and compare base models on your own data. +Follow [this tutorial](Evaluate_Base_Model_Endpoint\Evaluate_Base_Model_Endpoint.ipynb) to evaluate and compare base models on your own data. ### Pre-Production AI Application Evaluation Once a base model is selected, the next stage is building an AI application around it—such as an AI-powered copilot, a retrieval-augmented generation (RAG) system, an agentic AI system, or any other generative AI-based tool. Before deploying the AI application into a production environment, rigorous testing is essential to ensure the model is truly ready for real-world use. -Follow [this tutorial](\Evaluate_App_Endpoint\Evaluate_App_Endpoint.ipynb) to evaluate any AI application endpoint on your own data. +Follow [this tutorial](Evaluate_App_Endpoint\Evaluate_App_Endpoint.ipynb) to evaluate any AI application endpoint on your own data. ### Post-Production AI Application Evaluation (Monitoring) From 39e7e9e9399cf53c95fdfd86a39813a15e74dbe1 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Mon, 18 Nov 2024 17:06:19 -0800 Subject: [PATCH 08/10] update hyperlinks again --- scenarios/evaluate/README.md | 28 +++++++++---------- scenarios/evaluate/Simulators/README.md | 2 +- .../Supported_Evaluation_Metrics/README.md | 8 +++--- .../Supported_Evaluation_Targets/README.md | 4 +-- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/scenarios/evaluate/README.md b/scenarios/evaluate/README.md index 3c54883d..a372aacd 100644 --- a/scenarios/evaluate/README.md +++ b/scenarios/evaluate/README.md @@ -26,10 +26,10 @@ In both cases, running evaluations requires specific tools, methods, and dataset * Selecting the Appropriate Evaluators or Building Custom Ones - - Pre-Built Evaluators: Azure AI provides a range of [generation safety](Supported_Evaluation_Metrics\AI_Judge_Evaluators_Safety_Risks\) and [quality/NLP evaluators](Supported_Evaluation_Metrics\AI_Judge_Evaluators_Quality\) ready for immediate use. - - [Custom Evaluators](Supported_Evaluation_Metrics\Custom_Evaluators\): Using the Azure AI Evaluation SDK, you can design and implement evaluators that align with the unique requirements of your application. + - Pre-Built Evaluators: Azure AI provides a range of [generation safety](Supported_Evaluation_Metrics/AI_Judge_Evaluators_Safety_Risks/) and [quality/NLP evaluators](Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality/) ready for immediate use. + - [Custom Evaluators](Supported_Evaluation_Metrics/Custom_Evaluators/): Using the Azure AI Evaluation SDK, you can design and implement evaluators that align with the unique requirements of your application. -* Generating and Visualizing Evaluation Results: Azure AI Evaluation SDK enables you to evaluate the target functions (such as [endpoints of your AI application](Supported_Evaluation_Targets\Evaluate_App_Endpoint\) or your [model endpoints](Supported_Evaluation_Targets\Evaluate_Base_Model_Endpoint\) on your dataset with either built-in or custom evaluators. You can run evaluations [remotely](Supported_Evaluation_Targets\Evaluate_Remotely\) in the cloud or locally on your own machine. +* Generating and Visualizing Evaluation Results: Azure AI Evaluation SDK enables you to evaluate the target functions (such as [endpoints of your AI application](Supported_Evaluation_Targets/Evaluate_App_Endpoint/) or your [model endpoints](Supported_Evaluation_Targets/Evaluate_Base_Model_Endpoint/) on your dataset with either built-in or custom evaluators. You can run evaluations [remotely](Supported_Evaluation_Targets/Evaluate_Remotely/) in the cloud or locally on your own machine. ### Objective @@ -43,17 +43,17 @@ The main objective of this tutorial is to help users understand the process of e | Sample name | adversarial | simulator | conversation starter | index | raw text | against model endpoint | against app | qualitative metrics | custom metrics | quantitative NLP metrics | |----------------------------------------|-------------|-----------|---------------------|-------|----------|-----------------------|-------------|---------------------|----------------|----------------------| -| [Simulate_Adversarial.ipynb](\Simulators\Simulate_Adversarial_Data\simulate_adversarial.ipynb) | X | X | | | | X | | | | | -| [Simulate_From_Conversation_Starter.ipynb](\Simulators\Simulate_Context-Relevant_Data\Simulate_From_Conversation_Starter\Simulate_From_Conversation_Starter.ipynb) | | X | X | | | X | | | | | -| [Simulate_From_Azure_Search_Index.ipynb](\Simulators\Simulate_Context-Relevant_Data\Simulate_From_Azure_Search_Index\Simulate_From_Azure_Search_Index.ipynb) | | X | | X | | X | | | | | -| [Simulate_From_Input_Text.ipynb](\Simulators\Simulate_Context-Relevant_Data\Simulate_From_Input_Text\Simulate_From_Input_Text.ipynb) | | X | | | X | X | | | | | -| [Evaluate_Base_Model_Endpoint.ipynb](\Supported_Evaluation_Targets\Evaluate_Base_Model_Endpoint\Evaluate_Base_Model_Endpoint.ipynb) | | | | | | X | | X | | | -| [Evaluate_App_Endpoint.ipynb](scenarios\evaluate\Supported_Evaluation_Targets\Evaluate_App_Endpoint\Evaluate_App_Endpoint.ipynb) | | | | | | | X | X | | | -| [AI_Judge_Evaluators_Quality.ipynb](\Supported_Evaluation_Metrics\AI_Judge_evaluators_quality\AI_Judge_Evaluators_Quality.ipynb) | | | | | | X | | X | | | -| [Custum_Evaluators.ipynb](\Supported_Evaluation_Metrics\Custom_Evaluators\Custom_Evaluators.ipynb) | | | | | | X | | | X | | -| [NLP_Evaluators.ipynb](\Supported_Evaluation_Metrics\NLP_Evaluators\NLP_Evaluators.ipynb) | | | | | | X | | | | X | -| [AI_Judge_Evaluators_Safety_Risks.ipynb](\Supported_Evaluation_Metrics\AI_Judge_Evaluators_Safety_Risks\AI_Judge_Evaluators_Safety_Risks.ipynb) | X | | | | | X | | | | | -| [Simulate_Evaluate_Groundedness.py](\Simulators\Simulate_Evaluate_Groundedness\Simulate_Evaluate_Groundedness.ipynb) | | X | | | X | X | | X | | | +| [Simulate_Adversarial.ipynb](Simulators/Simulate_Adversarial_Data/simulate_adversarial.ipynb) | X | X | | | | X | | | | | +| [Simulate_From_Conversation_Starter.ipynb](Simulators/Simulate_Context-Relevant_Data/Simulate_From_Conversation_Starter/Simulate_From_Conversation_Starter.ipynb) | | X | X | | | X | | | | | +| [Simulate_From_Azure_Search_Index.ipynb](Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/Simulate_From_Azure_Search_Index.ipynb) | | X | | X | | X | | | | | +| [Simulate_From_Input_Text.ipynb](Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/Simulate_From_Input_Text.ipynb) | | X | | | X | X | | | | | +| [Evaluate_Base_Model_Endpoint.ipynb](Supported_Evaluation_Targets/Evaluate_Base_Model_Endpoint/Evaluate_Base_Model_Endpoint.ipynb) | | | | | | X | | X | | | +| [Evaluate_App_Endpoint.ipynb](Supported_Evaluation_Targets/Evaluate_App_Endpoint/Evaluate_App_Endpoint.ipynb) | | | | | | | X | X | | | +| [AI_Judge_Evaluators_Quality.ipynb](Supported_Evaluation_Metrics/AI_Judge_evaluators_quality/AI_Judge_Evaluators_Quality.ipynb) | | | | | | X | | X | | | +| [Custum_Evaluators.ipynb](Supported_Evaluation_Metrics/Custom_Evaluators/Custom_Evaluators.ipynb) | | | | | | X | | | X | | +| [NLP_Evaluators.ipynb](Supported_Evaluation_Metrics/NLP_Evaluators/NLP_Evaluators.ipynb) | | | | | | X | | | | X | +| [AI_Judge_Evaluators_Safety_Risks.ipynb](Supported_Evaluation_Metrics/AI_Judge_Evaluators_Safety_Risks/AI_Judge_Evaluators_Safety_Risks.ipynb) | X | | | | | X | | | | | +| [Simulate_Evaluate_Groundedness.py](Simulators/Simulate_Evaluate_Groundedness/Simulate_Evaluate_Groundedness.ipynb) | | X | | | X | X | | X | | | diff --git a/scenarios/evaluate/Simulators/README.md b/scenarios/evaluate/Simulators/README.md index e0577dc4..0d99482f 100644 --- a/scenarios/evaluate/Simulators/README.md +++ b/scenarios/evaluate/Simulators/README.md @@ -20,5 +20,5 @@ Relevant, robust evaluation data is essential for effective evaluations. This da - Adversarial simulators, on the other hand, generate queries designed to challenge the AI system, mimicking potential security threats or attempting to provoke undesirable behaviors. This approach helps identify the model's limitations and prepares it to perform well in unexpected or hostile conditions. -Azure AI Studio provides tools for both topic-related and adversarial simulations, enabling comprehensive evaluation and enhancing confidence in deployment. For topic-related simulations, Azure AI enables you to simulate relevant conversations using [your data](Simulate_Context-Relevant_Data/Simulate_From_Input_Text/Simulate_From_Input_Text.ipynb), [your Azure Search Index](Simulate_Context-Relevant_Data\Simulate_From_Azure_Search_Index\Simulate_From_Azure_Search_Index.ipynb), or [your pre-defined conversation starters](Simulate_Context-Relevant_Data\Simulate_From_Conversation_Starter\Simulate_From_Conversation_Starter.ipynb). +Azure AI Studio provides tools for both topic-related and adversarial simulations, enabling comprehensive evaluation and enhancing confidence in deployment. For topic-related simulations, Azure AI enables you to simulate relevant conversations using [your data](Simulate_Context-Relevant_Data/Simulate_From_Input_Text/Simulate_From_Input_Text.ipynb), [your Azure Search Index](Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/Simulate_From_Azure_Search_Index.ipynb), or [your pre-defined conversation starters](Simulate_Context-Relevant_Data/Simulate_From_Conversation_Starter/Simulate_From_Conversation_Starter.ipynb). diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/README.md b/scenarios/evaluate/Supported_Evaluation_Metrics/README.md index d364322c..a2043756 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/README.md +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/README.md @@ -17,13 +17,13 @@ Currently, Azure AI Evaluation SDK supports three types of evaluators: ![Types of Evaluators](./AutomatedEvaluationAzureAIFoundry.jpg) -* [Risk and safety evaluators](\AI_Judge_Evaluators_Safety_Risks\): Evaluating potential risks associated with AI-generated content is essential for safeguarding against content risks with varying degrees of severity. This includes evaluating an AI system's predisposition towards generating harmful or inappropriate content. +* [Risk and safety evaluators](AI_Judge_Evaluators_Safety_Risks/): Evaluating potential risks associated with AI-generated content is essential for safeguarding against content risks with varying degrees of severity. This includes evaluating an AI system's predisposition towards generating harmful or inappropriate content. -* Generation quality evaluators: This involves assessing metrics such as the groundedness, coherence and relevance of generated content using robust [AI-assisted](AI_Judge_Evaluators_Quality\) and [NLP](NLP_Evaluators\) metrics. +* Generation quality evaluators: This involves assessing metrics such as the groundedness, coherence and relevance of generated content using robust [AI-assisted](AI_Judge_Evaluators_Quality/) and [NLP](NLP_Evaluators/) metrics. -* [Custom evaluators](Custom_Evaluators\): Tailored evaluation metrics can be designed to meet specific needs and goals, providing flexibility and precision in assessing unique aspects of AI-generated content. These custom evaluators allow for more detailed and specific analyses, addressing particular concerns or requirements that standard metrics may not cover. +* [Custom evaluators](Custom_Evaluators/): Tailored evaluation metrics can be designed to meet specific needs and goals, providing flexibility and precision in assessing unique aspects of AI-generated content. These custom evaluators allow for more detailed and specific analyses, addressing particular concerns or requirements that standard metrics may not cover. -You can run evaluators locally or [remotely](..\Supported_Evaluation_Targets\Evaluate_Remotely\Evaluate_Remotely.ipynb), log results in the cloud using the evaluation SDK, or integrate them into automated evaluations within the Azure AI Studio UI. +You can run evaluators locally or [remotely](../Supported_Evaluation_Targets/Evaluate_Remotely/Evaluate_Remotely.ipynb), log results in the cloud using the evaluation SDK, or integrate them into automated evaluations within the Azure AI Studio UI. diff --git a/scenarios/evaluate/Supported_Evaluation_Targets/README.md b/scenarios/evaluate/Supported_Evaluation_Targets/README.md index 79382ede..ff05e8e8 100644 --- a/scenarios/evaluate/Supported_Evaluation_Targets/README.md +++ b/scenarios/evaluate/Supported_Evaluation_Targets/README.md @@ -23,13 +23,13 @@ Evaluation plays a critical role at three pivotal stages: The first stage of the AI lifecycle involves selecting an appropriate base model. With over 1,600 base models available in the Azure AI Studio's model catalog, it's crucial to choose one that best aligns with your specific use case. This involves comparing models based on quality and safety metrics like groundedness, relevance, and safety scores, as well as considering cost, computational efficiency, and latency. -Follow [this tutorial](Evaluate_Base_Model_Endpoint\Evaluate_Base_Model_Endpoint.ipynb) to evaluate and compare base models on your own data. +Follow [this tutorial](Evaluate_Base_Model_Endpoint/Evaluate_Base_Model_Endpoint.ipynb) to evaluate and compare base models on your own data. ### Pre-Production AI Application Evaluation Once a base model is selected, the next stage is building an AI application around it—such as an AI-powered copilot, a retrieval-augmented generation (RAG) system, an agentic AI system, or any other generative AI-based tool. Before deploying the AI application into a production environment, rigorous testing is essential to ensure the model is truly ready for real-world use. -Follow [this tutorial](Evaluate_App_Endpoint\Evaluate_App_Endpoint.ipynb) to evaluate any AI application endpoint on your own data. +Follow [this tutorial](Evaluate_App_Endpoint/Evaluate_App_Endpoint.ipynb) to evaluate any AI application endpoint on your own data. ### Post-Production AI Application Evaluation (Monitoring) From 0039cbe023343f7d70fd13cfce9a0c519b1950b5 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Mon, 18 Nov 2024 17:12:54 -0800 Subject: [PATCH 09/10] update hyperlinks third and final --- scenarios/evaluate/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scenarios/evaluate/README.md b/scenarios/evaluate/README.md index a372aacd..b00008eb 100644 --- a/scenarios/evaluate/README.md +++ b/scenarios/evaluate/README.md @@ -43,13 +43,13 @@ The main objective of this tutorial is to help users understand the process of e | Sample name | adversarial | simulator | conversation starter | index | raw text | against model endpoint | against app | qualitative metrics | custom metrics | quantitative NLP metrics | |----------------------------------------|-------------|-----------|---------------------|-------|----------|-----------------------|-------------|---------------------|----------------|----------------------| -| [Simulate_Adversarial.ipynb](Simulators/Simulate_Adversarial_Data/simulate_adversarial.ipynb) | X | X | | | | X | | | | | +| [Simulate_Adversarial.ipynb](Simulators/Simulate_Adversarial_Data/Simulate_Adversarial.ipynb) | X | X | | | | X | | | | | | [Simulate_From_Conversation_Starter.ipynb](Simulators/Simulate_Context-Relevant_Data/Simulate_From_Conversation_Starter/Simulate_From_Conversation_Starter.ipynb) | | X | X | | | X | | | | | | [Simulate_From_Azure_Search_Index.ipynb](Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/Simulate_From_Azure_Search_Index.ipynb) | | X | | X | | X | | | | | | [Simulate_From_Input_Text.ipynb](Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/Simulate_From_Input_Text.ipynb) | | X | | | X | X | | | | | | [Evaluate_Base_Model_Endpoint.ipynb](Supported_Evaluation_Targets/Evaluate_Base_Model_Endpoint/Evaluate_Base_Model_Endpoint.ipynb) | | | | | | X | | X | | | | [Evaluate_App_Endpoint.ipynb](Supported_Evaluation_Targets/Evaluate_App_Endpoint/Evaluate_App_Endpoint.ipynb) | | | | | | | X | X | | | -| [AI_Judge_Evaluators_Quality.ipynb](Supported_Evaluation_Metrics/AI_Judge_evaluators_quality/AI_Judge_Evaluators_Quality.ipynb) | | | | | | X | | X | | | +| [AI_Judge_Evaluators_Quality.ipynb](Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality/AI_Judge_Evaluators_Quality.ipynb) | | | | | | X | | X | | | | [Custum_Evaluators.ipynb](Supported_Evaluation_Metrics/Custom_Evaluators/Custom_Evaluators.ipynb) | | | | | | X | | | X | | | [NLP_Evaluators.ipynb](Supported_Evaluation_Metrics/NLP_Evaluators/NLP_Evaluators.ipynb) | | | | | | X | | | | X | | [AI_Judge_Evaluators_Safety_Risks.ipynb](Supported_Evaluation_Metrics/AI_Judge_Evaluators_Safety_Risks/AI_Judge_Evaluators_Safety_Risks.ipynb) | X | | | | | X | | | | | From c40ac177c7405566c7058e1ca1202dbca67421e8 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Tue, 19 Nov 2024 09:22:56 -0800 Subject: [PATCH 10/10] change remote eval to eval on cloud --- scenarios/evaluate/README.md | 2 +- .../Supported_Evaluation_Metrics/README.md | 2 +- .../Evaluate_On_Cloud.ipynb} | 23 ++++++++++--------- .../README.md | 9 ++++---- .../evaluate_test_data.jsonl | 0 5 files changed, 18 insertions(+), 18 deletions(-) rename scenarios/evaluate/Supported_Evaluation_Targets/{Evaluate_Remotely/Evaluate_Remotely.ipynb => Evaluate_On_Cloud/Evaluate_On_Cloud.ipynb} (79%) rename scenarios/evaluate/Supported_Evaluation_Targets/{Evaluate_Remotely => Evaluate_On_Cloud}/README.md (78%) rename scenarios/evaluate/Supported_Evaluation_Targets/{Evaluate_Remotely => Evaluate_On_Cloud}/evaluate_test_data.jsonl (100%) diff --git a/scenarios/evaluate/README.md b/scenarios/evaluate/README.md index b00008eb..68072df8 100644 --- a/scenarios/evaluate/README.md +++ b/scenarios/evaluate/README.md @@ -29,7 +29,7 @@ In both cases, running evaluations requires specific tools, methods, and dataset - Pre-Built Evaluators: Azure AI provides a range of [generation safety](Supported_Evaluation_Metrics/AI_Judge_Evaluators_Safety_Risks/) and [quality/NLP evaluators](Supported_Evaluation_Metrics/AI_Judge_Evaluators_Quality/) ready for immediate use. - [Custom Evaluators](Supported_Evaluation_Metrics/Custom_Evaluators/): Using the Azure AI Evaluation SDK, you can design and implement evaluators that align with the unique requirements of your application. -* Generating and Visualizing Evaluation Results: Azure AI Evaluation SDK enables you to evaluate the target functions (such as [endpoints of your AI application](Supported_Evaluation_Targets/Evaluate_App_Endpoint/) or your [model endpoints](Supported_Evaluation_Targets/Evaluate_Base_Model_Endpoint/) on your dataset with either built-in or custom evaluators. You can run evaluations [remotely](Supported_Evaluation_Targets/Evaluate_Remotely/) in the cloud or locally on your own machine. +* Generating and Visualizing Evaluation Results: Azure AI Evaluation SDK enables you to evaluate the target functions (such as [endpoints of your AI application](Supported_Evaluation_Targets/Evaluate_App_Endpoint/) or your [model endpoints](Supported_Evaluation_Targets/Evaluate_Base_Model_Endpoint/) on your dataset with either built-in or custom evaluators. You can run evaluations [remotely](Supported_Evaluation_Targets/Evaluate_On_Cloud/) in the cloud or locally on your own machine. ### Objective diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/README.md b/scenarios/evaluate/Supported_Evaluation_Metrics/README.md index a2043756..8b9e3dd4 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/README.md +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/README.md @@ -26,4 +26,4 @@ Currently, Azure AI Evaluation SDK supports three types of evaluators: -You can run evaluators locally or [remotely](../Supported_Evaluation_Targets/Evaluate_Remotely/Evaluate_Remotely.ipynb), log results in the cloud using the evaluation SDK, or integrate them into automated evaluations within the Azure AI Studio UI. +You can run evaluators locally or [remotely](../Supported_Evaluation_Targets/Evaluate_On_Cloud/Evaluate_On_Cloud.ipynb), log results in the cloud using the evaluation SDK, or integrate them into automated evaluations within the Azure AI Studio UI. diff --git a/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/Evaluate_Remotely.ipynb b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_On_Cloud/Evaluate_On_Cloud.ipynb similarity index 79% rename from scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/Evaluate_Remotely.ipynb rename to scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_On_Cloud/Evaluate_On_Cloud.ipynb index 2d37962a..b6b5f351 100644 --- a/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/Evaluate_Remotely.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_On_Cloud/Evaluate_On_Cloud.ipynb @@ -4,11 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Remote Evaluations: Evaluating in the Cloud \n", + "# Cloud evaluation: Evaluating AI app data remotely in the cloud \n", "\n", "## Objective\n", "\n", - "This tutorial provides a step-by-step guide on how to evaluate data generated by LLMs remotely in the cloud. \n", + "This tutorial provides a step-by-step guide on how to evaluate data generated by AI applications or LLMs remotely in the cloud. \n", "\n", "This tutorial uses the following Azure AI services:\n", "\n", @@ -21,14 +21,15 @@ "\n", "## About this example\n", "\n", - "This example demonstrates the remote evaluation of query and response pairs that were generated by a LLM. It is important to have access to AzureOpenAI credentials and an AzureAI project. **To create data to use in your own evaluation, learn more [here](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/simulator-interaction-data)** . This example demonstrates: \n", + "This example demonstrates the cloud evaluation of query and response pairs that were generated by an AI app or a LLM. It is important to have access to AzureOpenAI credentials and an AzureAI project. **To create data to use in your own evaluation, learn more [here](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/simulator-interaction-data)** . This example demonstrates: \n", "\n", - "- Single-instance, triggered Remote Evaluation (to be used for pre-deployment evaluation of LLMs)\n", + "- Single-instance, triggered cloud evaluation on a test dataset (to be used for pre-deployment evaluation of an AI application).\n", "\n", "## Before you begin\n", "### Prerequesite\n", - "- [Have an online deployment on Azure Open AI studio supporting `chat completion` such as `gpt-4`](https://learn.microsoft.com/en-us/azure/machine-learning/concept-endpoints-online?view=azureml-api-2)\n", - "- You also might want to evaluate data generated by your LLM, to see how to generate data to be evaluated using the Azure AI Evaluation SDK, see our samples on simulation \n", + "- Have an Azure OpenAI Deployment with GPT model supporting `chat completion`, for example `gpt-4`.\n", + "- Make sure you're first logged into your Azure subscription by running `az login`.\n", + "- You have some test data you want to evaluate, which includes the user queries and responses (and perhaps context, or ground truth) from your AI applications. See [data requirements for our built-in evaluators](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk#data-requirements-for-built-in-evaluators). Alternatively, if you want to simulate data against your application endpoints using Azure AI Evaluation SDK, see our samples on simulation. \n", "\n", "### Installation\n", "\n", @@ -124,7 +125,7 @@ "metadata": {}, "source": [ "### Configure Evaluators to Run\n", - "The code below demonstrates how to configure the evaluators you want to run. In this example, we use the `F1ScoreEvaluator`, `RelevanceEvaluator` and the `ViolenceEvaluator`, but all evaluators supported by [Azure AI Evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/concepts/evaluation-metrics-built-in?tabs=warning) are supported by Online Evaluation and can be configured here. You can either import the classes from the SDK and reference them with the `.id` property, or you can find the fully formed `id` of the evaluator in the AI Studio registry of evaluators, and use it here. " + "The code below demonstrates how to configure the evaluators you want to run. In this example, we use the `F1ScoreEvaluator`, `RelevanceEvaluator` and the `ViolenceEvaluator`, but all evaluators supported by [Azure AI Evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/concepts/evaluation-metrics-built-in?tabs=warning) are supported by cloud evaluation and can be configured here. You can either import the classes from the SDK and reference them with the `.id` property, or you can find the fully formed `id` of the evaluator in the AI Studio registry of evaluators, and use it here. " ] }, { @@ -157,8 +158,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Evaluate in the Cloud with Remote Evaluation\n", - "Below we demonstrate how to trigger a single-instance Remote Evaluation in the Cloud. This can be used for pre-deployment testing of an LLM. \n", + "### Create cloud evaluation\n", + "Below we demonstrate how to trigger a single-instance Cloud Evaluation remotely on a test dataset. This can be used for pre-deployment testing of your AI application. \n", " \n", "Here we pass in the `data_id` we would like to use for the evaluation. " ] @@ -170,8 +171,8 @@ "outputs": [], "source": [ "evaluation = Evaluation(\n", - " display_name=\"Remote Evaluation\",\n", - " description=\"Evaluation of dataset\",\n", + " display_name=\"Cloud Evaluation\",\n", + " description=\"Cloud Evaluation of dataset\",\n", " data=Dataset(id=data_id),\n", " evaluators=evaluators,\n", ")\n", diff --git a/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/README.md b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_On_Cloud/README.md similarity index 78% rename from scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/README.md rename to scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_On_Cloud/README.md index 190f49cc..fe017505 100644 --- a/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/README.md +++ b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_On_Cloud/README.md @@ -5,14 +5,14 @@ languages: products: - ai-services - azure-openai -description: Evaluating remotely +description: Cloud evaluation --- ## Evaluating in the cloud ### Overview -This tutorial provides a step-by-step guide on how to evaluate generative AI or LLMs remotely using a triggered evaluation. +This tutorial provides a step-by-step guide on how to evaluate generative AI applications or LLMs remotely using a triggered cloud evaluation. ### Objective @@ -22,16 +22,15 @@ The main objective of this tutorial is to help users understand the process of e - Evaluate LLM using various evaluators from Azure AI Evaluations SDK remotely in the cloud. ### Note -Remote evaluations do not support `Retrieval-Evaluator`, `ContentSafetyEvaluator`, and `QAEvaluator`. +Cloud evaluations do not support `ContentSafetyEvaluator`, and `QAEvaluator`. -#### Region Support for Evaluations +#### Region Support for Cloud evaluations | Region | Hate and Unfairness, Sexual, Violent, Self-Harm, XPIA | Groundedness Pro | Protected Material | | - | - | - | - | | UK South | Will be deprecated 12/1/24 | no | no | | East US 2 | yes | yes | yes | | Sweden Central | yes | yes | no | -| US North Central | yes | no | no | | France Central | yes | no | no | | Switzerland West | yes | no | no | diff --git a/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/evaluate_test_data.jsonl b/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_On_Cloud/evaluate_test_data.jsonl similarity index 100% rename from scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Remotely/evaluate_test_data.jsonl rename to scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_On_Cloud/evaluate_test_data.jsonl