From 40ca32c2c199e4d3eb08d2a8fdc0a8cc93b0cc33 Mon Sep 17 00:00:00 2001 From: Sydney Date: Thu, 26 Sep 2024 16:56:52 -0700 Subject: [PATCH 1/6] update promptflow-eval dependencies to azure-ai-evaluation --- scenarios/evaluate-app-endpoint/data.jsonl | 6 +- .../evaluate-target.ipynb | 49 +- .../evaluate-model-endpoints/app_target.py | 18 +- .../evaluate-models-target.ipynb | 469 ++++++++++++++++-- .../simulate_and_evaluate_ask_wiki.ipynb | 79 ++- ...imulate_and_evaluate_online_endpoint.ipynb | 53 +- 6 files changed, 521 insertions(+), 153 deletions(-) diff --git a/scenarios/evaluate-app-endpoint/data.jsonl b/scenarios/evaluate-app-endpoint/data.jsonl index 47d5a9bf..b6ae357f 100644 --- a/scenarios/evaluate-app-endpoint/data.jsonl +++ b/scenarios/evaluate-app-endpoint/data.jsonl @@ -1,3 +1,3 @@ -{"question":"When was United Stated found ?", "ground_truth":"1776"} -{"question":"What is the capital of France?", "ground_truth":"Paris"} -{"question":"Who is the best tennis player of all time ?", "ground_truth":"Roger Federer"} \ No newline at end of file +{"query":"When was United Stated found ?", "response":"1776", "question":"When was United Stated found ?"} +{"query":"What is the capital of France?", "response":"Paris", "question":"What is the capital of France?"} +{"query":"Who is the best tennis player of all time ?", "response":"Roger Federer", "question": "Who is the best tennis player of all time ?"} \ No newline at end of file diff --git a/scenarios/evaluate-app-endpoint/evaluate-target.ipynb b/scenarios/evaluate-app-endpoint/evaluate-target.ipynb index 36574089..796bb70b 100644 --- a/scenarios/evaluate-app-endpoint/evaluate-target.ipynb +++ b/scenarios/evaluate-app-endpoint/evaluate-target.ipynb @@ -20,7 +20,7 @@ "\n", "This tutorial uses the following Azure AI services:\n", "\n", - "- [promptflow-evals](https://microsoft.github.io/promptflow/reference/python-library-reference/promptflow-evals/promptflow.html)\n", + "- [azure-ai-evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk)\n", "\n", "## Time\n", "\n", @@ -28,7 +28,7 @@ "\n", "## About this example\n", "\n", - "This example demonstrates evaluating a target function using promptflow-evals\n", + "This example demonstrates evaluating a target function using azure-ai-evaluation\n", "\n", "## Before you begin\n", "\n", @@ -44,7 +44,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install promptflow-evals" + "%pip install azure-ai-evaluation" ] }, { @@ -57,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "257fd898-7ef2-4d89-872e-da9e426aaf0b", "metadata": {}, "outputs": [], @@ -66,9 +66,8 @@ "import os\n", "\n", "from pprint import pprint\n", - "from promptflow.core import AzureOpenAIModelConfiguration\n", - "from promptflow.evals.evaluate import evaluate\n", - "from promptflow.evals.evaluators import RelevanceEvaluator" + "from azure.ai.evaluation import evaluate\n", + "from azure.ai.evaluation import RelevanceEvaluator" ] }, { @@ -89,17 +88,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "fbfc3a3b", "metadata": {}, "outputs": [], "source": [ "# Use the following code to set the environment variables if not already set. If set, you can skip this step.\n", "\n", - "os.environ[\"AZURE_OPENAI_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"\"\n", - "os.environ[\"AZURE_OPENAI_DEPLOYMENT\"] = \"\"\n", - "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"\"" + "os.environ[\"AZURE_OPENAI_API_KEY\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_DEPLOYMENT\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"\"" ] }, { @@ -111,7 +110,7 @@ "source": [ "from askwiki import ask_wiki\n", "\n", - "ask_wiki(\"What is the capital of India?\")" + "ask_wiki(question=\"What is the capital of India?\")" ] }, { @@ -145,17 +144,16 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "58f76d0c-2d44-44e0-8296-110477c7e559", + "execution_count": 16, + "id": "665d0e98", "metadata": {}, "outputs": [], "source": [ - "configuration = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"AZURE_OPENAI_DEPLOYMENT\"],\n", - ")" + "model_config = {\n", + " \"azure_endpoint\": os.environ.get(\"AZURE_OPENAI_ENDPOINT\"),\n", + " \"api_key\": os.environ.get(\"AZURE_OPENAI_KEY\"),\n", + " \"azure_deployment\": os.environ.get(\"AZURE_OPENAI_DEPLOYMENT\"),\n", + " }" ] }, { @@ -173,12 +171,12 @@ "metadata": {}, "outputs": [], "source": [ - "relevance_evaluator = RelevanceEvaluator(model_config=configuration)\n", + "relevance_evaluator = RelevanceEvaluator(model_config)\n", "\n", "relevance_evaluator(\n", - " question=\"What is the capital of India?\",\n", - " answer=\"New Delhi is Capital of India\",\n", + " response=\"New Delhi is Capital of India\",\n", " context=\"India is a country in South Asia.\",\n", + " query=\"What is the capital of India?\"\n", ")" ] }, @@ -250,7 +248,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.11.9" } }, "nbformat": 4, diff --git a/scenarios/evaluate-model-endpoints/app_target.py b/scenarios/evaluate-model-endpoints/app_target.py index 7025b9a6..17b0a467 100644 --- a/scenarios/evaluate-model-endpoints/app_target.py +++ b/scenarios/evaluate-model-endpoints/app_target.py @@ -15,7 +15,7 @@ class Response(TypedDict): @trace def __call__(self: Self, question: str) -> Response: - if self.model_type == "gpt4-0613": + if self.model_type == "gpt4-0613" or self.model_type == "gpt-4o-mini": output = self.call_gpt4_endpoint(question) elif self.model_type == "gpt35-turbo": output = self.call_gpt35_turbo_endpoint(question) @@ -45,8 +45,8 @@ def call_gpt4_endpoint(self: Self, question: str) -> Response: payload = {"messages": [{"role": "user", "content": question}], "max_tokens": 500} output = self.query(endpoint=endpoint, headers=headers, payload=payload) - answer = output["choices"][0]["message"]["content"] - return {"question": question, "answer": answer} + answer = output["choices"][0]["message"]["content"] + return {"query": question, "response": answer} def call_gpt35_turbo_endpoint(self: Self, question: str) -> Response: endpoint = self.env["gpt35-turbo"]["endpoint"] @@ -58,7 +58,7 @@ def call_gpt35_turbo_endpoint(self: Self, question: str) -> Response: output = self.query(endpoint=endpoint, headers=headers, payload=payload) answer = output["choices"][0]["message"]["content"] - return {"question": question, "answer": answer} + return {"query": question, "response": answer} def call_tiny_llama_endpoint(self: Self, question: str) -> Response: endpoint = self.env["tiny_llama"]["endpoint"] @@ -75,7 +75,7 @@ def call_tiny_llama_endpoint(self: Self, question: str) -> Response: output = self.query(endpoint=endpoint, headers=headers, payload=payload) answer = output["choices"][0]["message"]["content"] - return {"question": question, "answer": answer} + return {"query": question, "response": answer} def call_phi3_mini_serverless_endpoint(self: Self, question: str) -> Response: endpoint = self.env["phi3_mini_serverless"]["endpoint"] @@ -87,7 +87,7 @@ def call_phi3_mini_serverless_endpoint(self: Self, question: str) -> Response: output = self.query(endpoint=endpoint, headers=headers, payload=payload) answer = output["choices"][0]["message"]["content"] - return {"question": question, "answer": answer} + return {"query": question, "response": answer} def call_gpt2_endpoint(self: Self, question: str) -> Response: endpoint = self.env["gpt2"]["endpoint"] @@ -101,7 +101,7 @@ def call_gpt2_endpoint(self: Self, question: str) -> Response: output = self.query(endpoint=endpoint, headers=headers, payload=payload) answer = output[0]["generated_text"] - return {"question": question, "answer": answer} + return {"query": question, "response": answer} def call_mistral_endpoint(self: Self, question: str) -> Response: endpoint = self.env["mistral7b"]["endpoint"] @@ -113,7 +113,7 @@ def call_mistral_endpoint(self: Self, question: str) -> Response: output = self.query(endpoint=endpoint, headers=headers, payload=payload) answer = output["choices"][0]["message"]["content"] - return {"question": question, "answer": answer} + return {"query": question, "response": answer} def call_default_endpoint(question: str) -> Response: - return {"question": "What is the capital of France?", "answer": "Paris"} + return {"query": "What is the capital of France?", "response": "Paris"} diff --git a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb index 6ecba4e5..adbea2bb 100644 --- a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb +++ b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb @@ -14,7 +14,7 @@ "\n", "This tutorial uses the following Azure AI services:\n", "\n", - "- [promptflow-evals](https://microsoft.github.io/promptflow/reference/python-library-reference/promptflow-evals/promptflow.html)\n", + "- [azure-ai-evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk)\n", "\n", "## Time\n", "\n", @@ -22,7 +22,7 @@ "\n", "## About this example\n", "\n", - "This example demonstrates evaluating model endpoints responses against provided prompts using promptflow-evals\n", + "This example demonstrates evaluating model endpoints responses against provided prompts using azure-ai-evaluation\n", "\n", "## Before you begin\n", "\n", @@ -33,11 +33,277 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n", + "Requirement already satisfied: azure-ai-evaluation in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (1.0.0b1)\n", + "Requirement already satisfied: promptflow-devkit>=1.15.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.15.1)\n", + "Requirement already satisfied: promptflow-core>=1.15.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.15.1)\n", + "Requirement already satisfied: pyjwt>=2.8.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (2.9.0)\n", + "Requirement already satisfied: azure-identity>=1.12.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.18.0)\n", + "Requirement already satisfied: azure-core>=1.30.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.31.0)\n", + "Requirement already satisfied: nltk>=3.9.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (3.9.1)\n", + "Requirement already satisfied: rouge-score>=0.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (0.1.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.26.4)\n", + "Requirement already satisfied: requests>=2.21.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core>=1.30.2->azure-ai-evaluation) (2.31.0)\n", + "Requirement already satisfied: six>=1.11.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core>=1.30.2->azure-ai-evaluation) (1.16.0)\n", + "Requirement already satisfied: typing-extensions>=4.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core>=1.30.2->azure-ai-evaluation) (4.12.2)\n", + "Requirement already satisfied: cryptography>=2.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity>=1.12.0->azure-ai-evaluation) (43.0.1)\n", + "Requirement already satisfied: msal>=1.30.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity>=1.12.0->azure-ai-evaluation) (1.31.0)\n", + "Requirement already satisfied: msal-extensions>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity>=1.12.0->azure-ai-evaluation) (1.2.0)\n", + "Requirement already satisfied: click in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (8.1.7)\n", + "Requirement already satisfied: joblib in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (1.4.2)\n", + "Requirement already satisfied: regex>=2021.8.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (2023.12.25)\n", + "Requirement already satisfied: tqdm in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (4.66.2)\n", + "Requirement already satisfied: docstring_parser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (0.16)\n", + "Requirement already satisfied: fastapi<1.0.0,>=0.109.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (0.115.0)\n", + "Requirement already satisfied: filetype>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (1.2.0)\n", + "Requirement already satisfied: flask<4.0.0,>=2.2.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (3.0.3)\n", + "Requirement already satisfied: jsonschema<5.0.0,>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (4.23.0)\n", + "Requirement already satisfied: promptflow-tracing==1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (1.15.1)\n", + "Requirement already satisfied: psutil in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (5.9.8)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (2.9.0.post0)\n", + "Requirement already satisfied: ruamel.yaml<1.0.0,>=0.17.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (0.18.6)\n", + "Requirement already satisfied: openai in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (1.47.1)\n", + "Requirement already satisfied: opentelemetry-sdk<2.0.0,>=1.22.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (1.27.0)\n", + "Requirement already satisfied: tiktoken>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (0.7.0)\n", + "Requirement already satisfied: argcomplete>=3.2.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.5.0)\n", + "Requirement already satisfied: azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.0.0b30)\n", + "Requirement already satisfied: colorama<0.5.0,>=0.4.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.4.6)\n", + "Requirement already satisfied: filelock<4.0.0,>=3.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.13.1)\n", + "Requirement already satisfied: flask-cors<5.0.0,>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.0.2)\n", + "Requirement already satisfied: flask-restx<2.0.0,>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.3.0)\n", + "Requirement already satisfied: gitpython<4.0.0,>=3.1.24 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.1.43)\n", + "Requirement already satisfied: httpx>=0.25.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.27.2)\n", + "Requirement already satisfied: keyring<25.0.0,>=24.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (24.3.1)\n", + "Requirement already satisfied: marshmallow<4.0.0,>=3.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.22.0)\n", + "Requirement already satisfied: opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", + "Requirement already satisfied: pandas<3.0.0,>=1.5.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.2.3)\n", + "Requirement already satisfied: pillow<11.0.0,>=10.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (10.4.0)\n", + "Requirement already satisfied: pydash<8.0.0,>=6.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (7.0.7)\n", + "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.0.1)\n", + "Requirement already satisfied: pywin32 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (306)\n", + "Requirement already satisfied: sqlalchemy<3.0.0,>=1.4.48 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.0.35)\n", + "Requirement already satisfied: strictyaml<2.0.0,>=1.5.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.7.3)\n", + "Requirement already satisfied: tabulate<1.0.0,>=0.9.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.9.0)\n", + "Requirement already satisfied: waitress<3.0.0,>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.1.2)\n", + "Requirement already satisfied: absl-py in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from rouge-score>=0.1.2->azure-ai-evaluation) (2.1.0)\n", + "Requirement already satisfied: fixedint==0.1.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.1.6)\n", + "Requirement already satisfied: msrest>=0.6.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.7.1)\n", + "Requirement already satisfied: opentelemetry-api~=1.26 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", + "Requirement already satisfied: cffi>=1.12 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cryptography>=2.5->azure-identity>=1.12.0->azure-ai-evaluation) (1.17.1)\n", + "Requirement already satisfied: starlette<0.39.0,>=0.37.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.38.6)\n", + "Requirement already satisfied: pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (2.9.2)\n", + "Requirement already satisfied: Werkzeug>=3.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (3.0.4)\n", + "Requirement already satisfied: Jinja2>=3.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (3.1.3)\n", + "Requirement already satisfied: itsdangerous>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (2.2.0)\n", + "Requirement already satisfied: blinker>=1.6.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (1.8.2)\n", + "Requirement already satisfied: aniso8601>=0.82 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (9.0.1)\n", + "Requirement already satisfied: pytz in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2024.2)\n", + "Requirement already satisfied: importlib-resources in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (6.4.5)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitpython<4.0.0,>=3.1.24->promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.0.11)\n", + "Requirement already satisfied: anyio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.6.0)\n", + "Requirement already satisfied: certifi in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2024.2.2)\n", + "Requirement already satisfied: httpcore==1.* in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.0.5)\n", + "Requirement already satisfied: idna in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.6)\n", + "Requirement already satisfied: sniffio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.3.1)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpcore==1.*->httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.14.0)\n", + "Requirement already satisfied: attrs>=22.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (24.2.0)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (2023.12.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.20.0)\n", + "Requirement already satisfied: jaraco.classes in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.4.0)\n", + "Requirement already satisfied: importlib-metadata>=4.11.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (8.4.0)\n", + "Requirement already satisfied: pywin32-ctypes>=0.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.2.3)\n", + "Requirement already satisfied: packaging>=17.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from marshmallow<4.0.0,>=3.5->promptflow-devkit>=1.15.0->azure-ai-evaluation) (24.0)\n", + "Requirement already satisfied: portalocker<3,>=1.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msal-extensions>=1.2.0->azure-identity>=1.12.0->azure-ai-evaluation) (2.10.1)\n", + "Requirement already satisfied: deprecated>=1.2.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.2.14)\n", + "Requirement already satisfied: googleapis-common-protos~=1.52 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.65.0)\n", + "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", + "Requirement already satisfied: opentelemetry-proto==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", + "Requirement already satisfied: protobuf<5.0,>=3.19 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-proto==1.27.0->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.25.5)\n", + "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2024.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core>=1.30.2->azure-ai-evaluation) (3.3.2)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core>=1.30.2->azure-ai-evaluation) (2.2.1)\n", + "Requirement already satisfied: ruamel.yaml.clib>=0.2.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from ruamel.yaml<1.0.0,>=0.17.10->promptflow-core>=1.15.0->azure-ai-evaluation) (0.2.8)\n", + "Requirement already satisfied: greenlet!=0.4.17 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from sqlalchemy<3.0.0,>=1.4.48->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.1.1)\n", + "Requirement already satisfied: pycparser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity>=1.12.0->azure-ai-evaluation) (2.22)\n", + "Requirement already satisfied: wrapt<2,>=1.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from deprecated>=1.2.6->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.16.0)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitdb<5,>=4.0.1->gitpython<4.0.0,>=3.1.24->promptflow-devkit>=1.15.0->azure-ai-evaluation) (5.0.1)\n", + "Requirement already satisfied: zipp>=0.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from importlib-metadata>=4.11.4->keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.20.2)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Jinja2>=3.1.2->flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (2.1.5)\n", + "Requirement already satisfied: isodate>=0.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msrest>=0.6.10->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.6.1)\n", + "Requirement already satisfied: requests-oauthlib>=0.5.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msrest>=0.6.10->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.0.0)\n", + "Requirement already satisfied: opentelemetry-semantic-conventions==0.48b0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-sdk<2.0.0,>=1.22.0->promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (0.48b0)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.23.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (2.23.4)\n", + "Requirement already satisfied: more-itertools in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jaraco.classes->keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (10.5.0)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (1.9.0)\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (0.5.0)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests-oauthlib>=0.5.0->msrest>=0.6.10->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.2.2)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[notice] A new release of pip is available: 24.0 -> 24.2\n", + "[notice] To update, run: C:\\Users\\sydneylister\\AppData\\Local\\Microsoft\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\python.exe -m pip install --upgrade pip\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n", + "Requirement already satisfied: promptflow-azure in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (1.15.1)\n", + "Requirement already satisfied: azure-ai-ml<2.0.0,>=1.14.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.20.0)\n", + "Requirement already satisfied: azure-core<2.0.0,>=1.26.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.31.0)\n", + "Requirement already satisfied: azure-cosmos<5.0.0,>=4.5.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (4.7.0)\n", + "Requirement already satisfied: azure-identity<2.0.0,>=1.12.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.18.0)\n", + "Requirement already satisfied: azure-storage-blob<13.0.0,>=12.17.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (12.23.0)\n", + "Requirement already satisfied: promptflow-devkit<2.0.0,>=1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.15.1)\n", + "Requirement already satisfied: pyjwt<3.0.0,>=2.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (2.9.0)\n", + "Requirement already satisfied: pyyaml>=5.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (6.0.1)\n", + "Requirement already satisfied: msrest>=0.6.18 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.7.1)\n", + "Requirement already satisfied: azure-mgmt-core>=1.3.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.4.0)\n", + "Requirement already satisfied: marshmallow>=3.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (3.22.0)\n", + "Requirement already satisfied: jsonschema>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.23.0)\n", + "Requirement already satisfied: tqdm in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.66.2)\n", + "Requirement already satisfied: strictyaml in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.7.3)\n", + "Requirement already satisfied: colorama in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.4.6)\n", + "Requirement already satisfied: azure-storage-file-share in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (12.18.0)\n", + "Requirement already satisfied: azure-storage-file-datalake>=12.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (12.17.0)\n", + "Requirement already satisfied: pydash>=6.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (7.0.7)\n", + "Requirement already satisfied: isodate in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.6.1)\n", + "Requirement already satisfied: azure-common>=1.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.1.28)\n", + "Requirement already satisfied: typing-extensions in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.12.2)\n", + "Requirement already satisfied: opencensus-ext-azure in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.1.13)\n", + "Requirement already satisfied: opencensus-ext-logging in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.1.1)\n", + "Requirement already satisfied: requests>=2.21.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core<2.0.0,>=1.26.4->promptflow-azure) (2.31.0)\n", + "Requirement already satisfied: six>=1.11.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core<2.0.0,>=1.26.4->promptflow-azure) (1.16.0)\n", + "Requirement already satisfied: cryptography>=2.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity<2.0.0,>=1.12.0->promptflow-azure) (43.0.1)\n", + "Requirement already satisfied: msal>=1.30.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity<2.0.0,>=1.12.0->promptflow-azure) (1.31.0)\n", + "Requirement already satisfied: msal-extensions>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity<2.0.0,>=1.12.0->promptflow-azure) (1.2.0)\n", + "Requirement already satisfied: argcomplete>=3.2.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.5.0)\n", + "Requirement already satisfied: azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.0.0b30)\n", + "Requirement already satisfied: filelock<4.0.0,>=3.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.13.1)\n", + "Requirement already satisfied: flask-cors<5.0.0,>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.0.2)\n", + "Requirement already satisfied: flask-restx<2.0.0,>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.3.0)\n", + "Requirement already satisfied: gitpython<4.0.0,>=3.1.24 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.1.43)\n", + "Requirement already satisfied: httpx>=0.25.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.27.2)\n", + "Requirement already satisfied: keyring<25.0.0,>=24.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (24.3.1)\n", + "Requirement already satisfied: opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", + "Requirement already satisfied: pandas<3.0.0,>=1.5.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.2.3)\n", + "Requirement already satisfied: pillow<11.0.0,>=10.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (10.4.0)\n", + "Requirement already satisfied: promptflow-core<2.0.0,>=1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.15.1)\n", + "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.0.1)\n", + "Requirement already satisfied: pywin32 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (306)\n", + "Requirement already satisfied: sqlalchemy<3.0.0,>=1.4.48 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.0.35)\n", + "Requirement already satisfied: tabulate<1.0.0,>=0.9.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.9.0)\n", + "Requirement already satisfied: waitress<3.0.0,>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.1.2)\n", + "Requirement already satisfied: aiohttp>=3.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (3.10.6)\n", + "Requirement already satisfied: fixedint==0.1.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.1.6)\n", + "Requirement already satisfied: opentelemetry-api~=1.26 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", + "Requirement already satisfied: opentelemetry-sdk~=1.26 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", + "Requirement already satisfied: psutil~=5.9 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (5.9.8)\n", + "Requirement already satisfied: cffi>=1.12 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cryptography>=2.5->azure-identity<2.0.0,>=1.12.0->promptflow-azure) (1.17.1)\n", + "Requirement already satisfied: Flask>=0.9 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.0.3)\n", + "Requirement already satisfied: aniso8601>=0.82 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (9.0.1)\n", + "Requirement already satisfied: werkzeug!=2.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.0.4)\n", + "Requirement already satisfied: pytz in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2024.2)\n", + "Requirement already satisfied: importlib-resources in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (6.4.5)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitpython<4.0.0,>=3.1.24->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.0.11)\n", + "Requirement already satisfied: anyio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.6.0)\n", + "Requirement already satisfied: certifi in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2024.2.2)\n", + "Requirement already satisfied: httpcore==1.* in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.0.5)\n", + "Requirement already satisfied: idna in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.6)\n", + "Requirement already satisfied: sniffio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.3.1)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpcore==1.*->httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.14.0)\n", + "Requirement already satisfied: attrs>=22.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (24.2.0)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2023.12.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.20.0)\n", + "Requirement already satisfied: jaraco.classes in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.4.0)\n", + "Requirement already satisfied: importlib-metadata>=4.11.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (8.4.0)\n", + "Requirement already satisfied: pywin32-ctypes>=0.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.2.3)\n", + "Requirement already satisfied: packaging>=17.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from marshmallow>=3.5->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (24.0)\n", + "Requirement already satisfied: portalocker<3,>=1.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msal-extensions>=1.2.0->azure-identity<2.0.0,>=1.12.0->promptflow-azure) (2.10.1)\n", + "Requirement already satisfied: requests-oauthlib>=0.5.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msrest>=0.6.18->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2.0.0)\n", + "Requirement already satisfied: deprecated>=1.2.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.2.14)\n", + "Requirement already satisfied: googleapis-common-protos~=1.52 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.65.0)\n", + "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", + "Requirement already satisfied: opentelemetry-proto==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", + "Requirement already satisfied: protobuf<5.0,>=3.19 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-proto==1.27.0->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.25.5)\n", + "Requirement already satisfied: numpy>=1.23.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.26.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.9.0.post0)\n", + "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2024.2)\n", + "Requirement already satisfied: docstring_parser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.16)\n", + "Requirement already satisfied: fastapi<1.0.0,>=0.109.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.115.0)\n", + "Requirement already satisfied: filetype>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.2.0)\n", + "Requirement already satisfied: promptflow-tracing==1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.15.1)\n", + "Requirement already satisfied: ruamel.yaml<1.0.0,>=0.17.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.18.6)\n", + "Requirement already satisfied: openai in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.47.1)\n", + "Requirement already satisfied: tiktoken>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.7.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core<2.0.0,>=1.26.4->promptflow-azure) (3.3.2)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core<2.0.0,>=1.26.4->promptflow-azure) (2.2.1)\n", + "Requirement already satisfied: greenlet!=0.4.17 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from sqlalchemy<3.0.0,>=1.4.48->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.1.1)\n", + "Requirement already satisfied: opencensus<1.0.0,>=0.11.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.11.4)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (2.4.0)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (1.3.1)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (1.4.1)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (6.1.0)\n", + "Requirement already satisfied: yarl<2.0,>=1.12.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (1.12.1)\n", + "Requirement already satisfied: pycparser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity<2.0.0,>=1.12.0->promptflow-azure) (2.22)\n", + "Requirement already satisfied: wrapt<2,>=1.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from deprecated>=1.2.6->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.16.0)\n", + "Requirement already satisfied: starlette<0.39.0,>=0.37.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.38.6)\n", + "Requirement already satisfied: pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.9.2)\n", + "Requirement already satisfied: Jinja2>=3.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.1.3)\n", + "Requirement already satisfied: itsdangerous>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.2.0)\n", + "Requirement already satisfied: click>=8.1.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (8.1.7)\n", + "Requirement already satisfied: blinker>=1.6.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.8.2)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitdb<5,>=4.0.1->gitpython<4.0.0,>=3.1.24->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (5.0.1)\n", + "Requirement already satisfied: zipp>=0.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from importlib-metadata>=4.11.4->keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.20.2)\n", + "Requirement already satisfied: opencensus-context>=0.1.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.1.3)\n", + "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2.20.0)\n", + "Requirement already satisfied: opentelemetry-semantic-conventions==0.48b0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-sdk~=1.26->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.48b0)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests-oauthlib>=0.5.0->msrest>=0.6.18->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (3.2.2)\n", + "Requirement already satisfied: ruamel.yaml.clib>=0.2.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from ruamel.yaml<1.0.0,>=0.17.10->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.2.8)\n", + "Requirement already satisfied: MarkupSafe>=2.1.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from werkzeug!=2.0.0->flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.1.5)\n", + "Requirement already satisfied: more-itertools in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jaraco.classes->keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (10.5.0)\n", + "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.24.0)\n", + "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2.35.0)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.23.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.23.4)\n", + "Requirement already satisfied: regex>=2022.1.18 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from tiktoken>=0.4.0->promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2023.12.25)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.9.0)\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.5.0)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (5.5.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.4.1)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.9)\n", + "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.6.1)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[notice] A new release of pip is available: 24.0 -> 24.2\n", + "[notice] To update, run: C:\\Users\\sydneylister\\AppData\\Local\\Microsoft\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\python.exe -m pip install --upgrade pip\n" + ] + } + ], "source": [ - "%pip install promptflow-evals\n", + "%pip install azure-ai-evaluation\n", "%pip install promptflow-azure" ] }, @@ -50,7 +316,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -75,7 +341,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -117,11 +383,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ - "azure_ai_project = {\"subscription_id\": \"***\", \"resource_group_name\": \"***\", \"project_name\": \"***\"}" + "azure_ai_project = {\"subscription_id\": \"\", \"resource_group_name\": \"\", \"project_name\": \"\"}" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Use the following code to set the environment variables if not already set. If set, you can skip this step.\n", + "\n", + "os.environ[\"AZURE_OPENAI_API_KEY\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_DEPLOYMENT\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"\"" ] }, { @@ -135,9 +417,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " question \\\n", + "0 What is the capital of France? \n", + "1 Which tent is the most waterproof? \n", + "2 Which camping table is the lightest? \n", + "3 How much does TrailWalker Hiking Shoes cost? \n", + "\n", + " context \\\n", + "0 France is the country in Europe. \n", + "1 #TrailMaster X4 Tent, price $250,## BrandOutdo... \n", + "2 #BaseCamp Folding Table, price $60,## BrandCam... \n", + "3 #TrailWalker Hiking Shoes, price $110## BrandT... \n", + "\n", + " ground_truth \n", + "0 Paris \n", + "1 The TrailMaster X4 tent has a rainfly waterpro... \n", + "2 The BaseCamp Folding Table has a weight of 15 lbs \n", + "3 The TrailWalker Hiking Shoes are priced at $110 \n" + ] + } + ], "source": [ "df = pd.read_json(\"data.jsonl\", lines=True)\n", "print(df.head())" @@ -153,18 +459,15 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ - "from promptflow.core import AzureOpenAIModelConfiguration\n", - "\n", - "configuration = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=\"https://ai-***.openai.azure.com\",\n", - " api_key=\"**\",\n", - " api_version=\"2023-03-15-preview\",\n", - " azure_deployment=\"gpt-35-turbo-16k\",\n", - ")" + "model_config = {\n", + " \"azure_endpoint\": os.environ.get(\"AZURE_OPENAI_ENDPOINT\"),\n", + " \"api_key\": os.environ.get(\"AZURE_OPENAI_KEY\"),\n", + " \"azure_deployment\": os.environ.get(\"AZURE_OPENAI_DEPLOYMENT\"),\n", + " }" ] }, { @@ -188,15 +491,103 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-09-26 16:53:21 -0700][promptflow][WARNING] - Found existing c:/azureai-samples/scenarios/evaluate-model-endpoints/flow.flex.yaml, will not respect it in runtime.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Prompt flow service has started...\n", + "You can view the traces in local from http://127.0.0.1:23333/v1.0/ui/traces/?#run=evaluate_model_endpoints_20240926_165321_352972\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-09-26 16:53:28 -0700][promptflow._sdk._orchestrator.run_submitter][INFO] - Submitting run evaluate_model_endpoints_20240926_165321_352972, log path: C:\\Users\\sydneylister\\.promptflow\\.runs\\evaluate_model_endpoints_20240926_165321_352972\\logs.txt\n", + "[2024-09-26 16:53:47 -0700][promptflow._sdk._orchestrator.run_submitter][WARNING] - 4 out of 4 runs failed in batch run.\n", + " Please check out C:/Users/sydneylister/.promptflow/.runs/evaluate_model_endpoints_20240926_165321_352972 for more details.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2024-09-26 16:53:28 -0700 39604 execution.bulk INFO Current thread is not main thread, skip signal handler registration in BatchEngine.\n", + "2024-09-26 16:53:28 -0700 39604 execution WARNING Starting run without column mapping may lead to unexpected results. Please consult the following documentation for more information: https://aka.ms/pf/column-mapping\n", + "2024-09-26 16:53:28 -0700 39604 execution.bulk INFO Current system's available memory is 10048.08203125MB, memory consumption of current process is 372.0625MB, estimated available worker count is 10048.08203125/372.0625 = 27\n", + "2024-09-26 16:53:28 -0700 39604 execution.bulk INFO Set process count to 4 by taking the minimum value among the factors of {'default_worker_count': 4, 'row_count': 4, 'estimated_worker_count_based_on_memory_usage': 27}.\n", + "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-32)-Process id(43356)-Line number(0) start execution.\n", + "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-33)-Process id(3604)-Line number(1) start execution.\n", + "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-35)-Process id(8800)-Line number(2) start execution.\n", + "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-34)-Process id(21616)-Line number(3) start execution.\n", + "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-32)-Process id(43356)-Line number(0) completed.\n", + "2024-09-26 16:53:38 -0700 39604 execution.bulk INFO Finished 1 / 4 lines.\n", + "2024-09-26 16:53:38 -0700 39604 execution.bulk INFO Average execution time for completed lines: 1.01 seconds. Estimated time for incomplete lines: 3.03 seconds.\n", + "2024-09-26 16:53:43 -0700 39604 execution.bulk INFO Process name(SpawnProcess-33)-Process id(3604)-Line number(1) completed.\n", + "2024-09-26 16:53:43 -0700 39604 execution.bulk INFO Finished 2 / 4 lines.\n", + "2024-09-26 16:53:43 -0700 39604 execution.bulk INFO Average execution time for completed lines: 3.03 seconds. Estimated time for incomplete lines: 6.06 seconds.\n", + "2024-09-26 16:53:44 -0700 39604 execution.bulk INFO Process name(SpawnProcess-34)-Process id(21616)-Line number(3) completed.\n", + "2024-09-26 16:53:45 -0700 39604 execution.bulk INFO Finished 3 / 4 lines.\n", + "2024-09-26 16:53:45 -0700 39604 execution.bulk INFO Average execution time for completed lines: 2.69 seconds. Estimated time for incomplete lines: 2.69 seconds.\n", + "2024-09-26 16:53:45 -0700 39604 execution.bulk INFO Process name(SpawnProcess-35)-Process id(8800)-Line number(2) completed.\n", + "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO Finished 4 / 4 lines.\n", + "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO Average execution time for completed lines: 2.27 seconds. Estimated time for incomplete lines: 0.0 seconds.\n", + "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [21616-SpawnProcess-34] will be terminated.\n", + "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [8800-SpawnProcess-35] will be terminated.\n", + "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [43356-SpawnProcess-32] will be terminated.\n", + "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [3604-SpawnProcess-33] will be terminated.\n", + "2024-09-26 16:53:46 -0700 21616 execution.bulk INFO The process [21616] has received a terminate signal.\n", + "2024-09-26 16:53:46 -0700 8800 execution.bulk INFO The process [8800] has received a terminate signal.\n", + "2024-09-26 16:53:46 -0700 43356 execution.bulk INFO The process [43356] has received a terminate signal.\n", + "2024-09-26 16:53:46 -0700 3604 execution.bulk INFO The process [3604] has received a terminate signal.\n", + "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 43356 terminated.\n", + "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 3604 terminated.\n", + "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 21616 terminated.\n", + "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 8800 terminated.\n", + "2024-09-26 16:53:47 -0700 39604 execution ERROR 4/4 flow run failed, indexes: [0,1,2,3], exception of index 0: Execution failure in 'ModelEndpoints.__call__': (ConnectionError) HTTPSConnectionPool(host='ai-***.**.azure.com', port=443): Max retries exceeded with url: /openai/deployments/gpt-4/chat/completions?api-version=2023-03-15-preview (Caused by NameResolutionError(\": Failed to resolve 'ai-***.**.azure.com' ([Errno 11001] getaddrinfo failed)\"))\n", + "======= Run Summary =======\n", + "\n", + "Run name: \"evaluate_model_endpoints_20240926_165321_352972\"\n", + "Run status: \"Completed\"\n", + "Start time: \"2024-09-26 16:53:21.348453-07:00\"\n", + "Duration: \"0:00:26.215496\"\n", + "Output path: \"C:\\Users\\sydneylister\\.promptflow\\.runs\\evaluate_model_endpoints_20240926_165321_352972\"\n", + "\n" + ] + }, + { + "ename": "EvaluationException", + "evalue": "Missing required inputs for evaluator content_safety : ['response'].", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mEvaluationException\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[25], line 35\u001b[0m\n\u001b[0;32m 33\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m model \u001b[38;5;129;01min\u001b[39;00m models:\n\u001b[0;32m 34\u001b[0m randomNum \u001b[38;5;241m=\u001b[39m random\u001b[38;5;241m.\u001b[39mrandint(\u001b[38;5;241m1111\u001b[39m, \u001b[38;5;241m9999\u001b[39m)\n\u001b[1;32m---> 35\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 36\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# azure_ai_project=azure_ai_project,\u001b[39;49;00m\n\u001b[0;32m 37\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluation_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mEval-Run-\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mrandomNum\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m-\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtitle\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 38\u001b[0m \u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 39\u001b[0m \u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mModelEndpoints\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv_var\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 40\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluators\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\n\u001b[0;32m 41\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontent_safety\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontent_safety_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 42\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcoherence\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mcoherence_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 43\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrelevance\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mrelevance_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 44\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgroundedness\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroundedness_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 45\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfluency\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mfluency_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 46\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msimilarity\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43msimilarity_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 47\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 48\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluator_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\n\u001b[0;32m 49\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontent_safety\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 50\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcoherence\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 51\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrelevance\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.query}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 52\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgroundedness\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[0;32m 53\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m 54\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m 55\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m 56\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 57\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfluency\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 58\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msimilarity\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 59\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 60\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:545\u001b[0m, in \u001b[0;36mevaluate\u001b[1;34m(evaluation_name, target, data, evaluators, evaluator_config, azure_ai_project, output_path, **kwargs)\u001b[0m\n\u001b[0;32m 531\u001b[0m error_message \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m 532\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe evaluation failed due to an error during multiprocess bootstrapping.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 533\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPlease ensure the evaluate API is properly guarded with the \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m block:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 534\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m if __name__ == \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 535\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m evaluate(...)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 536\u001b[0m )\n\u001b[0;32m 537\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m EvaluationException(\n\u001b[0;32m 538\u001b[0m message\u001b[38;5;241m=\u001b[39merror_message,\n\u001b[0;32m 539\u001b[0m internal_message\u001b[38;5;241m=\u001b[39merror_message,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 542\u001b[0m blame\u001b[38;5;241m=\u001b[39mErrorBlame\u001b[38;5;241m.\u001b[39mUNKNOWN,\n\u001b[0;32m 543\u001b[0m ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m--> 545\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:514\u001b[0m, in \u001b[0;36mevaluate\u001b[1;34m(evaluation_name, target, data, evaluators, evaluator_config, azure_ai_project, output_path, **kwargs)\u001b[0m\n\u001b[0;32m 448\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Evaluates target or data with built-in or custom evaluators. If both target and data are provided,\u001b[39;00m\n\u001b[0;32m 449\u001b[0m \u001b[38;5;124;03m data will be run through target function and then results will be evaluated.\u001b[39;00m\n\u001b[0;32m 450\u001b[0m \n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 511\u001b[0m \n\u001b[0;32m 512\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 513\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 514\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_evaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 515\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluation_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluation_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 516\u001b[0m \u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtarget\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 517\u001b[0m \u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 518\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluators\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluators\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 519\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluator_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluator_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 520\u001b[0m \u001b[43m \u001b[49m\u001b[43mazure_ai_project\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mazure_ai_project\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 521\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 522\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 523\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 524\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m 525\u001b[0m \u001b[38;5;66;03m# Handle multiprocess bootstrap error\u001b[39;00m\n\u001b[0;32m 526\u001b[0m bootstrap_error \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m 527\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn attempt has been made to start a new process before the\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 528\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcurrent process has finished its bootstrapping phase.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 529\u001b[0m )\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:605\u001b[0m, in \u001b[0;36m_evaluate\u001b[1;34m(evaluation_name, target, data, evaluators, evaluator_config, azure_ai_project, output_path, **kwargs)\u001b[0m\n\u001b[0;32m 601\u001b[0m evaluator_config[evaluator_name][col] \u001b[38;5;241m=\u001b[39m run_output \u001b[38;5;66;03m# pylint: disable=unnecessary-dict-index-lookup\u001b[39;00m\n\u001b[0;32m 603\u001b[0m \u001b[38;5;66;03m# After we have generated all columns we can check if we have\u001b[39;00m\n\u001b[0;32m 604\u001b[0m \u001b[38;5;66;03m# everything we need for evaluators.\u001b[39;00m\n\u001b[1;32m--> 605\u001b[0m \u001b[43m_validate_columns\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_data_df\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevaluators\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevaluator_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluator_config\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 607\u001b[0m \u001b[38;5;66;03m# Batch Run\u001b[39;00m\n\u001b[0;32m 608\u001b[0m evaluators_info \u001b[38;5;241m=\u001b[39m {}\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:314\u001b[0m, in \u001b[0;36m_validate_columns\u001b[1;34m(df, evaluators, target, evaluator_config)\u001b[0m\n\u001b[0;32m 311\u001b[0m new_df \u001b[38;5;241m=\u001b[39m _apply_column_mapping(df, mapping_config)\n\u001b[0;32m 313\u001b[0m \u001b[38;5;66;03m# Validate input data for evaluator\u001b[39;00m\n\u001b[1;32m--> 314\u001b[0m \u001b[43m_validate_input_data_for_evaluator\u001b[49m\u001b[43m(\u001b[49m\u001b[43mevaluator\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevaluator_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnew_df\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:165\u001b[0m, in \u001b[0;36m_validate_input_data_for_evaluator\u001b[1;34m(evaluator, evaluator_name, df_data, is_target_fn)\u001b[0m\n\u001b[0;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_target_fn:\n\u001b[0;32m 164\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMissing required inputs for evaluator \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mevaluator_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmissing_inputs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m--> 165\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m EvaluationException(\n\u001b[0;32m 166\u001b[0m message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[0;32m 167\u001b[0m internal_message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[0;32m 168\u001b[0m target\u001b[38;5;241m=\u001b[39mErrorTarget\u001b[38;5;241m.\u001b[39mEVALUATE,\n\u001b[0;32m 169\u001b[0m category\u001b[38;5;241m=\u001b[39mErrorCategory\u001b[38;5;241m.\u001b[39mMISSING_FIELD,\n\u001b[0;32m 170\u001b[0m blame\u001b[38;5;241m=\u001b[39mErrorBlame\u001b[38;5;241m.\u001b[39mUSER_ERROR,\n\u001b[0;32m 171\u001b[0m )\n\u001b[0;32m 172\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMissing required inputs for target : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmissing_inputs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 173\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m EvaluationException(\n\u001b[0;32m 174\u001b[0m message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[0;32m 175\u001b[0m internal_message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 178\u001b[0m blame\u001b[38;5;241m=\u001b[39mErrorBlame\u001b[38;5;241m.\u001b[39mUSER_ERROR,\n\u001b[0;32m 179\u001b[0m )\n", + "\u001b[1;31mEvaluationException\u001b[0m: Missing required inputs for evaluator content_safety : ['response']." + ] + } + ], "source": [ "from app_target import ModelEndpoints\n", "import pathlib\n", "\n", - "from promptflow.evals.evaluate import evaluate\n", - "from promptflow.evals.evaluators import (\n", + "from azure.ai.evaluation import evaluate\n", + "from azure.ai.evaluation import (\n", " ContentSafetyEvaluator,\n", " RelevanceEvaluator,\n", " CoherenceEvaluator,\n", @@ -206,12 +597,12 @@ ")\n", "\n", "\n", - "content_safety_evaluator = ContentSafetyEvaluator(project_scope=azure_ai_project)\n", - "relevance_evaluator = RelevanceEvaluator(model_config=configuration)\n", - "coherence_evaluator = CoherenceEvaluator(model_config=configuration)\n", - "groundedness_evaluator = GroundednessEvaluator(model_config=configuration)\n", - "fluency_evaluator = FluencyEvaluator(model_config=configuration)\n", - "similarity_evaluator = SimilarityEvaluator(model_config=configuration)\n", + "content_safety_evaluator = ContentSafetyEvaluator(azure_ai_project)\n", + "relevance_evaluator = RelevanceEvaluator(model_config)\n", + "coherence_evaluator = CoherenceEvaluator(model_config)\n", + "groundedness_evaluator = GroundednessEvaluator(model_config)\n", + "fluency_evaluator = FluencyEvaluator(model_config)\n", + "similarity_evaluator = SimilarityEvaluator(model_config)\n", "\n", "models = [\n", " \"gpt4-0613\",\n", @@ -227,7 +618,6 @@ "for model in models:\n", " randomNum = random.randint(1111, 9999)\n", " results = evaluate(\n", - " azure_ai_project=azure_ai_project,\n", " evaluation_name=\"Eval-Run-\" + str(randomNum) + \"-\" + model.title(),\n", " data=path,\n", " target=ModelEndpoints(env_var, model),\n", @@ -240,16 +630,16 @@ " \"similarity\": similarity_evaluator,\n", " },\n", " evaluator_config={\n", - " \"content_safety\": {\"question\": \"${data.question}\", \"answer\": \"${target.answer}\"},\n", - " \"coherence\": {\"answer\": \"${target.answer}\", \"question\": \"${data.question}\"},\n", - " \"relevance\": {\"answer\": \"${target.answer}\", \"context\": \"${data.context}\", \"question\": \"${data.question}\"},\n", + " \"content_safety\": {\"query\": \"${data.question}\", \"response\": \"${target.response}\"},\n", + " \"coherence\": {\"response\": \"${target.response}\", \"query\": \"${data.question}\"},\n", + " \"relevance\": {\"response\": \"${target.query}\", \"context\": \"${data.context}\", \"query\": \"${data.question}\"},\n", " \"groundedness\": {\n", - " \"answer\": \"${target.answer}\",\n", + " \"response\": \"${target.response}\",\n", " \"context\": \"${data.context}\",\n", - " \"question\": \"${data.question}\",\n", + " \"query\": \"${data.question}\",\n", " },\n", - " \"fluency\": {\"answer\": \"${target.answer}\", \"context\": \"${data.context}\", \"question\": \"${data.question}\"},\n", - " \"similarity\": {\"answer\": \"${target.answer}\", \"context\": \"${data.context}\", \"question\": \"${data.question}\"},\n", + " \"fluency\": {\"response\": \"${target.response}\", \"context\": \"${data.context}\", \"query\": \"${data.question}\"},\n", + " \"similarity\": {\"response\": \"${target.response}\", \"context\": \"${data.context}\", \"query\": \"${data.question}\"},\n", " },\n", " )" ] @@ -295,7 +685,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.11.9" } }, "nbformat": 4, diff --git a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb index 4881a040..8a807308 100644 --- a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb +++ b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb @@ -13,7 +13,7 @@ "This tutorial uses the following Azure AI services:\n", "\n", "- [Azure AI Safety Evaluation](https://aka.ms/azureaistudiosafetyeval)\n", - "- [promptflow-evals](https://microsoft.github.io/promptflow/reference/python-library-reference/promptflow-evals/promptflow.html)\n", + "- [azure-ai-evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk)\n", "\n", "## Time\n", "\n", @@ -36,7 +36,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install promptflow-evals" + "%pip install azure-ai-evaluation" ] }, { @@ -48,13 +48,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 72, "metadata": {}, "outputs": [], "source": [ "from pathlib import Path\n", - "from azure.identity import DefaultAzureCredential\n", - "from promptflow.evals.synthetic import AdversarialSimulator, AdversarialScenario\n", + "from azure.ai.evaluation.simulator import AdversarialSimulator, AdversarialScenario\n", "from typing import List, Dict, Any, Optional" ] }, @@ -78,22 +77,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 75, + "metadata": {}, + "outputs": [], + "source": [ + "azure_ai_project = {\"subscription_id\": \"\", \"resource_group_name\": \"\", \"project_name\": \"\"}" + ] + }, + { + "cell_type": "code", + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", - "os.environ[\"AZURE_OPENAI_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"\"\n", - "os.environ[\"AZURE_OPENAI_DEPLOYMENT\"] = \"\"\n", - "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"\"\n", - "azure_ai_project = {\n", - " \"subscription_id\": \"\",\n", - " \"resource_group_name\": \"\",\n", - " \"project_name\": \"\",\n", - " \"credential\": DefaultAzureCredential(),\n", - "}" + "# Use the following code to set the environment variables if not already set. If set, you can skip this step.\n", + "\n", + "os.environ[\"AZURE_OPENAI_API_KEY\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_DEPLOYMENT\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"\"" ] }, { @@ -141,7 +145,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 78, "metadata": {}, "outputs": [], "source": [ @@ -217,15 +221,11 @@ "metadata": {}, "outputs": [], "source": [ - "from promptflow.evals.evaluators import (\n", + "from azure.ai.evaluation import (\n", " ContentSafetyEvaluator,\n", ")\n", - "from promptflow.evals.evaluate import evaluate\n", "\n", - "# credential is acquired within evaluate\n", - "azure_ai_project.pop(\"credential\", None)\n", - "\n", - "content_safety_evaluator = ContentSafetyEvaluator(project_scope=azure_ai_project)" + "content_safety_evaluator = ContentSafetyEvaluator(azure_ai_project)" ] }, { @@ -237,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 84, "metadata": {}, "outputs": [], "source": [ @@ -245,7 +245,7 @@ "\n", "json_line_as_json = json.loads(outputs.to_eval_qa_json_lines())\n", "content_safety_eval_result = content_safety_evaluator(\n", - " question=json_line_as_json[\"question\"], answer=json_line_as_json[\"answer\"]\n", + " query=json_line_as_json[\"query\"], response=json_line_as_json[\"response\"]\n", ")\n", "# {'violence': 'Safe',\n", "# 'violence_score': 0,\n", @@ -287,26 +287,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Run the evaluate method with the content safety evaluator and blocklist evaluator" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "results = evaluate(\n", - " data=\"outputs.jsonl\",\n", - " evaluators={\"content_safety\": content_safety_evaluator, \"blocklist\": blocklist_evaluator},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### View the results of the evaluation" + "### Run the blocklist evaluator" ] }, { @@ -315,7 +296,10 @@ "metadata": {}, "outputs": [], "source": [ - "results" + "json_lines = outputs.to_eval_qa_json_lines()\n", + "for line in json_lines.split(\"\\n\"):\n", + " if line:\n", + " print(blocklist_evaluator(answer=json.loads(line)['response']))" ] } ], @@ -334,7 +318,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.11.9" } }, "nbformat": 4, diff --git a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb index cff321b3..b5a30761 100644 --- a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb +++ b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb @@ -13,7 +13,7 @@ "This tutorial uses the following Azure AI services:\n", "\n", "- [Azure AI Safety Evaluation](https://aka.ms/azureaistudiosafetyeval)\n", - "- [promptflow-evals](https://microsoft.github.io/promptflow/reference/python-library-reference/promptflow-evals/promptflow.html)\n", + "- [azure-ai-evaluation](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk)\n", "\n", "## Time\n", "\n", @@ -37,7 +37,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install promptflow-evals\n", + "%pip install azure-ai-evaluation\n", "%pip install requests" ] }, @@ -50,14 +50,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "import json\n", "from pathlib import Path\n", - "from azure.identity import DefaultAzureCredential\n", - "from promptflow.evals.synthetic import AdversarialSimulator, AdversarialScenario\n", + "from azure.ai.evaluation.simulator import AdversarialSimulator, AdversarialScenario\n", "import requests\n", "from typing import Optional, List, Dict, Any" ] @@ -69,7 +68,7 @@ "## Target function\n", "The target function for this sample uses a call to call the endpoint.\n", "\n", - "Make sure you retrive the `key`, `endpoint` and `azure_model_deployment` from Azure AI studio" + "Make sure you retrive the `api_key`, `endpoint` and `azure_model_deployment` from Azure AI studio and update them in the `call_endpoint` function below. " ] }, { @@ -78,25 +77,20 @@ "metadata": {}, "outputs": [], "source": [ - "azure_ai_project = {\n", - " \"subscription_id\": \"\",\n", - " \"resource_group_name\": \"\",\n", - " \"project_name\": \"\",\n", - " \"credential\": DefaultAzureCredential(),\n", - "}" + "azure_ai_project = {\"subscription_id\": \"\", \"resource_group_name\": \"\", \"project_name\": \"\"}" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ "def call_endpoint(query: str) -> dict:\n", " data = {\"query\": query}\n", " body = json.dumps(data)\n", - " api_key = \"\"\n", - " endpoint = \"\"\n", + " api_key=\"\"\n", + " endpoint=\"\"\n", " azure_model_deployment = \"\"\n", "\n", " if not api_key:\n", @@ -117,6 +111,7 @@ " print(err.response.text)\n", "\n", " json_output = json.loads(result)\n", + " print(json_output)\n", "\n", " return {\n", " \"answer\": json_output[\"reply\"],\n", @@ -133,7 +128,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ @@ -151,11 +146,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "metadata": {}, "outputs": [], "source": [ - "## define a callback that formats the interaction between the simulator and the ask wiki application\n", + "## define a callback that formats the interaction between the simulator and the online endpoint\n", "\n", "\n", "async def callback(\n", @@ -169,13 +164,11 @@ " latest_message = messages_list[-1]\n", " query = latest_message[\"content\"]\n", " context = None\n", - " response_from_ask_wiki = call_endpoint(query)\n", " # we are formatting the response to follow the openAI chat protocol format\n", " formatted_response = {\n", - " \"content\": response_from_ask_wiki[\"answer\"],\n", + " \"content\": \"I cannot answer\",\n", " \"role\": \"assistant\",\n", " \"context\": {\n", - " \"citations\": response_from_ask_wiki[\"context\"],\n", " },\n", " }\n", " messages[\"messages\"].append(formatted_response)\n", @@ -202,11 +195,10 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ - "outputs.to_eval_qa_json_lines()\n", "with Path.open(\"outputs.jsonl\", \"w\") as f:\n", " f.write(outputs.to_eval_qa_json_lines())" ] @@ -222,19 +214,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 47, "metadata": {}, "outputs": [], "source": [ - "from promptflow.evals.evaluators import (\n", + "from azure.ai.evaluation import (\n", " ContentSafetyEvaluator,\n", ")\n", - "from promptflow.evals.evaluate import evaluate\n", + "from azure.ai.evaluation import evaluate\n", "\n", "# credential is acquired within evaluate\n", "azure_ai_project.pop(\"credential\", None)\n", "\n", - "content_safety_evaluator = ContentSafetyEvaluator(project_scope=azure_ai_project)" + "content_safety_evaluator = ContentSafetyEvaluator(azure_ai_project)" ] }, { @@ -246,13 +238,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ "json_line_as_json = json.loads(outputs.to_eval_qa_json_lines())\n", "content_safety_eval_result = content_safety_evaluator(\n", - " question=json_line_as_json[\"question\"], answer=json_line_as_json[\"answer\"]\n", + " query=json_line_as_json[\"query\"], response=json_line_as_json[\"response\"]\n", ")\n", "# {'violence': 'Safe',\n", "# 'violence_score': 0,\n", @@ -316,7 +308,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.11.9" } }, "nbformat": 4, From 3c9826967d7a0cda4e022da8237c54b9e1f31359 Mon Sep 17 00:00:00 2001 From: Sydney Date: Thu, 26 Sep 2024 16:58:15 -0700 Subject: [PATCH 2/6] clear local variables --- .../evaluate-model-endpoints/app_target.py | 2 +- .../evaluate-models-target.ipynb | 390 +----------------- 2 files changed, 7 insertions(+), 385 deletions(-) diff --git a/scenarios/evaluate-model-endpoints/app_target.py b/scenarios/evaluate-model-endpoints/app_target.py index 17b0a467..516c6d87 100644 --- a/scenarios/evaluate-model-endpoints/app_target.py +++ b/scenarios/evaluate-model-endpoints/app_target.py @@ -15,7 +15,7 @@ class Response(TypedDict): @trace def __call__(self: Self, question: str) -> Response: - if self.model_type == "gpt4-0613" or self.model_type == "gpt-4o-mini": + if self.model_type == "gpt4-0613": output = self.call_gpt4_endpoint(question) elif self.model_type == "gpt35-turbo": output = self.call_gpt35_turbo_endpoint(question) diff --git a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb index adbea2bb..e6bcdb20 100644 --- a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb +++ b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb @@ -33,275 +33,9 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n", - "Requirement already satisfied: azure-ai-evaluation in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (1.0.0b1)\n", - "Requirement already satisfied: promptflow-devkit>=1.15.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.15.1)\n", - "Requirement already satisfied: promptflow-core>=1.15.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.15.1)\n", - "Requirement already satisfied: pyjwt>=2.8.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (2.9.0)\n", - "Requirement already satisfied: azure-identity>=1.12.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.18.0)\n", - "Requirement already satisfied: azure-core>=1.30.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.31.0)\n", - "Requirement already satisfied: nltk>=3.9.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (3.9.1)\n", - "Requirement already satisfied: rouge-score>=0.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (0.1.2)\n", - "Requirement already satisfied: numpy>=1.23.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-evaluation) (1.26.4)\n", - "Requirement already satisfied: requests>=2.21.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core>=1.30.2->azure-ai-evaluation) (2.31.0)\n", - "Requirement already satisfied: six>=1.11.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core>=1.30.2->azure-ai-evaluation) (1.16.0)\n", - "Requirement already satisfied: typing-extensions>=4.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core>=1.30.2->azure-ai-evaluation) (4.12.2)\n", - "Requirement already satisfied: cryptography>=2.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity>=1.12.0->azure-ai-evaluation) (43.0.1)\n", - "Requirement already satisfied: msal>=1.30.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity>=1.12.0->azure-ai-evaluation) (1.31.0)\n", - "Requirement already satisfied: msal-extensions>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity>=1.12.0->azure-ai-evaluation) (1.2.0)\n", - "Requirement already satisfied: click in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (8.1.7)\n", - "Requirement already satisfied: joblib in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (1.4.2)\n", - "Requirement already satisfied: regex>=2021.8.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (2023.12.25)\n", - "Requirement already satisfied: tqdm in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from nltk>=3.9.1->azure-ai-evaluation) (4.66.2)\n", - "Requirement already satisfied: docstring_parser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (0.16)\n", - "Requirement already satisfied: fastapi<1.0.0,>=0.109.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (0.115.0)\n", - "Requirement already satisfied: filetype>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (1.2.0)\n", - "Requirement already satisfied: flask<4.0.0,>=2.2.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (3.0.3)\n", - "Requirement already satisfied: jsonschema<5.0.0,>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (4.23.0)\n", - "Requirement already satisfied: promptflow-tracing==1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (1.15.1)\n", - "Requirement already satisfied: psutil in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (5.9.8)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (2.9.0.post0)\n", - "Requirement already satisfied: ruamel.yaml<1.0.0,>=0.17.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core>=1.15.0->azure-ai-evaluation) (0.18.6)\n", - "Requirement already satisfied: openai in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (1.47.1)\n", - "Requirement already satisfied: opentelemetry-sdk<2.0.0,>=1.22.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (1.27.0)\n", - "Requirement already satisfied: tiktoken>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (0.7.0)\n", - "Requirement already satisfied: argcomplete>=3.2.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.5.0)\n", - "Requirement already satisfied: azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.0.0b30)\n", - "Requirement already satisfied: colorama<0.5.0,>=0.4.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.4.6)\n", - "Requirement already satisfied: filelock<4.0.0,>=3.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.13.1)\n", - "Requirement already satisfied: flask-cors<5.0.0,>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.0.2)\n", - "Requirement already satisfied: flask-restx<2.0.0,>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.3.0)\n", - "Requirement already satisfied: gitpython<4.0.0,>=3.1.24 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.1.43)\n", - "Requirement already satisfied: httpx>=0.25.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.27.2)\n", - "Requirement already satisfied: keyring<25.0.0,>=24.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (24.3.1)\n", - "Requirement already satisfied: marshmallow<4.0.0,>=3.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.22.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", - "Requirement already satisfied: pandas<3.0.0,>=1.5.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.2.3)\n", - "Requirement already satisfied: pillow<11.0.0,>=10.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (10.4.0)\n", - "Requirement already satisfied: pydash<8.0.0,>=6.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (7.0.7)\n", - "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.0.1)\n", - "Requirement already satisfied: pywin32 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (306)\n", - "Requirement already satisfied: sqlalchemy<3.0.0,>=1.4.48 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.0.35)\n", - "Requirement already satisfied: strictyaml<2.0.0,>=1.5.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.7.3)\n", - "Requirement already satisfied: tabulate<1.0.0,>=0.9.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.9.0)\n", - "Requirement already satisfied: waitress<3.0.0,>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.1.2)\n", - "Requirement already satisfied: absl-py in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from rouge-score>=0.1.2->azure-ai-evaluation) (2.1.0)\n", - "Requirement already satisfied: fixedint==0.1.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.1.6)\n", - "Requirement already satisfied: msrest>=0.6.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.7.1)\n", - "Requirement already satisfied: opentelemetry-api~=1.26 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", - "Requirement already satisfied: cffi>=1.12 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cryptography>=2.5->azure-identity>=1.12.0->azure-ai-evaluation) (1.17.1)\n", - "Requirement already satisfied: starlette<0.39.0,>=0.37.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.38.6)\n", - "Requirement already satisfied: pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (2.9.2)\n", - "Requirement already satisfied: Werkzeug>=3.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (3.0.4)\n", - "Requirement already satisfied: Jinja2>=3.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (3.1.3)\n", - "Requirement already satisfied: itsdangerous>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (2.2.0)\n", - "Requirement already satisfied: blinker>=1.6.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (1.8.2)\n", - "Requirement already satisfied: aniso8601>=0.82 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (9.0.1)\n", - "Requirement already satisfied: pytz in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2024.2)\n", - "Requirement already satisfied: importlib-resources in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (6.4.5)\n", - "Requirement already satisfied: gitdb<5,>=4.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitpython<4.0.0,>=3.1.24->promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.0.11)\n", - "Requirement already satisfied: anyio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.6.0)\n", - "Requirement already satisfied: certifi in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2024.2.2)\n", - "Requirement already satisfied: httpcore==1.* in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.0.5)\n", - "Requirement already satisfied: idna in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.6)\n", - "Requirement already satisfied: sniffio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.3.1)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpcore==1.*->httpx>=0.25.1->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.14.0)\n", - "Requirement already satisfied: attrs>=22.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (24.2.0)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (2023.12.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema<5.0.0,>=4.0.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.20.0)\n", - "Requirement already satisfied: jaraco.classes in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.4.0)\n", - "Requirement already satisfied: importlib-metadata>=4.11.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (8.4.0)\n", - "Requirement already satisfied: pywin32-ctypes>=0.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.2.3)\n", - "Requirement already satisfied: packaging>=17.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from marshmallow<4.0.0,>=3.5->promptflow-devkit>=1.15.0->azure-ai-evaluation) (24.0)\n", - "Requirement already satisfied: portalocker<3,>=1.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msal-extensions>=1.2.0->azure-identity>=1.12.0->azure-ai-evaluation) (2.10.1)\n", - "Requirement already satisfied: deprecated>=1.2.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.2.14)\n", - "Requirement already satisfied: googleapis-common-protos~=1.52 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.65.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", - "Requirement already satisfied: opentelemetry-proto==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.27.0)\n", - "Requirement already satisfied: protobuf<5.0,>=3.19 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-proto==1.27.0->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (4.25.5)\n", - "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2024.2)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core>=1.30.2->azure-ai-evaluation) (3.3.2)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core>=1.30.2->azure-ai-evaluation) (2.2.1)\n", - "Requirement already satisfied: ruamel.yaml.clib>=0.2.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from ruamel.yaml<1.0.0,>=0.17.10->promptflow-core>=1.15.0->azure-ai-evaluation) (0.2.8)\n", - "Requirement already satisfied: greenlet!=0.4.17 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from sqlalchemy<3.0.0,>=1.4.48->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.1.1)\n", - "Requirement already satisfied: pycparser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity>=1.12.0->azure-ai-evaluation) (2.22)\n", - "Requirement already satisfied: wrapt<2,>=1.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from deprecated>=1.2.6->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (1.16.0)\n", - "Requirement already satisfied: smmap<6,>=3.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitdb<5,>=4.0.1->gitpython<4.0.0,>=3.1.24->promptflow-devkit>=1.15.0->azure-ai-evaluation) (5.0.1)\n", - "Requirement already satisfied: zipp>=0.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from importlib-metadata>=4.11.4->keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.20.2)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Jinja2>=3.1.2->flask<4.0.0,>=2.2.3->promptflow-core>=1.15.0->azure-ai-evaluation) (2.1.5)\n", - "Requirement already satisfied: isodate>=0.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msrest>=0.6.10->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (0.6.1)\n", - "Requirement already satisfied: requests-oauthlib>=0.5.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msrest>=0.6.10->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (2.0.0)\n", - "Requirement already satisfied: opentelemetry-semantic-conventions==0.48b0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-sdk<2.0.0,>=1.22.0->promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (0.48b0)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.23.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core>=1.15.0->azure-ai-evaluation) (2.23.4)\n", - "Requirement already satisfied: more-itertools in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jaraco.classes->keyring<25.0.0,>=24.2.0->promptflow-devkit>=1.15.0->azure-ai-evaluation) (10.5.0)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (1.9.0)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core>=1.15.0->azure-ai-evaluation) (0.5.0)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests-oauthlib>=0.5.0->msrest>=0.6.10->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit>=1.15.0->azure-ai-evaluation) (3.2.2)\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "[notice] A new release of pip is available: 24.0 -> 24.2\n", - "[notice] To update, run: C:\\Users\\sydneylister\\AppData\\Local\\Microsoft\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\python.exe -m pip install --upgrade pip\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n", - "Requirement already satisfied: promptflow-azure in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (1.15.1)\n", - "Requirement already satisfied: azure-ai-ml<2.0.0,>=1.14.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.20.0)\n", - "Requirement already satisfied: azure-core<2.0.0,>=1.26.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.31.0)\n", - "Requirement already satisfied: azure-cosmos<5.0.0,>=4.5.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (4.7.0)\n", - "Requirement already satisfied: azure-identity<2.0.0,>=1.12.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.18.0)\n", - "Requirement already satisfied: azure-storage-blob<13.0.0,>=12.17.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (12.23.0)\n", - "Requirement already satisfied: promptflow-devkit<2.0.0,>=1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (1.15.1)\n", - "Requirement already satisfied: pyjwt<3.0.0,>=2.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-azure) (2.9.0)\n", - "Requirement already satisfied: pyyaml>=5.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (6.0.1)\n", - "Requirement already satisfied: msrest>=0.6.18 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.7.1)\n", - "Requirement already satisfied: azure-mgmt-core>=1.3.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.4.0)\n", - "Requirement already satisfied: marshmallow>=3.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (3.22.0)\n", - "Requirement already satisfied: jsonschema>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.23.0)\n", - "Requirement already satisfied: tqdm in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.66.2)\n", - "Requirement already satisfied: strictyaml in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.7.3)\n", - "Requirement already satisfied: colorama in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.4.6)\n", - "Requirement already satisfied: azure-storage-file-share in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (12.18.0)\n", - "Requirement already satisfied: azure-storage-file-datalake>=12.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (12.17.0)\n", - "Requirement already satisfied: pydash>=6.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (7.0.7)\n", - "Requirement already satisfied: isodate in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.6.1)\n", - "Requirement already satisfied: azure-common>=1.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.1.28)\n", - "Requirement already satisfied: typing-extensions in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.12.2)\n", - "Requirement already satisfied: opencensus-ext-azure in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.1.13)\n", - "Requirement already satisfied: opencensus-ext-logging in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.1.1)\n", - "Requirement already satisfied: requests>=2.21.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core<2.0.0,>=1.26.4->promptflow-azure) (2.31.0)\n", - "Requirement already satisfied: six>=1.11.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core<2.0.0,>=1.26.4->promptflow-azure) (1.16.0)\n", - "Requirement already satisfied: cryptography>=2.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity<2.0.0,>=1.12.0->promptflow-azure) (43.0.1)\n", - "Requirement already satisfied: msal>=1.30.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity<2.0.0,>=1.12.0->promptflow-azure) (1.31.0)\n", - "Requirement already satisfied: msal-extensions>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-identity<2.0.0,>=1.12.0->promptflow-azure) (1.2.0)\n", - "Requirement already satisfied: argcomplete>=3.2.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.5.0)\n", - "Requirement already satisfied: azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.0.0b30)\n", - "Requirement already satisfied: filelock<4.0.0,>=3.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.13.1)\n", - "Requirement already satisfied: flask-cors<5.0.0,>=4.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.0.2)\n", - "Requirement already satisfied: flask-restx<2.0.0,>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.3.0)\n", - "Requirement already satisfied: gitpython<4.0.0,>=3.1.24 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.1.43)\n", - "Requirement already satisfied: httpx>=0.25.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.27.2)\n", - "Requirement already satisfied: keyring<25.0.0,>=24.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (24.3.1)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", - "Requirement already satisfied: pandas<3.0.0,>=1.5.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.2.3)\n", - "Requirement already satisfied: pillow<11.0.0,>=10.1.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (10.4.0)\n", - "Requirement already satisfied: promptflow-core<2.0.0,>=1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.15.1)\n", - "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.0.1)\n", - "Requirement already satisfied: pywin32 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (306)\n", - "Requirement already satisfied: sqlalchemy<3.0.0,>=1.4.48 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.0.35)\n", - "Requirement already satisfied: tabulate<1.0.0,>=0.9.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.9.0)\n", - "Requirement already satisfied: waitress<3.0.0,>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.1.2)\n", - "Requirement already satisfied: aiohttp>=3.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (3.10.6)\n", - "Requirement already satisfied: fixedint==0.1.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.1.6)\n", - "Requirement already satisfied: opentelemetry-api~=1.26 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", - "Requirement already satisfied: opentelemetry-sdk~=1.26 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", - "Requirement already satisfied: psutil~=5.9 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (5.9.8)\n", - "Requirement already satisfied: cffi>=1.12 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cryptography>=2.5->azure-identity<2.0.0,>=1.12.0->promptflow-azure) (1.17.1)\n", - "Requirement already satisfied: Flask>=0.9 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.0.3)\n", - "Requirement already satisfied: aniso8601>=0.82 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (9.0.1)\n", - "Requirement already satisfied: werkzeug!=2.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.0.4)\n", - "Requirement already satisfied: pytz in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2024.2)\n", - "Requirement already satisfied: importlib-resources in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (6.4.5)\n", - "Requirement already satisfied: gitdb<5,>=4.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitpython<4.0.0,>=3.1.24->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.0.11)\n", - "Requirement already satisfied: anyio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.6.0)\n", - "Requirement already satisfied: certifi in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2024.2.2)\n", - "Requirement already satisfied: httpcore==1.* in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.0.5)\n", - "Requirement already satisfied: idna in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.6)\n", - "Requirement already satisfied: sniffio in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.3.1)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from httpcore==1.*->httpx>=0.25.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.14.0)\n", - "Requirement already satisfied: attrs>=22.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (24.2.0)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2023.12.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jsonschema>=4.0.0->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.20.0)\n", - "Requirement already satisfied: jaraco.classes in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.4.0)\n", - "Requirement already satisfied: importlib-metadata>=4.11.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (8.4.0)\n", - "Requirement already satisfied: pywin32-ctypes>=0.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.2.3)\n", - "Requirement already satisfied: packaging>=17.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from marshmallow>=3.5->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (24.0)\n", - "Requirement already satisfied: portalocker<3,>=1.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msal-extensions>=1.2.0->azure-identity<2.0.0,>=1.12.0->promptflow-azure) (2.10.1)\n", - "Requirement already satisfied: requests-oauthlib>=0.5.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from msrest>=0.6.18->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2.0.0)\n", - "Requirement already satisfied: deprecated>=1.2.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.2.14)\n", - "Requirement already satisfied: googleapis-common-protos~=1.52 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.65.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", - "Requirement already satisfied: opentelemetry-proto==1.27.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.27.0)\n", - "Requirement already satisfied: protobuf<5.0,>=3.19 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-proto==1.27.0->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (4.25.5)\n", - "Requirement already satisfied: numpy>=1.23.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.26.4)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.9.0.post0)\n", - "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pandas<3.0.0,>=1.5.3->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2024.2)\n", - "Requirement already satisfied: docstring_parser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.16)\n", - "Requirement already satisfied: fastapi<1.0.0,>=0.109.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.115.0)\n", - "Requirement already satisfied: filetype>=1.2.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.2.0)\n", - "Requirement already satisfied: promptflow-tracing==1.15.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.15.1)\n", - "Requirement already satisfied: ruamel.yaml<1.0.0,>=0.17.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.18.6)\n", - "Requirement already satisfied: openai in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.47.1)\n", - "Requirement already satisfied: tiktoken>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.7.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core<2.0.0,>=1.26.4->promptflow-azure) (3.3.2)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests>=2.21.0->azure-core<2.0.0,>=1.26.4->promptflow-azure) (2.2.1)\n", - "Requirement already satisfied: greenlet!=0.4.17 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from sqlalchemy<3.0.0,>=1.4.48->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.1.1)\n", - "Requirement already satisfied: opencensus<1.0.0,>=0.11.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.11.4)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (2.4.0)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (1.3.1)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (1.4.1)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (6.1.0)\n", - "Requirement already satisfied: yarl<2.0,>=1.12.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from aiohttp>=3.0->azure-core[aio]>=1.30.0; extra == \"aio\"->azure-storage-blob[aio]<13.0.0,>=12.17.0->promptflow-azure) (1.12.1)\n", - "Requirement already satisfied: pycparser in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity<2.0.0,>=1.12.0->promptflow-azure) (2.22)\n", - "Requirement already satisfied: wrapt<2,>=1.10 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from deprecated>=1.2.6->opentelemetry-exporter-otlp-proto-http<2.0.0,>=1.22.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.16.0)\n", - "Requirement already satisfied: starlette<0.39.0,>=0.37.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.38.6)\n", - "Requirement already satisfied: pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.9.2)\n", - "Requirement already satisfied: Jinja2>=3.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.1.3)\n", - "Requirement already satisfied: itsdangerous>=2.1.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.2.0)\n", - "Requirement already satisfied: click>=8.1.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (8.1.7)\n", - "Requirement already satisfied: blinker>=1.6.2 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from Flask>=0.9->flask-cors<5.0.0,>=4.0.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.8.2)\n", - "Requirement already satisfied: smmap<6,>=3.0.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from gitdb<5,>=4.0.1->gitpython<4.0.0,>=3.1.24->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (5.0.1)\n", - "Requirement already satisfied: zipp>=0.5 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from importlib-metadata>=4.11.4->keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (3.20.2)\n", - "Requirement already satisfied: opencensus-context>=0.1.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.1.3)\n", - "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2.20.0)\n", - "Requirement already satisfied: opentelemetry-semantic-conventions==0.48b0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from opentelemetry-sdk~=1.26->azure-monitor-opentelemetry-exporter<2.0.0,>=1.0.0b21->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.48b0)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from requests-oauthlib>=0.5.0->msrest>=0.6.18->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (3.2.2)\n", - "Requirement already satisfied: ruamel.yaml.clib>=0.2.7 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from ruamel.yaml<1.0.0,>=0.17.10->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.2.8)\n", - "Requirement already satisfied: MarkupSafe>=2.1.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from werkzeug!=2.0.0->flask-restx<2.0.0,>=1.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.1.5)\n", - "Requirement already satisfied: more-itertools in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from jaraco.classes->keyring<25.0.0,>=24.2.0->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (10.5.0)\n", - "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (1.24.0)\n", - "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (2.35.0)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.23.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4->fastapi<1.0.0,>=0.109.0->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2.23.4)\n", - "Requirement already satisfied: regex>=2022.1.18 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from tiktoken>=0.4.0->promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (2023.12.25)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (1.9.0)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from openai->promptflow-tracing==1.15.1->promptflow-core<2.0.0,>=1.15.1->promptflow-devkit<2.0.0,>=1.15.1->promptflow-azure) (0.5.0)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (5.5.0)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.4.1)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (4.9)\n", - "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in c:\\users\\sydneylister\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.11_qbz5n2kfra8p0\\localcache\\local-packages\\python311\\site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus<1.0.0,>=0.11.4->opencensus-ext-azure->azure-ai-ml<2.0.0,>=1.14.0->promptflow-azure) (0.6.1)\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "[notice] A new release of pip is available: 24.0 -> 24.2\n", - "[notice] To update, run: C:\\Users\\sydneylister\\AppData\\Local\\Microsoft\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\python.exe -m pip install --upgrade pip\n" - ] - } - ], + "outputs": [], "source": [ "%pip install azure-ai-evaluation\n", "%pip install promptflow-azure" @@ -417,33 +151,9 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " question \\\n", - "0 What is the capital of France? \n", - "1 Which tent is the most waterproof? \n", - "2 Which camping table is the lightest? \n", - "3 How much does TrailWalker Hiking Shoes cost? \n", - "\n", - " context \\\n", - "0 France is the country in Europe. \n", - "1 #TrailMaster X4 Tent, price $250,## BrandOutdo... \n", - "2 #BaseCamp Folding Table, price $60,## BrandCam... \n", - "3 #TrailWalker Hiking Shoes, price $110## BrandT... \n", - "\n", - " ground_truth \n", - "0 Paris \n", - "1 The TrailMaster X4 tent has a rainfly waterpro... \n", - "2 The BaseCamp Folding Table has a weight of 15 lbs \n", - "3 The TrailWalker Hiking Shoes are priced at $110 \n" - ] - } - ], + "outputs": [], "source": [ "df = pd.read_json(\"data.jsonl\", lines=True)\n", "print(df.head())" @@ -491,97 +201,9 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[2024-09-26 16:53:21 -0700][promptflow][WARNING] - Found existing c:/azureai-samples/scenarios/evaluate-model-endpoints/flow.flex.yaml, will not respect it in runtime.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Prompt flow service has started...\n", - "You can view the traces in local from http://127.0.0.1:23333/v1.0/ui/traces/?#run=evaluate_model_endpoints_20240926_165321_352972\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[2024-09-26 16:53:28 -0700][promptflow._sdk._orchestrator.run_submitter][INFO] - Submitting run evaluate_model_endpoints_20240926_165321_352972, log path: C:\\Users\\sydneylister\\.promptflow\\.runs\\evaluate_model_endpoints_20240926_165321_352972\\logs.txt\n", - "[2024-09-26 16:53:47 -0700][promptflow._sdk._orchestrator.run_submitter][WARNING] - 4 out of 4 runs failed in batch run.\n", - " Please check out C:/Users/sydneylister/.promptflow/.runs/evaluate_model_endpoints_20240926_165321_352972 for more details.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2024-09-26 16:53:28 -0700 39604 execution.bulk INFO Current thread is not main thread, skip signal handler registration in BatchEngine.\n", - "2024-09-26 16:53:28 -0700 39604 execution WARNING Starting run without column mapping may lead to unexpected results. Please consult the following documentation for more information: https://aka.ms/pf/column-mapping\n", - "2024-09-26 16:53:28 -0700 39604 execution.bulk INFO Current system's available memory is 10048.08203125MB, memory consumption of current process is 372.0625MB, estimated available worker count is 10048.08203125/372.0625 = 27\n", - "2024-09-26 16:53:28 -0700 39604 execution.bulk INFO Set process count to 4 by taking the minimum value among the factors of {'default_worker_count': 4, 'row_count': 4, 'estimated_worker_count_based_on_memory_usage': 27}.\n", - "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-32)-Process id(43356)-Line number(0) start execution.\n", - "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-33)-Process id(3604)-Line number(1) start execution.\n", - "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-35)-Process id(8800)-Line number(2) start execution.\n", - "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-34)-Process id(21616)-Line number(3) start execution.\n", - "2024-09-26 16:53:37 -0700 39604 execution.bulk INFO Process name(SpawnProcess-32)-Process id(43356)-Line number(0) completed.\n", - "2024-09-26 16:53:38 -0700 39604 execution.bulk INFO Finished 1 / 4 lines.\n", - "2024-09-26 16:53:38 -0700 39604 execution.bulk INFO Average execution time for completed lines: 1.01 seconds. Estimated time for incomplete lines: 3.03 seconds.\n", - "2024-09-26 16:53:43 -0700 39604 execution.bulk INFO Process name(SpawnProcess-33)-Process id(3604)-Line number(1) completed.\n", - "2024-09-26 16:53:43 -0700 39604 execution.bulk INFO Finished 2 / 4 lines.\n", - "2024-09-26 16:53:43 -0700 39604 execution.bulk INFO Average execution time for completed lines: 3.03 seconds. Estimated time for incomplete lines: 6.06 seconds.\n", - "2024-09-26 16:53:44 -0700 39604 execution.bulk INFO Process name(SpawnProcess-34)-Process id(21616)-Line number(3) completed.\n", - "2024-09-26 16:53:45 -0700 39604 execution.bulk INFO Finished 3 / 4 lines.\n", - "2024-09-26 16:53:45 -0700 39604 execution.bulk INFO Average execution time for completed lines: 2.69 seconds. Estimated time for incomplete lines: 2.69 seconds.\n", - "2024-09-26 16:53:45 -0700 39604 execution.bulk INFO Process name(SpawnProcess-35)-Process id(8800)-Line number(2) completed.\n", - "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO Finished 4 / 4 lines.\n", - "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO Average execution time for completed lines: 2.27 seconds. Estimated time for incomplete lines: 0.0 seconds.\n", - "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [21616-SpawnProcess-34] will be terminated.\n", - "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [8800-SpawnProcess-35] will be terminated.\n", - "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [43356-SpawnProcess-32] will be terminated.\n", - "2024-09-26 16:53:46 -0700 39604 execution.bulk INFO The thread monitoring the process [3604-SpawnProcess-33] will be terminated.\n", - "2024-09-26 16:53:46 -0700 21616 execution.bulk INFO The process [21616] has received a terminate signal.\n", - "2024-09-26 16:53:46 -0700 8800 execution.bulk INFO The process [8800] has received a terminate signal.\n", - "2024-09-26 16:53:46 -0700 43356 execution.bulk INFO The process [43356] has received a terminate signal.\n", - "2024-09-26 16:53:46 -0700 3604 execution.bulk INFO The process [3604] has received a terminate signal.\n", - "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 43356 terminated.\n", - "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 3604 terminated.\n", - "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 21616 terminated.\n", - "2024-09-26 16:53:47 -0700 39604 execution.bulk INFO Process 8800 terminated.\n", - "2024-09-26 16:53:47 -0700 39604 execution ERROR 4/4 flow run failed, indexes: [0,1,2,3], exception of index 0: Execution failure in 'ModelEndpoints.__call__': (ConnectionError) HTTPSConnectionPool(host='ai-***.**.azure.com', port=443): Max retries exceeded with url: /openai/deployments/gpt-4/chat/completions?api-version=2023-03-15-preview (Caused by NameResolutionError(\": Failed to resolve 'ai-***.**.azure.com' ([Errno 11001] getaddrinfo failed)\"))\n", - "======= Run Summary =======\n", - "\n", - "Run name: \"evaluate_model_endpoints_20240926_165321_352972\"\n", - "Run status: \"Completed\"\n", - "Start time: \"2024-09-26 16:53:21.348453-07:00\"\n", - "Duration: \"0:00:26.215496\"\n", - "Output path: \"C:\\Users\\sydneylister\\.promptflow\\.runs\\evaluate_model_endpoints_20240926_165321_352972\"\n", - "\n" - ] - }, - { - "ename": "EvaluationException", - "evalue": "Missing required inputs for evaluator content_safety : ['response'].", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mEvaluationException\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[25], line 35\u001b[0m\n\u001b[0;32m 33\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m model \u001b[38;5;129;01min\u001b[39;00m models:\n\u001b[0;32m 34\u001b[0m randomNum \u001b[38;5;241m=\u001b[39m random\u001b[38;5;241m.\u001b[39mrandint(\u001b[38;5;241m1111\u001b[39m, \u001b[38;5;241m9999\u001b[39m)\n\u001b[1;32m---> 35\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 36\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# azure_ai_project=azure_ai_project,\u001b[39;49;00m\n\u001b[0;32m 37\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluation_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mEval-Run-\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mrandomNum\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m-\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtitle\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 38\u001b[0m \u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 39\u001b[0m \u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mModelEndpoints\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv_var\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 40\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluators\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\n\u001b[0;32m 41\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontent_safety\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontent_safety_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 42\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcoherence\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mcoherence_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 43\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrelevance\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mrelevance_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 44\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgroundedness\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroundedness_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 45\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfluency\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mfluency_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 46\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msimilarity\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43msimilarity_evaluator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 47\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 48\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluator_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\n\u001b[0;32m 49\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontent_safety\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 50\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcoherence\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 51\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrelevance\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.query}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 52\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgroundedness\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[0;32m 53\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m 54\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m 55\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m 56\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 57\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfluency\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 58\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msimilarity\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{target.response}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.context}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$\u001b[39;49m\u001b[38;5;132;43;01m{data.question}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 59\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 60\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:545\u001b[0m, in \u001b[0;36mevaluate\u001b[1;34m(evaluation_name, target, data, evaluators, evaluator_config, azure_ai_project, output_path, **kwargs)\u001b[0m\n\u001b[0;32m 531\u001b[0m error_message \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m 532\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe evaluation failed due to an error during multiprocess bootstrapping.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 533\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPlease ensure the evaluate API is properly guarded with the \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m block:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 534\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m if __name__ == \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 535\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m evaluate(...)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 536\u001b[0m )\n\u001b[0;32m 537\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m EvaluationException(\n\u001b[0;32m 538\u001b[0m message\u001b[38;5;241m=\u001b[39merror_message,\n\u001b[0;32m 539\u001b[0m internal_message\u001b[38;5;241m=\u001b[39merror_message,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 542\u001b[0m blame\u001b[38;5;241m=\u001b[39mErrorBlame\u001b[38;5;241m.\u001b[39mUNKNOWN,\n\u001b[0;32m 543\u001b[0m ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m--> 545\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:514\u001b[0m, in \u001b[0;36mevaluate\u001b[1;34m(evaluation_name, target, data, evaluators, evaluator_config, azure_ai_project, output_path, **kwargs)\u001b[0m\n\u001b[0;32m 448\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Evaluates target or data with built-in or custom evaluators. If both target and data are provided,\u001b[39;00m\n\u001b[0;32m 449\u001b[0m \u001b[38;5;124;03m data will be run through target function and then results will be evaluated.\u001b[39;00m\n\u001b[0;32m 450\u001b[0m \n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 511\u001b[0m \n\u001b[0;32m 512\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 513\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 514\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_evaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 515\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluation_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluation_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 516\u001b[0m \u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtarget\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 517\u001b[0m \u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 518\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluators\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluators\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 519\u001b[0m \u001b[43m \u001b[49m\u001b[43mevaluator_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluator_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 520\u001b[0m \u001b[43m \u001b[49m\u001b[43mazure_ai_project\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mazure_ai_project\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 521\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 522\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 523\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 524\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m 525\u001b[0m \u001b[38;5;66;03m# Handle multiprocess bootstrap error\u001b[39;00m\n\u001b[0;32m 526\u001b[0m bootstrap_error \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m 527\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn attempt has been made to start a new process before the\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 528\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcurrent process has finished its bootstrapping phase.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 529\u001b[0m )\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:605\u001b[0m, in \u001b[0;36m_evaluate\u001b[1;34m(evaluation_name, target, data, evaluators, evaluator_config, azure_ai_project, output_path, **kwargs)\u001b[0m\n\u001b[0;32m 601\u001b[0m evaluator_config[evaluator_name][col] \u001b[38;5;241m=\u001b[39m run_output \u001b[38;5;66;03m# pylint: disable=unnecessary-dict-index-lookup\u001b[39;00m\n\u001b[0;32m 603\u001b[0m \u001b[38;5;66;03m# After we have generated all columns we can check if we have\u001b[39;00m\n\u001b[0;32m 604\u001b[0m \u001b[38;5;66;03m# everything we need for evaluators.\u001b[39;00m\n\u001b[1;32m--> 605\u001b[0m \u001b[43m_validate_columns\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_data_df\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevaluators\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevaluator_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mevaluator_config\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 607\u001b[0m \u001b[38;5;66;03m# Batch Run\u001b[39;00m\n\u001b[0;32m 608\u001b[0m evaluators_info \u001b[38;5;241m=\u001b[39m {}\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:314\u001b[0m, in \u001b[0;36m_validate_columns\u001b[1;34m(df, evaluators, target, evaluator_config)\u001b[0m\n\u001b[0;32m 311\u001b[0m new_df \u001b[38;5;241m=\u001b[39m _apply_column_mapping(df, mapping_config)\n\u001b[0;32m 313\u001b[0m \u001b[38;5;66;03m# Validate input data for evaluator\u001b[39;00m\n\u001b[1;32m--> 314\u001b[0m \u001b[43m_validate_input_data_for_evaluator\u001b[49m\u001b[43m(\u001b[49m\u001b[43mevaluator\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevaluator_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnew_df\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\azure\\ai\\evaluation\\_evaluate\\_evaluate.py:165\u001b[0m, in \u001b[0;36m_validate_input_data_for_evaluator\u001b[1;34m(evaluator, evaluator_name, df_data, is_target_fn)\u001b[0m\n\u001b[0;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_target_fn:\n\u001b[0;32m 164\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMissing required inputs for evaluator \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mevaluator_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmissing_inputs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m--> 165\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m EvaluationException(\n\u001b[0;32m 166\u001b[0m message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[0;32m 167\u001b[0m internal_message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[0;32m 168\u001b[0m target\u001b[38;5;241m=\u001b[39mErrorTarget\u001b[38;5;241m.\u001b[39mEVALUATE,\n\u001b[0;32m 169\u001b[0m category\u001b[38;5;241m=\u001b[39mErrorCategory\u001b[38;5;241m.\u001b[39mMISSING_FIELD,\n\u001b[0;32m 170\u001b[0m blame\u001b[38;5;241m=\u001b[39mErrorBlame\u001b[38;5;241m.\u001b[39mUSER_ERROR,\n\u001b[0;32m 171\u001b[0m )\n\u001b[0;32m 172\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMissing required inputs for target : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmissing_inputs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 173\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m EvaluationException(\n\u001b[0;32m 174\u001b[0m message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[0;32m 175\u001b[0m internal_message\u001b[38;5;241m=\u001b[39mmsg,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 178\u001b[0m blame\u001b[38;5;241m=\u001b[39mErrorBlame\u001b[38;5;241m.\u001b[39mUSER_ERROR,\n\u001b[0;32m 179\u001b[0m )\n", - "\u001b[1;31mEvaluationException\u001b[0m: Missing required inputs for evaluator content_safety : ['response']." - ] - } - ], + "outputs": [], "source": [ "from app_target import ModelEndpoints\n", "import pathlib\n", From 2ccdfb21780b683febeedfc6d004cde7109d75dc Mon Sep 17 00:00:00 2001 From: Sydney Date: Fri, 27 Sep 2024 13:39:40 -0700 Subject: [PATCH 3/6] fix errors and remove 'question' col from data --- scenarios/evaluate-app-endpoint/askwiki.py | 14 ++++---- scenarios/evaluate-app-endpoint/data.jsonl | 6 ++-- .../evaluate-target.ipynb | 21 ++++++----- .../evaluate-model-endpoints/app_target.py | 2 +- .../evaluate-models-target.ipynb | 27 +++++++------- .../simulate_and_evaluate_ask_wiki.ipynb | 21 ++++++----- ...imulate_and_evaluate_online_endpoint.ipynb | 35 +++++++++---------- 7 files changed, 65 insertions(+), 61 deletions(-) diff --git a/scenarios/evaluate-app-endpoint/askwiki.py b/scenarios/evaluate-app-endpoint/askwiki.py index fe9523f0..203261b6 100644 --- a/scenarios/evaluate-app-endpoint/askwiki.py +++ b/scenarios/evaluate-app-endpoint/askwiki.py @@ -160,10 +160,10 @@ def format(doc: dict) -> str: # Function to perform augmented QA -def augemented_qa(question: str, context: str) -> str: +def augemented_qa(query: str, context: str) -> str: system_message = system_message_template.render(contexts=context) - messages = [{"role": "system", "content": system_message}, {"role": "user", "content": question}] + messages = [{"role": "system", "content": system_message}, {"role": "user", "content": query}] with AzureOpenAI( azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], @@ -181,17 +181,17 @@ def augemented_qa(question: str, context: str) -> str: class Response(TypedDict): - answer: str + response: str context: str -def ask_wiki(question: str) -> Response: - url_list = get_wiki_url(question, count=2) +def ask_wiki(query: str) -> Response: + url_list = get_wiki_url(query, count=2) search_result = search_result_from_url(url_list, count=10) context = process_search_result(search_result) - answer = augemented_qa(question, context) + response = augemented_qa(query, context) - return {"answer": answer, "context": str(context)} + return {"response": response, "context": str(context)} # Main function diff --git a/scenarios/evaluate-app-endpoint/data.jsonl b/scenarios/evaluate-app-endpoint/data.jsonl index b6ae357f..37f5c4ca 100644 --- a/scenarios/evaluate-app-endpoint/data.jsonl +++ b/scenarios/evaluate-app-endpoint/data.jsonl @@ -1,3 +1,3 @@ -{"query":"When was United Stated found ?", "response":"1776", "question":"When was United Stated found ?"} -{"query":"What is the capital of France?", "response":"Paris", "question":"What is the capital of France?"} -{"query":"Who is the best tennis player of all time ?", "response":"Roger Federer", "question": "Who is the best tennis player of all time ?"} \ No newline at end of file +{"query":"When was United Stated found ?", "response":"1776"} +{"query":"What is the capital of France?", "response":"Paris"} +{"query":"Who is the best tennis player of all time ?", "response":"Roger Federer"} \ No newline at end of file diff --git a/scenarios/evaluate-app-endpoint/evaluate-target.ipynb b/scenarios/evaluate-app-endpoint/evaluate-target.ipynb index 796bb70b..8bf00ad7 100644 --- a/scenarios/evaluate-app-endpoint/evaluate-target.ipynb +++ b/scenarios/evaluate-app-endpoint/evaluate-target.ipynb @@ -57,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "257fd898-7ef2-4d89-872e-da9e426aaf0b", "metadata": {}, "outputs": [], @@ -88,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "fbfc3a3b", "metadata": {}, "outputs": [], @@ -110,7 +110,7 @@ "source": [ "from askwiki import ask_wiki\n", "\n", - "ask_wiki(question=\"What is the capital of India?\")" + "ask_wiki(query=\"What is the capital of India?\")" ] }, { @@ -144,16 +144,16 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "665d0e98", "metadata": {}, "outputs": [], "source": [ "model_config = {\n", - " \"azure_endpoint\": os.environ.get(\"AZURE_OPENAI_ENDPOINT\"),\n", - " \"api_key\": os.environ.get(\"AZURE_OPENAI_KEY\"),\n", - " \"azure_deployment\": os.environ.get(\"AZURE_OPENAI_DEPLOYMENT\"),\n", - " }" + " \"azure_endpoint\": os.environ.get(\"AZURE_OPENAI_ENDPOINT\"),\n", + " \"api_key\": os.environ.get(\"AZURE_OPENAI_KEY\"),\n", + " \"azure_deployment\": os.environ.get(\"AZURE_OPENAI_DEPLOYMENT\"),\n", + "}" ] }, { @@ -176,7 +176,7 @@ "relevance_evaluator(\n", " response=\"New Delhi is Capital of India\",\n", " context=\"India is a country in South Asia.\",\n", - " query=\"What is the capital of India?\"\n", + " query=\"What is the capital of India?\",\n", ")" ] }, @@ -248,8 +248,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" + "pygments_lexer": "ipython3" } }, "nbformat": 4, diff --git a/scenarios/evaluate-model-endpoints/app_target.py b/scenarios/evaluate-model-endpoints/app_target.py index 516c6d87..759c03e8 100644 --- a/scenarios/evaluate-model-endpoints/app_target.py +++ b/scenarios/evaluate-model-endpoints/app_target.py @@ -45,7 +45,7 @@ def call_gpt4_endpoint(self: Self, question: str) -> Response: payload = {"messages": [{"role": "user", "content": question}], "max_tokens": 500} output = self.query(endpoint=endpoint, headers=headers, payload=payload) - answer = output["choices"][0]["message"]["content"] + answer = output["choices"][0]["message"]["content"] return {"query": question, "response": answer} def call_gpt35_turbo_endpoint(self: Self, question: str) -> Response: diff --git a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb index e6bcdb20..8fa44d27 100644 --- a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb +++ b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -75,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -117,16 +117,20 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "azure_ai_project = {\"subscription_id\": \"\", \"resource_group_name\": \"\", \"project_name\": \"\"}" + "azure_ai_project = {\n", + " \"subscription_id\": \"\",\n", + " \"resource_group_name\": \"\",\n", + " \"project_name\": \"\",\n", + "}" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -169,15 +173,15 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model_config = {\n", - " \"azure_endpoint\": os.environ.get(\"AZURE_OPENAI_ENDPOINT\"),\n", - " \"api_key\": os.environ.get(\"AZURE_OPENAI_KEY\"),\n", - " \"azure_deployment\": os.environ.get(\"AZURE_OPENAI_DEPLOYMENT\"),\n", - " }" + " \"azure_endpoint\": os.environ.get(\"AZURE_OPENAI_ENDPOINT\"),\n", + " \"api_key\": os.environ.get(\"AZURE_OPENAI_KEY\"),\n", + " \"azure_deployment\": os.environ.get(\"AZURE_OPENAI_DEPLOYMENT\"),\n", + "}" ] }, { @@ -307,8 +311,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" + "pygments_lexer": "ipython3" } }, "nbformat": 4, diff --git a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb index 8a807308..3302b63f 100644 --- a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb +++ b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/askwiki/simulate_and_evaluate_ask_wiki.ipynb @@ -48,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -77,16 +77,20 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "azure_ai_project = {\"subscription_id\": \"\", \"resource_group_name\": \"\", \"project_name\": \"\"}" + "azure_ai_project = {\n", + " \"subscription_id\": \"\",\n", + " \"resource_group_name\": \"\",\n", + " \"project_name\": \"\",\n", + "}" ] }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -145,7 +149,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -237,7 +241,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -299,7 +303,7 @@ "json_lines = outputs.to_eval_qa_json_lines()\n", "for line in json_lines.split(\"\\n\"):\n", " if line:\n", - " print(blocklist_evaluator(answer=json.loads(line)['response']))" + " print(blocklist_evaluator(answer=json.loads(line)[\"response\"]))" ] } ], @@ -318,8 +322,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" + "pygments_lexer": "ipython3" } }, "nbformat": 4, diff --git a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb index b5a30761..b4229ad3 100644 --- a/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb +++ b/scenarios/generate-synthetic-data/simulate-adversarial-interactions/promptflow-online-endpoint/simulate_and_evaluate_online_endpoint.ipynb @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -77,20 +77,24 @@ "metadata": {}, "outputs": [], "source": [ - "azure_ai_project = {\"subscription_id\": \"\", \"resource_group_name\": \"\", \"project_name\": \"\"}" + "azure_ai_project = {\n", + " \"subscription_id\": \"\",\n", + " \"resource_group_name\": \"\",\n", + " \"project_name\": \"\",\n", + "}" ] }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def call_endpoint(query: str) -> dict:\n", " data = {\"query\": query}\n", " body = json.dumps(data)\n", - " api_key=\"\"\n", - " endpoint=\"\"\n", + " api_key = \"\"\n", + " endpoint = \"\"\n", " azure_model_deployment = \"\"\n", "\n", " if not api_key:\n", @@ -128,7 +132,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -146,7 +150,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -160,19 +164,15 @@ " context: Optional[Dict[str, Any]] = None,\n", ") -> dict:\n", " messages_list = messages[\"messages\"]\n", - " # get last message\n", - " latest_message = messages_list[-1]\n", - " query = latest_message[\"content\"]\n", " context = None\n", " # we are formatting the response to follow the openAI chat protocol format\n", " formatted_response = {\n", " \"content\": \"I cannot answer\",\n", " \"role\": \"assistant\",\n", - " \"context\": {\n", - " },\n", + " \"context\": {},\n", " }\n", " messages[\"messages\"].append(formatted_response)\n", - " return {\"messages\": messages[\"messages\"], \"stream\": stream, \"session_state\": session_state, \"context\": context}" + " return {\"messages\": messages_list, \"stream\": stream, \"session_state\": session_state, \"context\": context}" ] }, { @@ -195,7 +195,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -214,7 +214,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -238,7 +238,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -308,8 +308,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" + "pygments_lexer": "ipython3" } }, "nbformat": 4, From fc46d6c8415ddfa8c8f574105be7ffae74117ead Mon Sep 17 00:00:00 2001 From: Sydney Date: Fri, 27 Sep 2024 13:51:50 -0700 Subject: [PATCH 4/6] small fix in evaluator config --- scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb index 8fa44d27..486a5405 100644 --- a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb +++ b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb @@ -258,7 +258,7 @@ " evaluator_config={\n", " \"content_safety\": {\"query\": \"${data.question}\", \"response\": \"${target.response}\"},\n", " \"coherence\": {\"response\": \"${target.response}\", \"query\": \"${data.question}\"},\n", - " \"relevance\": {\"response\": \"${target.query}\", \"context\": \"${data.context}\", \"query\": \"${data.question}\"},\n", + " \"relevance\": {\"response\": \"${target.response}\", \"context\": \"${data.context}\", \"query\": \"${data.question}\"},\n", " \"groundedness\": {\n", " \"response\": \"${target.response}\",\n", " \"context\": \"${data.context}\",\n", From bd0d912158111ffca22989a0f1bfd45a19cd961a Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Wed, 11 Dec 2024 17:25:55 -0800 Subject: [PATCH 5/6] init --- .../Simulate_Evaluate_ContentSafety/README.md | 56 ++++ .../Simulate_Evaluate_ContentSafety.ipynb | 255 ++++++++++++++++++ 2 files changed, 311 insertions(+) create mode 100644 scenarios/evaluate/Simulators/Simulate_Evaluate_ContentSafety/README.md create mode 100644 scenarios/evaluate/Simulators/Simulate_Evaluate_ContentSafety/Simulate_Evaluate_ContentSafety.ipynb diff --git a/scenarios/evaluate/Simulators/Simulate_Evaluate_ContentSafety/README.md b/scenarios/evaluate/Simulators/Simulate_Evaluate_ContentSafety/README.md new file mode 100644 index 00000000..93e0595a --- /dev/null +++ b/scenarios/evaluate/Simulators/Simulate_Evaluate_ContentSafety/README.md @@ -0,0 +1,56 @@ +--- +page_type: sample +languages: +- language1 +- language2 +products: +- ai-services +- azure-openai +description: Simulate and Evaluate Content Safety Harms +--- + +# Simulate and Evaluate Content Safety Harms + +## Overview + +This notebook walks through how to generate a multi-turn simulated conversation targeting a deployed AzureOpenAI model and then evaluate that conversation dataset for Content Safety harms. + +### Objective + +The main objective of this tutorial is to help users understand how to use the azure-ai-evaluation SDK to simulate a multi-turn conversation with an AI system and then evaluate that dataset on various safety metrics. By the end of this tutorial, you should be able to: + +- Use azure-ai-evaluation SDK to generate a multi-turn simulated conversation dataset +- Evaluate the generated dataset for Content Safety harms + +### Basic requirements + +To use Azure AI Safety Evaluation for different scenarios(simulation, annotation, etc..), you need an **Azure AI Project.** You should provide Azure AI project to run your safety evaluations or simulations with. First[create an Azure AI hub](https://learn.microsoft.com/en-us/azure/ai-studio/concepts/ai-resources)then [create an Azure AI project]( https://learn.microsoft.com/en-us/azure/ai-studio/how-to/create-projects?tabs=ai-studio).You **do not** need to provide your own LLM deployment as the Azure AI Safety Evaluation servicehosts adversarial models for both simulation and evaluation of harmful content and connects to it via your Azure AI project. Ensure that your Azure AI project is in one of the supported regions for your desiredevaluation metric + +#### Region support for evaluations + +| Region | Hate and unfairness, sexual, violent, self-harm, XPIA | Groundedness | Protected material | +| - | - | - | - | +|UK South | Will be deprecated 12/1/24| no | no | +|East US 2 | yes| yes | yes | +|Sweden Central | yes| yes | no| +|US North Central | yes| no | no | +|France Central | yes| no | no | +|SwitzerlandWest| yes | no |no| + +For built-in quality and performance metrics, connect your own deployment of LLMs and therefore youcan evaluate in any region your deployment is in. + +#### Region support for adversarial simulation + +| Region | Adversarial simulation | +| - | - | +|UK South | yes| +|East US 2 | yes| +|Sweden Central | yes| +|US North Central | yes| +|France Central | yes| + +### Programming Languages + +- Python + +### Estimated Runtime: 30 mins \ No newline at end of file diff --git a/scenarios/evaluate/Simulators/Simulate_Evaluate_ContentSafety/Simulate_Evaluate_ContentSafety.ipynb b/scenarios/evaluate/Simulators/Simulate_Evaluate_ContentSafety/Simulate_Evaluate_ContentSafety.ipynb new file mode 100644 index 00000000..00c3bafb --- /dev/null +++ b/scenarios/evaluate/Simulators/Simulate_Evaluate_ContentSafety/Simulate_Evaluate_ContentSafety.ipynb @@ -0,0 +1,255 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Simulating and Evaluating Multiturn Conversations for Content Harms\n", + "\n", + "## Objective\n", + "\n", + "This notebook walks through how to generate a simulated multi-turn conversation targeting a deployed AzureOpenAI model and then evaluate that test dataset for Content Safety harms. \n", + "\n", + "## Time\n", + "You should expect to spend about 30 minutes running this notebook. If you increase or decrease the number of simulated conversations, the time will vary accordingly.\n", + "\n", + "## Before you begin\n", + "\n", + "### Installation\n", + "Install the following packages required to execute this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install openai azure-ai-evaluation azure-identity promptflow-azure" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set the following environment variables for use in this notebook:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ[\"AZURE_DEPLOYMENT_NAME\"] = \"\"\n", + "os.environ[\"AZURE_API_VERSION\"] = \"\"\n", + "os.environ[\"AZURE_SUBSCRIPTION_ID\"] = \"\"\n", + "os.environ[\"AZURE_RESOURCE_GROUP\"] = \"\"\n", + "os.environ[\"AZURE_PROJECT_NAME\"] = \"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Configuration\n", + "The following simulator and evaluators require an Azure AI Studio project configuration and an Azure credential to use. \n", + "Your project configuration will be what is used to log your evaluation results in your project after the evaluation run is finished.\n", + "\n", + "For full region supportability, see [our documentation](https://learn.microsoft.com/azure/ai-studio/how-to/develop/flow-evaluate-sdk#built-in-evaluators)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n", + "from azure.ai.evaluation import evaluate\n", + "from azure.ai.evaluation import ContentSafetyEvaluator\n", + "from azure.ai.evaluation.simulator import AdversarialSimulator, AdversarialScenario\n", + "from openai import AzureOpenAI\n", + "\n", + "\n", + "azure_ai_project = {\n", + " \"subscription_id\": os.environ.get(\"AZURE_SUBSCRIPTION_ID\"),\n", + " \"resource_group_name\": os.environ.get(\"AZURE_RESOURCE_GROUP\"),\n", + " \"project_name\": os.environ.get(\"AZURE_PROJECT_NAME\"),\n", + "}\n", + "\n", + "credential = DefaultAzureCredential()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run this example\n", + "\n", + "To keep this notebook lightweight, let's create a dummy application that calls an AzureOpenAI model, such as GPT 4. When we are testing your application for certain safety metrics like Content Safety, it's important to have a way to automate a basic style of red-teaming to elicit behaviors from a simulated malicious user. We will use the `Simulator` class and this is how we will generate a synthetic test dataset against your application. Once we have the test dataset, we can evaluate them with our `ContentSafetyEvaluator` class.\n", + "\n", + "The `Simulator` needs a structured contract with your application in order to simulate conversations or other types of interactions with it. This is achieved via a callback function. This is the function you would rewrite to actually format the response from your generative AI application." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from typing import List, Dict, Optional\n", + "\n", + "\n", + "async def content_safety_callback(\n", + " messages: List[Dict], stream: bool = False, session_state: Optional[str] = None, context: Optional[Dict] = None\n", + ") -> dict:\n", + " deployment = os.environ.get(\"AZURE_DEPLOYMENT_NAME\")\n", + " endpoint = os.environ.get(\"AZURE_ENDPOINT\")\n", + " token_provider = get_bearer_token_provider(DefaultAzureCredential(), \"https://cognitiveservices.azure.com/.default\")\n", + " # Get a client handle for the model\n", + " client = AzureOpenAI(\n", + " azure_endpoint=endpoint,\n", + " api_version=os.environ.get(\"AZURE_API_VERSION\"),\n", + " azure_ad_token_provider=token_provider,\n", + " )\n", + " # Call the model\n", + " try:\n", + " completion = client.chat.completions.create(\n", + " model=deployment,\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": messages[\"messages\"][0][\"content\"],\n", + " }\n", + " ],\n", + " max_tokens=800,\n", + " temperature=0.7,\n", + " top_p=0.95,\n", + " frequency_penalty=0,\n", + " presence_penalty=0,\n", + " stop=None,\n", + " stream=False,\n", + " )\n", + " formatted_response = completion.to_dict()[\"choices\"][0][\"message\"]\n", + " except Exception:\n", + " formatted_response = {\n", + " \"content\": \"I don't know\",\n", + " \"role\": \"assistant\",\n", + " \"context\": {\"key\": {}},\n", + " }\n", + " messages[\"messages\"].append(formatted_response)\n", + " return {\n", + " \"messages\": messages[\"messages\"],\n", + " \"stream\": stream,\n", + " \"session_state\": session_state,\n", + " \"context\": context,\n", + " }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Testing your application for Content Safety\n", + "\n", + "When building your application, you want to test that Content Safety harms (i.e. Hate and unfairness, Sexual, Violent, Self-harm) are not being generated by your generative AI applications. The following example uses an `AdversarialSimulator` paired with a conversation scenario to prompt your model to respond with material that contains content safety harms." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "content_safety_simulator = AdversarialSimulator(azure_ai_project=azure_ai_project, credential=credential)\n", + "\n", + "content_safety_scenario = AdversarialScenario.ADVERSARIAL_CONVERSATION" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Below we explicitly request that the conversation has multiple turns between the User and Assistant. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "content_safety_outputs = await content_safety_simulator(\n", + " scenario=content_safety_scenario,\n", + " max_conversation_turns=5, # define the number of conversation turns\n", + " max_simulation_results=5, # define the number of simulation results\n", + " target=content_safety_callback, # define the target model callback\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from pathlib import Path\n", + "\n", + "with Path.open(\"adv_convo_eval.jsonl\", \"w\") as f:\n", + " for output in content_safety_outputs:\n", + " f.write(json.dumps({\"conversation\": output}))\n", + " f.write(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we have our dataset, we can evaluate it for Content Safety harms. The `ContentSafetyEvaluator` class can take in the dataset and detect whether your data contains harmful content. Let's use the `evaluate()` API to run the evaluation and log it to our Azure AI Studio Project." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cs_eval = ContentSafetyEvaluator(azure_ai_project=azure_ai_project, credential=credential)\n", + "\n", + "result = evaluate(\n", + " name=\"content-safety-conversation\",\n", + " data=\"adv_convo_eval.jsonl\",\n", + " evaluators={\"content_safety\": cs_eval},\n", + " # Optionally provide your AI Studio project information to track your evaluation results in your Azure AI Studio project\n", + " azure_ai_project=azure_ai_project,\n", + " # Optionally provide an output path to dump a json of metric summary, row level data and metric and studio URL\n", + " output_path=\"./content-safety-conversation_results.json\",\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 9ae1febd7f92bc0baf3fc69a2190237582fa5ef1 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Thu, 12 Dec 2024 10:48:54 -0800 Subject: [PATCH 6/6] updates --- .../Simulate_From_Azure_Search_Index.ipynb | 2 -- .../Simulators/Simulate_Evaluate_ContentSafety/README.md | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/Simulate_From_Azure_Search_Index.ipynb b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/Simulate_From_Azure_Search_Index.ipynb index 21cca016..99ebf5e4 100644 --- a/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/Simulate_From_Azure_Search_Index.ipynb +++ b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Azure_Search_Index/Simulate_From_Azure_Search_Index.ipynb @@ -18,7 +18,6 @@ "This tutorial uses the following Azure AI services:\n", "\n", "- Access to Azure OpenAI Service - you can apply for access [here](https://go.microsoft.com/fwlink/?linkid=2222006)\n", - "- An Azure AI Studio project - go to [aka.ms/azureaistudio](https://aka.ms/azureaistudio) to create a project\n", "- An Azure AI Search service - go to [aka.ms/azuresearch](https://aka.ms/azuresearch) to create a service " ] }, @@ -127,7 +126,6 @@ "source": [ "import os\n", "\n", - "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"\"\n", "os.environ[\"AZURE_OPENAI_DEPLOYMENT\"] = \"\"\n", "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"\"" ] diff --git a/scenarios/evaluate/Simulators/Simulate_Evaluate_ContentSafety/README.md b/scenarios/evaluate/Simulators/Simulate_Evaluate_ContentSafety/README.md index 93e0595a..72df6ae1 100644 --- a/scenarios/evaluate/Simulators/Simulate_Evaluate_ContentSafety/README.md +++ b/scenarios/evaluate/Simulators/Simulate_Evaluate_ContentSafety/README.md @@ -1,8 +1,7 @@ --- page_type: sample languages: -- language1 -- language2 +- python products: - ai-services - azure-openai