diff --git a/README.md b/README.md index 3e94bc5..503fb58 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ ## About The Project -Tonic Validate is a framework for the evaluation of LLM outputs, such as Retrieval Augmented Generation (RAG) pipelines. Validate makes it easy to evaluate, track, and monitor your LLM and RAG applications. Validate allows you to evaluate your LLM outputs through the use of our provided metrics which measure everything from answer correctness to LLM hallucination. Additionally, Validate has an optional UI to visualize your evaluation results for easy tracking and monitoring. +Tonic Validate is a framework for the evaluation of LLM outputs, such as Retrieval Augmented Generation (RAG) pipelines. Validate makes it easy to evaluate, track, and monitor your LLM and RAG applications. Validate allows you to evaluate your LLM outputs through the use of our provided metrics which measure everything from answer correctness to LLM hallucination. ## Preparing Data for RAG @@ -497,7 +497,7 @@ Tonic Validate collects minimal telemetry to help us figure out what users want We do **NOT** track things such as the contents of the questions / answers, your scores, or any other sensitive information. For detecting CI/CD, we only check for common environment variables in different CI/CD environments. We do not log the values of these environment variables. -We also generate a random UUID to help us figure out how many users are using the product. This UUID is linked to your Validate account only to help track who is using the SDK and UI at once and to get user counts. +We also generate a random UUID to help us figure out how many users are using the product. We also collect information on version of python being used and characteristics of the machine (e.g. calls to `platform.system()` and `platform.machine()`). This information is sent to Scarf which helps us better understand our open-source community. diff --git a/docs/source/api_reference/validate_api.rst b/docs/source/api_reference/validate_api.rst deleted file mode 100644 index 3d88082..0000000 --- a/docs/source/api_reference/validate_api.rst +++ /dev/null @@ -1,6 +0,0 @@ -Validate Api -============ - -.. automodule:: tonic_validate.validate_api - :members: - :undoc-members: diff --git a/pyproject.toml b/pyproject.toml index 6d3a8ec..557c6ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tonic-validate" -version = "6.2.0" +version = "7.0.0" description = "RAG evaluation metrics." authors = ["Joe Ferrara ", "Ethan Philpott ", "Adam Kamor "] readme = "README.md" diff --git a/tonic_validate/validate_api.py b/tonic_validate/validate_api.py index ae0ab7b..5a7f2c5 100644 --- a/tonic_validate/validate_api.py +++ b/tonic_validate/validate_api.py @@ -2,11 +2,7 @@ from pydantic import ConfigDict, validate_call from tonic_validate.classes.benchmark import Benchmark from tonic_validate.classes.run import Run -from tonic_validate.config import Config - -from tonic_validate.utils.http_client import HttpClient -from tonic_validate.utils.telemetry import Telemetry - +from warnings import warn class ValidateApi: @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) @@ -22,21 +18,7 @@ def __init__( api_key : str The access token for the Tonic Validate application. Obtained from the web application """ - self.config = Config() - if api_key is None: - api_key = self.config.TONIC_VALIDATE_API_KEY - if api_key is None: - exception_message = ( - "No api key provided. Please provide an api key or set " - "TONIC_VALIDATE_API_KEY environment variable." - ) - raise Exception(exception_message) - self.client = HttpClient(self.config.TONIC_VALIDATE_BASE_URL, api_key) - try: - telemetry = Telemetry(api_key) - telemetry.link_user() - except Exception as _: - pass + warn("As of 2025-07-30 the Validate API has been turned off and can no longer be used.") @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def upload_run( @@ -46,7 +28,7 @@ def upload_run( run_metadata: Optional[Dict[str, Any]] = {}, tags: Optional[List[str]] = [], ) -> str: - """Upload a run to a Tonic Validate project. + """(This function is deprecated) Upload a run to a Tonic Validate project. Parameters ---------- @@ -60,21 +42,11 @@ def upload_run( tags : Optional[List[str]] A list of tags which can be used to identify this run. Tags will be rendered in the UI and can also make run searchable. """ - if run_metadata and "llm_evaluator" not in run_metadata: - run_metadata["llm_evaluator"] = run.llm_evaluator - run_response = self.client.http_post( - f"/projects/{project_id}/runs/with_data", - data={ - "run_metadata": run_metadata, - "tags": tags, - "data": [run_data.to_dict() for run_data in run.run_data], - }, - ) - return run_response["id"] + warn("As of 2025-07-30 the Validate API has been turned off and can no longer be used.") @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def get_benchmark(self, benchmark_id: str) -> Benchmark: - """Get a Tonic Validate benchmark by its ID. + """(This function is deprecated) Get a Tonic Validate benchmark by its ID. Get a benchmark to create a new project with that benchmark. @@ -83,20 +55,11 @@ def get_benchmark(self, benchmark_id: str) -> Benchmark: benchmark_id : str The ID of the benchmark. """ - benchmark_response = self.client.http_get(f"/benchmarks/{benchmark_id}") - benchmark_items_response = self.client.http_get( - f"/benchmarks/{benchmark_id}/items" - ) - questions: List[str] = [] - answers: List[str] = [] - for benchmark_item_response in benchmark_items_response: - questions += [benchmark_item_response["question"]] - answers += [benchmark_item_response["answer"]] - return Benchmark(questions, answers, benchmark_response["name"]) + warn("As of 2025-07-30 the Validate API has been turned off and can no longer be used.") @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def new_benchmark(self, benchmark: Benchmark, benchmark_name: str) -> str: - """Create a new Tonic Validate benchmark. + """(This function is deprecated) Create a new Tonic Validate benchmark. Parameters ---------- @@ -105,15 +68,4 @@ def new_benchmark(self, benchmark: Benchmark, benchmark_name: str) -> str: benchmark_name : str The name of the benchmark. """ - benchmark_response = self.client.http_post( - "/benchmarks", data={"name": benchmark_name} - ) - for benchmark_item in benchmark.items: - _ = self.client.http_post( - f"/benchmarks/{benchmark_response['id']}/items", - data={ - "question": benchmark_item.question, - "answer": benchmark_item.answer, - }, - ) - return benchmark_response["id"] + warn("As of 2025-07-30 the Validate API has been turned off and can no longer be used.")