Update pyproject.toml

amrit110 · amrit110 · commit b3dd6fe160c8 · 2025-12-03T11:44:39.000-05:00
diff --git a/README.md b/README.md
@@ -1,42 +1,113 @@
-# Implementation Template Repository
+# Agent Bootcamp
 
-This repository serves as the template for implementations created by the Vector AI
-Engineering team. It is designed to be used as a starting point for bootcamps, labs,
-or workshops.
+----------------------------------------------------------------------------------------
 
-## About [Implementation Name]
+This is a collection of reference implementations for Vector Institute's **Agentic AI Evaluation Bootcamp**.
 
-*Add info on the implementations.*
+## Reference Implementations
 
-## Repository Structure
+This repository includes several modules, each showcasing a different aspect of agent-based RAG systems:
 
-- **docs/**: Contains detailed documentation, additional resources, installation guides, and setup instructions that are not covered in this README.
-- **implementations/**: Implementations are organized by topics. Each topic has its own directory containing notebooks, and a README for guidance.
-- **pyproject.toml**: The `pyproject.toml` file in this repository configures various build system requirements and dependencies, centralizing project settings in a standardized format.
+**3. Evals: Automated Evaluation Pipelines**
+  Contains scripts and utilities for evaluating agent performance using LLM-as-a-judge and synthetic data generation. Includes tools for uploading datasets, running evaluations, and integrating with [Langfuse](https://langfuse.com/) for traceability.
 
-### Implementations Directory
+- **[3.1 LLM-as-a-Judge](src/3_evals/1_llm_judge/README.md)**
+  Automated evaluation pipelines using LLM-as-a-judge with Langfuse integration.
 
-Each topic within the [choice of bootcamp/lab/workshop] has a dedicated directory in the `implementations/` directory. In each directory, there is a README file that provides an overview of the topic, prerequisites, and notebook descriptions.
+- **[3.2 Evaluation on Synthetic Dataset](src/3_evals/2_synthetic_data/README.md)**
+  Showcases the generation of synthetic evaluation data for testing agents.
 
-Here is the list of the covered topics:
-- [Implementation 1]
-- [Implementation 2]
 
 ## Getting Started
 
-To get started with this bootcamp (*Change or modify the following steps based your needs.*):
-1. Clone this repository to your machine.
-2. *Include setup and installation instructions here. For additional documentation, refer to the `docs/` directory.*
-3. Begin with each topic in the `implementations/` directory, as guided by the README files.
+Set your API keys in `.env`. Use `.env.example` as a template.
 
-## License
-*Add appropriate LICENSE for this bootcamp in the main directory.*
-This project is licensed under the terms of the [LICENSE](LICENSE.md) file located in the root directory of this repository.
+```bash
+cp -v .env.example .env
+```
 
-## Contribution
-*Add appropriate CONTRIBUTING.md for this bootcamp in the main directory.*
-To get started with contributing to our project, please read our [CONTRIBUTING.md](CONTRIBUTING.md) guide.
+Run integration tests to validate that your API keys are set up correctly.
 
-## Contact Information
+```bash
+uv run --env-file .env pytest -sv tests/tool_tests/test_integration.py
+```
 
-For more information or help with navigating this repository, please contact [Vector AI ENG Team/Individual] at [Contact Email].
+## Reference Implementations
+
+For "Gradio App" reference implementations, running the script would print out a "public URL" ending in `gradio.live` (might take a few seconds to appear.) To access the gradio app with the full streaming capabilities, copy and paste this `gradio.live` URL into a new browser tab.
+
+For all reference implementations, to exit, press "Ctrl/Control-C" and wait up to ten seconds. If you are a Mac user, you should use "Control-C" and not "Command-C". Please note that by default, the gradio web app reloads automatically as you edit the Python script. There is no need to manually stop and restart the program each time you make some code changes.
+
+You might see warning messages like the following:
+
+```json
+ERROR:openai.agents:[non-fatal] Tracing client error 401: {
+  "error": {
+    "message": "Incorrect API key provided. You can find your API key at https://platform.openai.com/account/api-keys.",
+    "type": "invalid_request_error",
+    "param": null,
+    "code": "invalid_api_key"
+  }
+}
+```
+
+These warnings can be safely ignored, as they are the result of a bug in the upstream libraries. Your agent traces will be uploaded to LangFuse as configured.
+
+### 3. Evals
+
+Synthetic data.
+
+```bash
+uv run --env-file .env \
+-m src.3_evals.2_synthetic_data.synthesize_data \
+--source_dataset hf://vector-institute/hotpotqa@d997ecf:train \
+--langfuse_dataset_name search-dataset-synthetic-20250609 \
+--limit 18
+```
+
+Quantify embedding diversity of synthetic data
+
+```bash
+# Baseline: "Real" dataset
+uv run \
+--env-file .env \
+-m src.3_evals.2_synthetic_data.annotate_diversity \
+--langfuse_dataset_name search-dataset \
+--run_name cosine_similarity_bge_m3
+
+# Synthetic dataset
+uv run \
+--env-file .env \
+-m src.3_evals.2_synthetic_data.annotate_diversity \
+--langfuse_dataset_name search-dataset-synthetic-20250609 \
+--run_name cosine_similarity_bge_m3
+```
+
+Visualize embedding diversity of synthetic data
+
+```bash
+uv run \
+--env-file .env \
+gradio src/3_evals/2_synthetic_data/gradio_visualize_diversity.py
+```
+
+Run LLM-as-a-judge Evaluation on synthetic data
+
+```bash
+uv run \
+--env-file .env \
+-m src.3_evals.1_llm_judge.run_eval \
+--langfuse_dataset_name search-dataset-synthetic-20250609 \
+--run_name enwiki_weaviate \
+--limit 18
+```
+
+## Requirements
+
+- Python 3.12+
+- API keys as configured in `.env`.
+
+### Tidbit
+
+If you're curious about what "uv" stands for, it appears to have been more or
+less chosen [randomly](https://github.com/astral-sh/uv/issues/1349#issuecomment-1986451785).
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,60 +1,76 @@
 [project]
-name = "aieng-template-implementation"
+name = "agentic-ai-evaluation-bootcamp-202602"
 version = "0.1.0"
-description = ""
-authors = [ {name = "Vector AI Engineering", email = "ai_engineering@vectorinstitute.ai"}]
+description = "Vector Institute Agentic AI Evaluation Bootcamp 202602"
 readme = "README.md"
-repository = "https://github.com/VectorInstitute/aieng-template-implementation"
+authors = [ {name = "Vector AI Engineering", email = "ai_engineering@vectorinstitute.ai"}]
+license = "Apache-2.0"
 requires-python = ">=3.12"
 dependencies = [
-    "aieng-topic-impl",
-    "jupyterlab>=4.4.8",
-    "pip>=25.3",
+    "aiohttp>=3.12.14",
+    "beautifulsoup4>=4.13.4",
+    "datasets>=3.6.0",
+    "e2b-code-interpreter>=1.5.2",
+    "gradio>=5.37.0",
+    "langfuse>=3.1.3",
+    "lxml>=6.0.0",
+    "nest-asyncio>=1.6.0",
+    "numpy<2.3.0",
+    "openai>=1.93.1",
+    "openai-agents>=0.1.0",
+    "plotly>=6.2.0",
+    "pydantic>=2.11.7",
+    "pydantic-ai-slim[logfire]>=0.3.7",
+    "pytest-asyncio>=0.25.2",
+    "scikit-learn>=1.7.0",
+    "weaviate-client>=4.15.4",
 ]
 
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src"]
+
 [dependency-groups]
 dev = [
+    "aieng-platform-onboard>=0.3.6",
     "codecov>=2.1.13",
-    "mypy>=1.14.1",
+    "ipykernel>=6.29.5",
+    "ipython>=9.4.0",
+    "ipywidgets>=8.1.7",
+    "jupyter>=1.1.1",
+    "jupyterlab>=4.4.8",
     "nbqa>=1.9.1",
     "pip-audit>=2.7.3",
     "pre-commit>=4.1.0",
     "pytest>=8.3.4",
     "pytest-asyncio>=0.25.2",
     "pytest-cov>=6.0.0",
     "pytest-mock>=3.14.0",
-    "ruff>=0.9.2",
+    "ruff>=0.12.2",
+    "transformers>=4.54.1",
 ]
-
-[tool.uv.workspace]
-members = [
-  "aieng-topic-impl",
+docs = [
+    "jinja2>=3.1.6", # Pinning version to address vulnerability GHSA-cpwx-vrp4-4pq7
+    "mkdocs>=1.6.0",
+    "mkdocs-material>=9.6.15",
+    "mkdocstrings>=0.24.1",
+    "mkdocstrings-python>=1.16.12",
+    "ipykernel>=6.29.5",
+    "ipython>=9.4.0",
+]
+web-search = [
+    "google-cloud-firestore>=2.21.0",
+    "fastapi[standard]>=0.116.1",
+    "google-genai>=1.46.0",
+    "simplejson>=3.20.2",
 ]
 
-[tool.uv.sources]
-aieng-topic-impl = { workspace = true }
-
-[tool.mypy]
-ignore_missing_imports = true
-install_types = true
-pretty = true
-namespace_packages = true
-explicit_package_bases = true
-non_interactive = true
-warn_unused_configs = true
-allow_any_generics = false
-allow_subclassing_any = false
-allow_untyped_calls = false
-allow_untyped_defs = false
-allow_incomplete_defs = false
-check_untyped_defs = true
-allow_untyped_decorators = false
-warn_redundant_casts = true
-warn_unused_ignores = true
-warn_return_any = true
-implicit_reexport = false
-strict_equality = true
-extra_checks = true
+# Default dependency groups to be installed
+[tool.uv]
+default-groups = ["dev", "docs"]
 
 [tool.ruff]
 include = ["*.py", "pyproject.toml", "*.ipynb"]
@@ -94,11 +110,16 @@ ignore = [
     "PLR2004", # Replace magic number with named constant
     "PLR0913", # Too many arguments
     "COM812", # Missing trailing comma
+    "N999", # Number in module names.
+    "ERA001", # Commented out lines.
 ]
 
 # Ignore import violations in all `__init__.py` files.
+# Ignore missing docstring in `__init__.py` files
 [tool.ruff.lint.per-file-ignores]
-"__init__.py" = ["E402", "F401", "F403", "F811"]
+"__init__.py" = ["E402", "F401", "F403", "F811", "D104"]
+
+
 
 [tool.ruff.lint.pep8-naming]
 ignore-names = ["X*", "setUp"]
@@ -116,3 +137,8 @@ max-doc-length = 88
 markers = [
     "integration_test: marks tests as integration tests",
 ]
+
+[tool.coverage]
+    [tool.coverage.run]
+    source=["src/"]
+    omit=["tests/*", "*__init__.py"]
diff --git a/uv.lock b/uv.lock
