Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions capabilities/classification/evaluation/promptfooconfig.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,27 +7,27 @@ prompts:
- prompts.py:rag_chain_of_thought_classify

providers:
- id: anthropic:messages:claude-3-haiku-20240307
- id: anthropic:messages:claude-haiku-4-5
label: "Haiku: T-0.0"
config:
max_tokens: 4096
temperature: 0
- id: anthropic:messages:claude-3-haiku-20240307
- id: anthropic:messages:claude-haiku-4-5
label: "Haiku: T-0.2"
config:
max_tokens: 4096
temperature: 0.2
- id: anthropic:messages:claude-3-haiku-20240307
- id: anthropic:messages:claude-haiku-4-5
label: "Haiku: T-0.4"
config:
max_tokens: 4096
temperature: 0.4
- id: anthropic:messages:claude-3-haiku-20240307
- id: anthropic:messages:claude-haiku-4-5
label: "Haiku: T-0.6"
config:
max_tokens: 4096
temperature: 0.6
- id: anthropic:messages:claude-3-haiku-20240307
- id: anthropic:messages:claude-haiku-4-5
label: "Haiku: T-0.8"
config:
max_tokens: 4096
Expand Down
6 changes: 3 additions & 3 deletions capabilities/classification/guide.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,7 @@
" stop_sequences=[\"</category>\"], \n",
" max_tokens=4096, \n",
" temperature=0.0,\n",
" model=\"claude-3-haiku-20240307\"\n",
" model=\"claude-haiku-4-5\"\n",
" )\n",
" \n",
" # Extract the result from the response\n",
Expand Down Expand Up @@ -734,7 +734,7 @@
" stop_sequences=[\"</category>\"], \n",
" max_tokens=4096, \n",
" temperature=0.0,\n",
" model=\"claude-3-haiku-20240307\"\n",
" model=\"claude-haiku-4-5\"\n",
" )\n",
" \n",
" # Extract the result from the response\n",
Expand Down Expand Up @@ -846,7 +846,7 @@
" stop_sequences=[\"</category>\"], \n",
" max_tokens=4096, \n",
" temperature=0.0,\n",
" model=\"claude-3-haiku-20240307\"\n",
" model=\"claude-haiku-4-5\"\n",
" )\n",
" \n",
" # Extract the result from the response\n",
Expand Down
4 changes: 2 additions & 2 deletions capabilities/contextual-embeddings/guide.ipynb

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def evaluate_end_to_end(query, generated_answer, correct_answer):
client = Anthropic(api_key=os.environ.get('ANTHROPIC_API_KEY'))
try:
response = client.messages.create(
model="claude-3-5-sonnet-20241022",
model="claude-sonnet-4-5",
max_tokens=1500,
messages=[
{"role": "user", "content": prompt},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@ prompts:
- prompts.py:answer_query_level_three

providers:
- id: anthropic:messages:claude-3-haiku-20240307
- id: anthropic:messages:claude-haiku-4-5
label: "Haiku: T-0.0"
config:
max_tokens: 2500
temperature: 0

- id: anthropic:messages:claude-3-5-sonnet-20241022
- id: anthropic:messages:claude-sonnet-4-5
label: "3.5 Sonnet: T-0.0"
config:
max_tokens: 2500
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ def _rerank_results(query: str, results: List[Dict], k: int = 5) -> List[Dict]:
"""
try:
response = client.messages.create(
model="claude-3-haiku-20240307",
model="claude-haiku-4-5",
max_tokens=50,
messages=[{"role": "user", "content": prompt}, {"role": "assistant", "content": "<relevant_indices>"}],
temperature=0,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ def _rerank_results(query: str, results: List[Dict], k: int = 3) -> List[Dict]:
client = Anthropic(api_key=os.environ.get('ANTHROPIC_API_KEY'))
try:
response = client.messages.create(
model="claude-3-5-sonnet-20241022",
model="claude-sonnet-4-5",
max_tokens=50,
messages=[{"role": "user", "content": prompt}, {"role": "assistant", "content": "<relevant_indices>"}],
temperature=0,
Expand Down
12 changes: 6 additions & 6 deletions capabilities/retrieval_augmented_generation/guide.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@
" Answer the question now, and avoid providing preamble such as 'Here is the answer', etc\n",
" \"\"\"\n",
" response = client.messages.create(\n",
" model=\"claude-3-haiku-20240307\",\n",
" model=\"claude-haiku-4-5\",\n",
" max_tokens=2500,\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": prompt}\n",
Expand Down Expand Up @@ -643,7 +643,7 @@
" \n",
" try:\n",
" response = client.messages.create(\n",
" model=\"claude-3-5-sonnet-20241022\",\n",
" model=\"claude-sonnet-4-5\",\n",
" max_tokens=1500,\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": prompt},\n",
Expand Down Expand Up @@ -3264,7 +3264,7 @@
" \"\"\"\n",
"\n",
" response = client.messages.create(\n",
" model=\"claude-3-haiku-20240307\",\n",
" model=\"claude-haiku-4-5\",\n",
" max_tokens=150,\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": prompt}\n",
Expand Down Expand Up @@ -3461,7 +3461,7 @@
" Answer the question now, and avoid providing preamble such as 'Here is the answer', etc\n",
" \"\"\"\n",
" response = client.messages.create(\n",
" model=\"claude-3-haiku-20240307\",\n",
" model=\"claude-haiku-4-5\",\n",
" max_tokens=2500,\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": prompt}\n",
Expand Down Expand Up @@ -5975,7 +5975,7 @@
" \"\"\"\n",
" try:\n",
" response = client.messages.create(\n",
" model=\"claude-3-haiku-20240307\",\n",
" model=\"claude-haiku-4-5\",\n",
" max_tokens=50,\n",
" messages=[{\"role\": \"user\", \"content\": prompt}, {\"role\": \"assistant\", \"content\": \"<relevant_indices>\"}],\n",
" temperature=0,\n",
Expand Down Expand Up @@ -6044,7 +6044,7 @@
" Answer the question now, and avoid providing preamble such as 'Here is the answer', etc\n",
" \"\"\"\n",
" response = client.messages.create(\n",
" model=\"claude-3-haiku-20240307\",\n",
" model=\"claude-haiku-4-5\",\n",
" max_tokens=2500,\n",
" messages=[{\"role\": \"user\", \"content\": prompt}],\n",
" temperature=0\n",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def llm_eval(summary, input):
Evaluation (JSON format):"""

response = client.messages.create(
model="claude-3-5-sonnet-20241022",
model="claude-sonnet-4-5",
max_tokens=1000,
temperature=0,
messages=[
Expand Down
4 changes: 2 additions & 2 deletions capabilities/summarization/evaluation/promptfooconfig.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ prompts:
- prompts.py:summarize_long_document

providers:
- id: anthropic:messages:claude-3-haiku-20240307
- id: anthropic:messages:claude-haiku-4-5
label: "3.0 Haiku"
config:
max_tokens: 4096
temperature: 0
- id: anthropic:messages:claude-3-5-sonnet-20241022
- id: anthropic:messages:claude-sonnet-4-5
label: "3.5 Sonnet"
config:
max_tokens: 4096
Expand Down
18 changes: 9 additions & 9 deletions capabilities/summarization/guide.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@
" \"\"\"\n",
"\n",
" response = client.messages.create(\n",
" model=\"claude-3-5-sonnet-20241022\",\n",
" model=\"claude-sonnet-4-5\",\n",
" max_tokens=max_tokens,\n",
" system=\"You are a legal analyst known for highly accurate and detailed summaries of legal documents.\",\n",
" messages=[\n",
Expand Down Expand Up @@ -338,7 +338,7 @@
" \"\"\"\n",
"\n",
" response = client.messages.create(\n",
" model=\"claude-3-5-sonnet-20241022\",\n",
" model=\"claude-sonnet-4-5\",\n",
" max_tokens=max_tokens,\n",
" system=\"You are a legal analyst known for highly accurate and detailed summaries of legal documents.\",\n",
" messages=[\n",
Expand Down Expand Up @@ -463,7 +463,7 @@
" \"\"\"\n",
"\n",
" response = client.messages.create(\n",
" model=\"claude-3-5-sonnet-20241022\",\n",
" model=\"claude-sonnet-4-5\",\n",
" max_tokens=max_tokens,\n",
" system=\"You are a legal analyst known for highly accurate and detailed summaries of legal documents.\",\n",
" messages=[\n",
Expand Down Expand Up @@ -561,7 +561,7 @@
"metadata": {},
"outputs": [],
"source": [
"def guided_sublease_summary(text, model=\"claude-3-5-sonnet-20241022\", max_tokens=1000):\n",
"def guided_sublease_summary(text, model=\"claude-sonnet-4-5\", max_tokens=1000):\n",
"\n",
" # Prompt the model to summarize the sublease agreement\n",
" prompt = f\"\"\"Summarize the following sublease agreement. Focus on these key aspects:\n",
Expand Down Expand Up @@ -793,7 +793,7 @@
" # Iterate over chunks and summarize each one\n",
" # We use guided_legal_summary here, but you can use basic_summarize or any other summarization function\n",
" # Note that we'll also use haiku for the interim summaries, and the 3.5 sonnet for the final summary\n",
" chunk_summaries = [guided_sublease_summary(chunk, model=\"claude-3-haiku-20240307\", max_tokens=max_tokens) for chunk in chunks]\n",
" chunk_summaries = [guided_sublease_summary(chunk, model=\"claude-haiku-4-5\", max_tokens=max_tokens) for chunk in chunks]\n",
" \n",
" final_summary_prompt = f\"\"\"\n",
" \n",
Expand Down Expand Up @@ -821,7 +821,7 @@
" \"\"\"\n",
"\n",
" response = client.messages.create(\n",
" model=\"claude-3-5-sonnet-20241022\",\n",
" model=\"claude-sonnet-4-5\",\n",
" max_tokens=max_tokens,\n",
" system=\"You are a legal expert that summarizes notes on one document.\",\n",
" messages=[\n",
Expand Down Expand Up @@ -922,7 +922,7 @@
" \"\"\"\n",
"\n",
" response = client.messages.create(\n",
" model=\"claude-3-5-sonnet-20241022\",\n",
" model=\"claude-sonnet-4-5\",\n",
" max_tokens=500,\n",
" temperature=0.2,\n",
" messages=[\n",
Expand All @@ -944,7 +944,7 @@
" prompt=f\"Legal document summary: {summary}\\n\\nLegal query: {query}\\n\\nRate the relevance of this legal document to the query on a scale of 0 to 10. Only output the numeric value:\"\n",
"\n",
" response = client.messages.create(\n",
" model=\"claude-3-haiku-20240307\",\n",
" model=\"claude-haiku-4-5\",\n",
" max_tokens=2,\n",
" temperature=0,\n",
" messages=[\n",
Expand Down Expand Up @@ -974,7 +974,7 @@
" Relevant clauses or sections (separated by '---'):\"\"\"\n",
"\n",
" response = client.messages.create(\n",
" model=\"claude-3-5-sonnet-20241022\",\n",
" model=\"claude-sonnet-4-5\",\n",
" max_tokens=1000,\n",
" temperature=0,\n",
" messages=[\n",
Expand Down
4 changes: 2 additions & 2 deletions capabilities/text_to_sql/evaluation/promptfooconfig.yaml
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
providers:
- id: anthropic:messages:claude-3-haiku-20240307
- id: anthropic:messages:claude-haiku-4-5
label: "3 Haiku"
config:
max_tokens: 4096
temperature: 0
- id: anthropic:messages:claude-3-5-sonnet-20241022
- id: anthropic:messages:claude-sonnet-4-5
label: "3.5 Sonnet"
config:
max_tokens: 4096
Expand Down
2 changes: 1 addition & 1 deletion capabilities/text_to_sql/guide.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@
"\n",
"# Initialize the Anthropic client\n",
"client = Anthropic()\n",
"MODEL_NAME = \"claude-3-5-sonnet-20241022\"\n",
"MODEL_NAME = \"claude-sonnet-4-5\"\n",
"\n",
"# Filepath to the SQLite database\n",
"DATABASE_PATH = \"data/data.db\""
Expand Down
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"from dotenv import load_dotenv\n",
"from utils.agent_visualizer import print_activity\n",
"\n",
"from claude_code_sdk import ClaudeCodeOptions, ClaudeSDKClient, query\n",
"from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient, query\n",
"\n",
"load_dotenv()"
]
Expand Down Expand Up @@ -67,7 +67,7 @@
"messages = []\n",
"async for msg in query(\n",
" prompt=\"Research the latest trends in AI agents and give me a brief summary\",\n",
" options=ClaudeCodeOptions(model=\"claude-sonnet-4-20250514\", allowed_tools=[\"WebSearch\"]),\n",
" options=ClaudeAgentOptions(model=\"claude-sonnet-4-5\", allowed_tools=[\"WebSearch\"]),\n",
"):\n",
" print_activity(msg)\n",
" messages.append(msg)"
Expand Down Expand Up @@ -157,8 +157,8 @@
"source": [
"messages = []\n",
"async with ClaudeSDKClient(\n",
" options=ClaudeCodeOptions(\n",
" model=\"claude-sonnet-4-20250514\",\n",
" options=ClaudeAgentOptions(\n",
" model=\"claude-sonnet-4-5\",\n",
" cwd=\"research_agent\",\n",
" system_prompt=\"You are a research agent specialized in AI\",\n",
" allowed_tools=[\"WebSearch\", \"Read\"],\n",
Expand Down
Loading
Loading