We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent a58f374 commit f491499Copy full SHA for f491499
test/e2e/temp-eval-echo-test.yaml
@@ -0,0 +1,15 @@
1
+evals:
2
+ models: ['claude-3-haiku-20240307']
3
+ timeout: 30000
4
+ max_steps: 3
5
+ tests:
6
+ - name: 'Uses echo tool correctly via config mode'
7
+ prompt: 'Please echo the message "Hello from eval test"'
8
+ expected_tool_calls:
9
+ required: ['echo']
10
+ response_scorers:
11
+ - type: 'regex'
12
+ pattern: 'Hello from eval test'
13
+ - type: 'llm-judge'
14
+ criteria: 'Did the assistant successfully echo the requested message?'
15
+ threshold: 0.7
0 commit comments