Skip to content

Commit 483f175

Browse files
sfierrosfierro
authored
Merge pull request #956 from Kiln-AI/sfierro/specs_block_copilot_tools
specs block use copilot button when default run config has tools
2 parents f6aa8aa + 7c2f879 commit 483f175

File tree

3 files changed

+198
-91
lines changed

3 files changed

+198
-91
lines changed

‎app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/select_workflow/+page.svelte‎

Lines changed: 132 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,36 @@
11
<script lang="ts">
22
import { goto } from "$app/navigation"
33
import { page } from "$app/stores"
4+
import { onMount } from "svelte"
45
import AppPage from "../../../../app_page.svelte"
6+
import { load_task } from "$lib/stores"
7+
import { checkDefaultRunConfigHasTools } from "../spec_utils"
8+
import { createKilnError, type KilnError } from "$lib/utils/error_handlers"
59
610
$: project_id = $page.params.project_id
711
$: task_id = $page.params.task_id
812
13+
let loading = true
14+
let default_run_config_has_tools = false
15+
let error: KilnError | null = null
16+
17+
onMount(async () => {
18+
try {
19+
const task = await load_task(project_id, task_id)
20+
if (!task) {
21+
throw new Error("Failed to load task")
22+
}
23+
default_run_config_has_tools = await checkDefaultRunConfigHasTools(
24+
project_id,
25+
task,
26+
)
27+
} catch (e) {
28+
error = createKilnError(e)
29+
} finally {
30+
loading = false
31+
}
32+
})
33+
934
function proceed_to_select_template() {
1035
goto(`/specs/${project_id}/${task_id}/select_template`)
1136
}
@@ -24,104 +49,123 @@
2449
},
2550
]}
2651
>
27-
<div class="my-4 max-w-[680px]">
28-
<div class="overflow-x-auto border-b">
29-
<table class="table table-fixed w-full">
52+
{#if loading}
53+
<div class="w-full min-h-[50vh] flex justify-center items-center">
54+
<div class="loading loading-spinner loading-lg"></div>
55+
</div>
56+
{:else if error}
57+
<div class="text-error text-sm">
58+
{error.getMessage() || "An unknown error occurred"}
59+
</div>
60+
{:else}
61+
<div class="my-4 max-w-[680px]">
62+
<div class="overflow-x-auto border-b">
63+
<table class="table table-fixed w-full">
64+
<colgroup>
65+
<col class="w-[240px]" />
66+
<col />
67+
<col />
68+
</colgroup>
69+
<thead>
70+
<tr class="border-b-0">
71+
<th></th>
72+
<th class="text-center text-lg">Manual</th>
73+
<th class="text-center text-lg">
74+
<div class="flex items-center justify-center gap-2">
75+
<img
76+
src="/images/animated_logo.svg"
77+
alt="Kiln Copilot"
78+
class="size-4"
79+
/>
80+
<span>Kiln Copilot</span>
81+
</div>
82+
</th>
83+
</tr>
84+
</thead>
85+
<tbody>
86+
<tr>
87+
<th class="font-bold text-xs text-gray-500"
88+
>Eval Judge Creation</th
89+
>
90+
<td class="text-center">Manual</td>
91+
<td class="text-center border-l">Automatic</td>
92+
</tr>
93+
<tr>
94+
<th class="font-bold text-xs text-gray-500"
95+
>Edge Case Discovery</th
96+
>
97+
<td class="text-center">Manual</td>
98+
<td class="text-center border-l">Automatic</td>
99+
</tr>
100+
<tr>
101+
<th class="font-bold text-xs text-gray-500"
102+
>Eval Data Creation</th
103+
>
104+
<td class="text-center">Manual</td>
105+
<td class="text-center border-l">Automatic</td>
106+
</tr>
107+
<tr>
108+
<th class="font-bold text-xs text-base-content/60"
109+
>Eval Accuracy</th
110+
>
111+
<td class="text-center">Varies</td>
112+
<td class="text-center border-l">High</td>
113+
</tr>
114+
<tr>
115+
<th class="font-bold text-xs text-gray-500">Approx. Effort</th>
116+
<td class="text-center">20 min</td>
117+
<td class="text-center border-l">3 min</td>
118+
</tr>
119+
<tr>
120+
<th class="font-bold text-xs text-base-content/60"
121+
>Kiln Account</th
122+
>
123+
<td class="text-center">Optional</td>
124+
<td class="text-center border-l">Required</td>
125+
</tr>
126+
</tbody>
127+
</table>
128+
</div>
129+
<table class="table-fixed w-full mt-4">
30130
<colgroup>
31131
<col class="w-[240px]" />
32132
<col />
33133
<col />
34134
</colgroup>
35-
<thead>
36-
<tr class="border-b-0">
37-
<th></th>
38-
<th class="text-center text-lg">Manual</th>
39-
<th class="text-center text-lg">
40-
<div class="flex items-center justify-center gap-2">
41-
<img
42-
src="/images/animated_logo.svg"
43-
alt="Kiln Copilot"
44-
class="size-4"
45-
/>
46-
<span>Kiln Copilot</span>
47-
</div>
48-
</th>
49-
</tr>
50-
</thead>
51135
<tbody>
52136
<tr>
53-
<th class="font-bold text-xs text-gray-500"
54-
>Eval Judge Creation</th
55-
>
56-
<td class="text-center">Manual</td>
57-
<td class="text-center border-l">Automatic</td>
58-
</tr>
59-
<tr>
60-
<th class="font-bold text-xs text-gray-500"
61-
>Edge Case Discovery</th
62-
>
63-
<td class="text-center">Manual</td>
64-
<td class="text-center border-l">Automatic</td>
65-
</tr>
66-
<tr>
67-
<th class="font-bold text-xs text-gray-500">Eval Data Creation</th
68-
>
69-
<td class="text-center">Manual</td>
70-
<td class="text-center border-l">Automatic</td>
71-
</tr>
72-
<tr>
73-
<th class="font-bold text-xs text-base-content/60"
74-
>Eval Accuracy</th
75-
>
76-
<td class="text-center">Varies</td>
77-
<td class="text-center border-l">High</td>
78-
</tr>
79-
<tr>
80-
<th class="font-bold text-xs text-gray-500">Approx. Effort</th>
81-
<td class="text-center">20 min</td>
82-
<td class="text-center border-l">3 min</td>
83-
</tr>
84-
<tr>
85-
<th class="font-bold text-xs text-base-content/60"
86-
>Kiln Account</th
87-
>
88-
<td class="text-center">Optional</td>
89-
<td class="text-center border-l">Required</td>
137+
<td></td>
138+
<td class="text-center">
139+
<button
140+
class="btn btn-outline btn-sm"
141+
on:click={proceed_to_select_template}
142+
>
143+
Create Manually
144+
</button>
145+
</td>
146+
<td class="text-center">
147+
<div
148+
class="tooltip"
149+
data-tip={default_run_config_has_tools
150+
? "Tool calling is not yet supported in Kiln Copilot. Please create the spec manually for now."
151+
: undefined}
152+
>
153+
<button
154+
class="btn btn-primary btn-sm"
155+
disabled={loading || default_run_config_has_tools}
156+
on:click={() =>
157+
goto(`/specs/copilot_auth`, {
158+
replaceState: true,
159+
})}
160+
>
161+
Connect Kiln Copilot
162+
</button>
163+
</div>
164+
</td>
90165
</tr>
91166
</tbody>
92167
</table>
93168
</div>
94-
<table class="table-fixed w-full mt-4">
95-
<colgroup>
96-
<col class="w-[240px]" />
97-
<col />
98-
<col />
99-
</colgroup>
100-
<tbody>
101-
<tr>
102-
<td></td>
103-
<td class="text-center">
104-
<button
105-
class="btn btn-outline btn-sm"
106-
on:click={proceed_to_select_template}
107-
>
108-
Create Manually
109-
</button>
110-
</td>
111-
<td class="text-center">
112-
<button
113-
class="btn btn-primary btn-sm"
114-
on:click={() =>
115-
goto(`/specs/copilot_auth`, {
116-
replaceState: true,
117-
})}
118-
>
119-
Connect Kiln Copilot
120-
</button>
121-
</td>
122-
</tr>
123-
</tbody>
124-
</table>
125-
</div>
169+
{/if}
126170
</AppPage>
127171
</div>

‎app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_builder/+page.svelte‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@
77
import type { SpecType, ModelProviderName, Task } from "$lib/types"
88
import { goto } from "$app/navigation"
99
import { spec_field_configs } from "../select_template/spec_templates"
10-
import { checkKilnCopilotAvailable, buildSpecDefinition } from "../spec_utils"
10+
import {
11+
checkKilnCopilotAvailable,
12+
checkDefaultRunConfigHasTools,
13+
buildSpecDefinition,
14+
} from "../spec_utils"
1115
import {
1216
createSpec,
1317
type JudgeInfo,
@@ -51,6 +55,7 @@
5155
5256
// Copilot availability
5357
let has_kiln_copilot = false
58+
let default_run_config_has_tools = false
5459
5560
// Task data (loaded once in initialize)
5661
let task: Task | null = null
@@ -123,8 +128,12 @@
123128
$: if (is_tool_use_spec) evaluate_full_trace = true
124129
125130
// Tool call and RAG specs don't support copilot
131+
// Also disable copilot when the default run config has tools (tool calling not supported yet)
126132
$: copilot_enabled =
127-
has_kiln_copilot && !is_tool_use_spec && !is_reference_answer_spec
133+
has_kiln_copilot &&
134+
!is_tool_use_spec &&
135+
!is_reference_answer_spec &&
136+
!default_run_config_has_tools
128137
129138
// Initialize form from URL params
130139
async function initialize() {
@@ -144,6 +153,12 @@
144153
throw new Error("Failed to load task")
145154
}
146155
156+
// Check if default run config has tools (copilot doesn't support tool calling)
157+
default_run_config_has_tools = await checkDefaultRunConfigHasTools(
158+
project_id,
159+
task,
160+
)
161+
147162
// Get spec type from URL params
148163
const spec_type_param = $page.url.searchParams.get("type")
149164
if (!spec_type_param) {

‎app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_utils.ts‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,18 @@
11
import { client } from "$lib/api_client"
2-
import type { Spec, SpecStatus, SpecType } from "$lib/types"
2+
import type {
3+
Spec,
4+
SpecStatus,
5+
SpecType,
6+
Task,
7+
TaskRunConfig,
8+
} from "$lib/types"
39
import { spec_field_configs } from "./select_template/spec_templates"
10+
import {
11+
load_task_run_configs,
12+
run_configs_by_task_composite_id,
13+
} from "$lib/stores/run_configs_store"
14+
import { get_task_composite_id } from "$lib/stores"
15+
import { get } from "svelte/store"
416

517
/**
618
* Build a definition string from properties
@@ -41,6 +53,42 @@ export async function checkKilnCopilotAvailable(): Promise<boolean> {
4153
return !!data["kiln_copilot_api_key"]
4254
}
4355

56+
/**
57+
* Check if the task's default run config has any tools configured
58+
* @param project_id - The project ID
59+
* @param task - The task to check
60+
* @returns true if the default run config has tools, false otherwise
61+
*/
62+
export async function checkDefaultRunConfigHasTools(
63+
project_id: string,
64+
task: Task,
65+
): Promise<boolean> {
66+
if (!task.id) {
67+
throw new Error("Task ID is required")
68+
}
69+
70+
if (!task.default_run_config_id) {
71+
return false
72+
}
73+
74+
await load_task_run_configs(project_id, task.id)
75+
const run_configs =
76+
get(run_configs_by_task_composite_id)[
77+
get_task_composite_id(project_id, task.id)
78+
] ?? []
79+
80+
const default_config = run_configs.find(
81+
(config: TaskRunConfig) => config.id === task.default_run_config_id,
82+
)
83+
84+
if (!default_config) {
85+
return false
86+
}
87+
88+
const tools = default_config.run_config_properties?.tools_config?.tools ?? []
89+
return tools.length > 0
90+
}
91+
4492
/**
4593
* Update a spec's priority via the API
4694
* @param project_id - The project ID

0 commit comments

Comments
 (0)