Skip to content

specs block use copilot button when default run config has tools #956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sfierro merged 2 commits into from
Jan 21, 2026
Merged

specs block use copilot button when default run config has tools #956

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7c5050c
block copilot workflow button when deafult run config has tools
sfierro Jan 21, 2026
7c2f879
feedback
sfierro Jan 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
220 changes: 132 additions & 88 deletions app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/select_workflow/+page.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,36 @@
<script lang="ts">
import { goto } from "$app/navigation"
import { page } from "$app/stores"
import { onMount } from "svelte"
import AppPage from "../../../../app_page.svelte"
import { load_task } from "$lib/stores"
import { checkDefaultRunConfigHasTools } from "../spec_utils"
import { createKilnError, type KilnError } from "$lib/utils/error_handlers"

$: project_id = $page.params.project_id
$: task_id = $page.params.task_id

let loading = true
let default_run_config_has_tools = false
let error: KilnError | null = null

onMount(async () => {
try {
const task = await load_task(project_id, task_id)
if (!task) {
throw new Error("Failed to load task")
}
default_run_config_has_tools = await checkDefaultRunConfigHasTools(
project_id,
task,
)
} catch (e) {
error = createKilnError(e)
} finally {
loading = false
}
})

function proceed_to_select_template() {
goto(`/specs/${project_id}/${task_id}/select_template`)
}
Expand All @@ -24,104 +49,123 @@
},
]}
>
<div class="my-4 max-w-[680px]">
<div class="overflow-x-auto border-b">
<table class="table table-fixed w-full">
{#if loading}
<div class="w-full min-h-[50vh] flex justify-center items-center">
<div class="loading loading-spinner loading-lg"></div>
</div>
{:else if error}
<div class="text-error text-sm">
{error.getMessage() || "An unknown error occurred"}
</div>
{:else}
<div class="my-4 max-w-[680px]">
<div class="overflow-x-auto border-b">
<table class="table table-fixed w-full">
<colgroup>
<col class="w-[240px]" />
<col />
<col />
</colgroup>
<thead>
<tr class="border-b-0">
<th></th>
<th class="text-center text-lg">Manual</th>
<th class="text-center text-lg">
<div class="flex items-center justify-center gap-2">
<img
src="/images/animated_logo.svg"
alt="Kiln Copilot"
class="size-4"
/>
<span>Kiln Copilot</span>
</div>
</th>
</tr>
</thead>
<tbody>
<tr>
<th class="font-bold text-xs text-gray-500"
>Eval Judge Creation</th
>
<td class="text-center">Manual</td>
<td class="text-center border-l">Automatic</td>
</tr>
<tr>
<th class="font-bold text-xs text-gray-500"
>Edge Case Discovery</th
>
<td class="text-center">Manual</td>
<td class="text-center border-l">Automatic</td>
</tr>
<tr>
<th class="font-bold text-xs text-gray-500"
>Eval Data Creation</th
>
<td class="text-center">Manual</td>
<td class="text-center border-l">Automatic</td>
</tr>
<tr>
<th class="font-bold text-xs text-base-content/60"
>Eval Accuracy</th
>
<td class="text-center">Varies</td>
<td class="text-center border-l">High</td>
</tr>
<tr>
<th class="font-bold text-xs text-gray-500">Approx. Effort</th>
<td class="text-center">20 min</td>
<td class="text-center border-l">3 min</td>
</tr>
<tr>
<th class="font-bold text-xs text-base-content/60"
>Kiln Account</th
>
<td class="text-center">Optional</td>
<td class="text-center border-l">Required</td>
</tr>
</tbody>
</table>
</div>
<table class="table-fixed w-full mt-4">
<colgroup>
<col class="w-[240px]" />
<col />
<col />
</colgroup>
<thead>
<tr class="border-b-0">
<th></th>
<th class="text-center text-lg">Manual</th>
<th class="text-center text-lg">
<div class="flex items-center justify-center gap-2">
<img
src="/images/animated_logo.svg"
alt="Kiln Copilot"
class="size-4"
/>
<span>Kiln Copilot</span>
</div>
</th>
</tr>
</thead>
<tbody>
<tr>
<th class="font-bold text-xs text-gray-500"
>Eval Judge Creation</th
>
<td class="text-center">Manual</td>
<td class="text-center border-l">Automatic</td>
</tr>
<tr>
<th class="font-bold text-xs text-gray-500"
>Edge Case Discovery</th
>
<td class="text-center">Manual</td>
<td class="text-center border-l">Automatic</td>
</tr>
<tr>
<th class="font-bold text-xs text-gray-500">Eval Data Creation</th
>
<td class="text-center">Manual</td>
<td class="text-center border-l">Automatic</td>
</tr>
<tr>
<th class="font-bold text-xs text-base-content/60"
>Eval Accuracy</th
>
<td class="text-center">Varies</td>
<td class="text-center border-l">High</td>
</tr>
<tr>
<th class="font-bold text-xs text-gray-500">Approx. Effort</th>
<td class="text-center">20 min</td>
<td class="text-center border-l">3 min</td>
</tr>
<tr>
<th class="font-bold text-xs text-base-content/60"
>Kiln Account</th
>
<td class="text-center">Optional</td>
<td class="text-center border-l">Required</td>
<td></td>
<td class="text-center">
<button
class="btn btn-outline btn-sm"
on:click={proceed_to_select_template}
>
Create Manually
</button>
</td>
<td class="text-center">
<div
class="tooltip"
data-tip={default_run_config_has_tools
? "Tool calling is not yet supported in Kiln Copilot. Please create the spec manually for now."
: undefined}
>
<button
class="btn btn-primary btn-sm"
disabled={loading || default_run_config_has_tools}
on:click={() =>
goto(`/specs/copilot_auth`, {
replaceState: true,
})}
>
Connect Kiln Copilot
</button>
</div>
</td>
</tr>
</tbody>
</table>
</div>
<table class="table-fixed w-full mt-4">
<colgroup>
<col class="w-[240px]" />
<col />
<col />
</colgroup>
<tbody>
<tr>
<td></td>
<td class="text-center">
<button
class="btn btn-outline btn-sm"
on:click={proceed_to_select_template}
>
Create Manually
</button>
</td>
<td class="text-center">
<button
class="btn btn-primary btn-sm"
on:click={() =>
goto(`/specs/copilot_auth`, {
replaceState: true,
})}
>
Connect Kiln Copilot
</button>
</td>
</tr>
</tbody>
</table>
</div>
{/if}
</AppPage>
</div>
19 changes: 17 additions & 2 deletions app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_builder/+page.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@
import type { SpecType, ModelProviderName, Task } from "$lib/types"
import { goto } from "$app/navigation"
import { spec_field_configs } from "../select_template/spec_templates"
import { checkKilnCopilotAvailable, buildSpecDefinition } from "../spec_utils"
import {
checkKilnCopilotAvailable,
checkDefaultRunConfigHasTools,
buildSpecDefinition,
} from "../spec_utils"
import {
createSpec,
type JudgeInfo,
Expand Down Expand Up @@ -51,6 +55,7 @@

// Copilot availability
let has_kiln_copilot = false
let default_run_config_has_tools = false

// Task data (loaded once in initialize)
let task: Task | null = null
Expand Down Expand Up @@ -123,8 +128,12 @@
$: if (is_tool_use_spec) evaluate_full_trace = true

// Tool call and RAG specs don't support copilot
// Also disable copilot when the default run config has tools (tool calling not supported yet)
$: copilot_enabled =
has_kiln_copilot && !is_tool_use_spec && !is_reference_answer_spec
has_kiln_copilot &&
!is_tool_use_spec &&
!is_reference_answer_spec &&
!default_run_config_has_tools

// Initialize form from URL params
async function initialize() {
Expand All @@ -144,6 +153,12 @@
throw new Error("Failed to load task")
}

// Check if default run config has tools (copilot doesn't support tool calling)
default_run_config_has_tools = await checkDefaultRunConfigHasTools(
project_id,
task,
)

// Get spec type from URL params
const spec_type_param = $page.url.searchParams.get("type")
if (!spec_type_param) {
Expand Down
50 changes: 49 additions & 1 deletion app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,18 @@
import { client } from "$lib/api_client"
import type { Spec, SpecStatus, SpecType } from "$lib/types"
import type {
Spec,
SpecStatus,
SpecType,
Task,
TaskRunConfig,
} from "$lib/types"
import { spec_field_configs } from "./select_template/spec_templates"
import {
load_task_run_configs,
run_configs_by_task_composite_id,
} from "$lib/stores/run_configs_store"
import { get_task_composite_id } from "$lib/stores"
import { get } from "svelte/store"

/**
* Build a definition string from properties
Expand Down Expand Up @@ -41,6 +53,42 @@ export async function checkKilnCopilotAvailable(): Promise<boolean> {
return !!data["kiln_copilot_api_key"]
}

/**
* Check if the task's default run config has any tools configured
* @param project_id - The project ID
* @param task - The task to check
* @returns true if the default run config has tools, false otherwise
*/
export async function checkDefaultRunConfigHasTools(
project_id: string,
task: Task,
): Promise<boolean> {
if (!task.id) {
throw new Error("Task ID is required")
}

if (!task.default_run_config_id) {
return false
}

await load_task_run_configs(project_id, task.id)
const run_configs =
get(run_configs_by_task_composite_id)[
get_task_composite_id(project_id, task.id)
] ?? []

const default_config = run_configs.find(
(config: TaskRunConfig) => config.id === task.default_run_config_id,
)

if (!default_config) {
return false
}

const tools = default_config.run_config_properties?.tools_config?.tools ?? []
return tools.length > 0
}

/**
* Update a spec's priority via the API
* @param project_id - The project ID
Expand Down
Loading