From 7c5050c086af55110b0c9167504d87d1af88dbdd Mon Sep 17 00:00:00 2001
From: Sam Fierro <13154106+sfierro@users.noreply.github.com>
Date: Tue, 20 Jan 2026 20:13:08 -0800
Subject: [PATCH 1/2] block copilot workflow button when deafult run config has
 tools

---
 .../[task_id]/select_workflow/+page.svelte    | 220 +++++++++++-------
 .../[task_id]/spec_builder/+page.svelte       |  19 +-
 .../[project_id]/[task_id]/spec_utils.ts      |  46 +++-
 3 files changed, 194 insertions(+), 91 deletions(-)

diff --git a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/select_workflow/+page.svelte b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/select_workflow/+page.svelte
index 39da1017e..05023d008 100644
--- a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/select_workflow/+page.svelte
+++ b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/select_workflow/+page.svelte
@@ -1,11 +1,36 @@
 <script lang="ts">
   import { goto } from "$app/navigation"
   import { page } from "$app/stores"
+  import { onMount } from "svelte"
   import AppPage from "../../../../app_page.svelte"
+  import { load_task } from "$lib/stores"
+  import { checkDefaultRunConfigHasTools } from "../spec_utils"
+  import { createKilnError, type KilnError } from "$lib/utils/error_handlers"
 
   $: project_id = $page.params.project_id
   $: task_id = $page.params.task_id
 
+  let loading = true
+  let default_run_config_has_tools = false
+  let error: KilnError | null = null
+
+  onMount(async () => {
+    try {
+      const task = await load_task(project_id, task_id)
+      if (!task) {
+        throw new Error("Failed to load task")
+      }
+      default_run_config_has_tools = await checkDefaultRunConfigHasTools(
+        project_id,
+        task,
+      )
+    } catch (e) {
+      error = createKilnError(e)
+    } finally {
+      loading = false
+    }
+  })
+
   function proceed_to_select_template() {
     goto(`/specs/${project_id}/${task_id}/select_template`)
   }
@@ -24,104 +49,123 @@
       },
     ]}
   >
-    <div class="my-4 max-w-[680px]">
-      <div class="overflow-x-auto border-b">
-        <table class="table table-fixed w-full">
+    {#if loading}
+      <div class="w-full min-h-[50vh] flex justify-center items-center">
+        <div class="loading loading-spinner loading-lg"></div>
+      </div>
+    {:else if error}
+      <div class="text-error text-sm">
+        {error.getMessage() || "An unknown error occurred"}
+      </div>
+    {:else}
+      <div class="my-4 max-w-[680px]">
+        <div class="overflow-x-auto border-b">
+          <table class="table table-fixed w-full">
+            <colgroup>
+              <col class="w-[240px]" />
+              <col />
+              <col />
+            </colgroup>
+            <thead>
+              <tr class="border-b-0">
+                <th></th>
+                <th class="text-center text-lg">Manual</th>
+                <th class="text-center text-lg">
+                  <div class="flex items-center justify-center gap-2">
+                    <img
+                      src="/images/animated_logo.svg"
+                      alt="Kiln Copilot"
+                      class="size-4"
+                    />
+                    <span>Kiln Copilot</span>
+                  </div>
+                </th>
+              </tr>
+            </thead>
+            <tbody>
+              <tr>
+                <th class="font-bold text-xs text-gray-500"
+                  >Eval Judge Creation</th
+                >
+                <td class="text-center">Manual</td>
+                <td class="text-center border-l">Automatic</td>
+              </tr>
+              <tr>
+                <th class="font-bold text-xs text-gray-500"
+                  >Edge Case Discovery</th
+                >
+                <td class="text-center">Manual</td>
+                <td class="text-center border-l">Automatic</td>
+              </tr>
+              <tr>
+                <th class="font-bold text-xs text-gray-500"
+                  >Eval Data Creation</th
+                >
+                <td class="text-center">Manual</td>
+                <td class="text-center border-l">Automatic</td>
+              </tr>
+              <tr>
+                <th class="font-bold text-xs text-base-content/60"
+                  >Eval Accuracy</th
+                >
+                <td class="text-center">Varies</td>
+                <td class="text-center border-l">High</td>
+              </tr>
+              <tr>
+                <th class="font-bold text-xs text-gray-500">Approx. Effort</th>
+                <td class="text-center">20 min</td>
+                <td class="text-center border-l">3 min</td>
+              </tr>
+              <tr>
+                <th class="font-bold text-xs text-base-content/60"
+                  >Kiln Account</th
+                >
+                <td class="text-center">Optional</td>
+                <td class="text-center border-l">Required</td>
+              </tr>
+            </tbody>
+          </table>
+        </div>
+        <table class="table-fixed w-full mt-4">
           <colgroup>
             <col class="w-[240px]" />
             <col />
             <col />
           </colgroup>
-          <thead>
-            <tr class="border-b-0">
-              <th></th>
-              <th class="text-center text-lg">Manual</th>
-              <th class="text-center text-lg">
-                <div class="flex items-center justify-center gap-2">
-                  <img
-                    src="/images/animated_logo.svg"
-                    alt="Kiln Copilot"
-                    class="size-4"
-                  />
-                  <span>Kiln Copilot</span>
-                </div>
-              </th>
-            </tr>
-          </thead>
           <tbody>
             <tr>
-              <th class="font-bold text-xs text-gray-500"
-                >Eval Judge Creation</th
-              >
-              <td class="text-center">Manual</td>
-              <td class="text-center border-l">Automatic</td>
-            </tr>
-            <tr>
-              <th class="font-bold text-xs text-gray-500"
-                >Edge Case Discovery</th
-              >
-              <td class="text-center">Manual</td>
-              <td class="text-center border-l">Automatic</td>
-            </tr>
-            <tr>
-              <th class="font-bold text-xs text-gray-500">Eval Data Creation</th
-              >
-              <td class="text-center">Manual</td>
-              <td class="text-center border-l">Automatic</td>
-            </tr>
-            <tr>
-              <th class="font-bold text-xs text-base-content/60"
-                >Eval Accuracy</th
-              >
-              <td class="text-center">Varies</td>
-              <td class="text-center border-l">High</td>
-            </tr>
-            <tr>
-              <th class="font-bold text-xs text-gray-500">Approx. Effort</th>
-              <td class="text-center">20 min</td>
-              <td class="text-center border-l">3 min</td>
-            </tr>
-            <tr>
-              <th class="font-bold text-xs text-base-content/60"
-                >Kiln Account</th
-              >
-              <td class="text-center">Optional</td>
-              <td class="text-center border-l">Required</td>
+              <td></td>
+              <td class="text-center">
+                <button
+                  class="btn btn-outline btn-sm"
+                  on:click={proceed_to_select_template}
+                >
+                  Create Manually
+                </button>
+              </td>
+              <td class="text-center">
+                <div
+                  class="tooltip"
+                  data-tip={default_run_config_has_tools
+                    ? "Tool calling is not yet supported in Kiln Copilot. Please create the spec manually for now."
+                    : undefined}
+                >
+                  <button
+                    class="btn btn-primary btn-sm"
+                    disabled={loading || default_run_config_has_tools}
+                    on:click={() =>
+                      goto(`/specs/copilot_auth`, {
+                        replaceState: true,
+                      })}
+                  >
+                    Connect Kiln Copilot
+                  </button>
+                </div>
+              </td>
             </tr>
           </tbody>
         </table>
       </div>
-      <table class="table-fixed w-full mt-4">
-        <colgroup>
-          <col class="w-[240px]" />
-          <col />
-          <col />
-        </colgroup>
-        <tbody>
-          <tr>
-            <td></td>
-            <td class="text-center">
-              <button
-                class="btn btn-outline btn-sm"
-                on:click={proceed_to_select_template}
-              >
-                Create Manually
-              </button>
-            </td>
-            <td class="text-center">
-              <button
-                class="btn btn-primary btn-sm"
-                on:click={() =>
-                  goto(`/specs/copilot_auth`, {
-                    replaceState: true,
-                  })}
-              >
-                Connect Kiln Copilot
-              </button>
-            </td>
-          </tr>
-        </tbody>
-      </table>
-    </div>
+    {/if}
   </AppPage>
 </div>
diff --git a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_builder/+page.svelte b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_builder/+page.svelte
index 316aa687a..db7bd1a84 100644
--- a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_builder/+page.svelte
+++ b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_builder/+page.svelte
@@ -7,7 +7,11 @@
   import type { SpecType, ModelProviderName, Task } from "$lib/types"
   import { goto } from "$app/navigation"
   import { spec_field_configs } from "../select_template/spec_templates"
-  import { checkKilnCopilotAvailable, buildSpecDefinition } from "../spec_utils"
+  import {
+    checkKilnCopilotAvailable,
+    checkDefaultRunConfigHasTools,
+    buildSpecDefinition,
+  } from "../spec_utils"
   import {
     createSpec,
     type JudgeInfo,
@@ -51,6 +55,7 @@
 
   // Copilot availability
   let has_kiln_copilot = false
+  let default_run_config_has_tools = false
 
   // Task data (loaded once in initialize)
   let task: Task | null = null
@@ -123,8 +128,12 @@
   $: if (is_tool_use_spec) evaluate_full_trace = true
 
   // Tool call and RAG specs don't support copilot
+  // Also disable copilot when the default run config has tools (tool calling not supported yet)
   $: copilot_enabled =
-    has_kiln_copilot && !is_tool_use_spec && !is_reference_answer_spec
+    has_kiln_copilot &&
+    !is_tool_use_spec &&
+    !is_reference_answer_spec &&
+    !default_run_config_has_tools
 
   // Initialize form from URL params
   async function initialize() {
@@ -144,6 +153,12 @@
         throw new Error("Failed to load task")
       }
 
+      // Check if default run config has tools (copilot doesn't support tool calling)
+      default_run_config_has_tools = await checkDefaultRunConfigHasTools(
+        project_id,
+        task,
+      )
+
       // Get spec type from URL params
       const spec_type_param = $page.url.searchParams.get("type")
       if (!spec_type_param) {
diff --git a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_utils.ts b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_utils.ts
index af44b56e2..c1e991c0c 100644
--- a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_utils.ts
+++ b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_utils.ts
@@ -1,6 +1,18 @@
 import { client } from "$lib/api_client"
-import type { Spec, SpecStatus, SpecType } from "$lib/types"
+import type {
+  Spec,
+  SpecStatus,
+  SpecType,
+  Task,
+  TaskRunConfig,
+} from "$lib/types"
 import { spec_field_configs } from "./select_template/spec_templates"
+import {
+  load_task_run_configs,
+  run_configs_by_task_composite_id,
+} from "$lib/stores/run_configs_store"
+import { get_task_composite_id } from "$lib/stores"
+import { get } from "svelte/store"
 
 /**
  * Build a definition string from properties
@@ -41,6 +53,38 @@ export async function checkKilnCopilotAvailable(): Promise<boolean> {
   return !!data["kiln_copilot_api_key"]
 }
 
+/**
+ * Check if the task's default run config has any tools configured
+ * @param project_id - The project ID
+ * @param task - The task to check
+ * @returns true if the default run config has tools, false otherwise
+ */
+export async function checkDefaultRunConfigHasTools(
+  project_id: string,
+  task: Task,
+): Promise<boolean> {
+  if (!task.id || !task.default_run_config_id) {
+    return false
+  }
+
+  await load_task_run_configs(project_id, task.id)
+  const run_configs =
+    get(run_configs_by_task_composite_id)[
+      get_task_composite_id(project_id, task.id)
+    ] ?? []
+
+  const default_config = run_configs.find(
+    (config: TaskRunConfig) => config.id === task.default_run_config_id,
+  )
+
+  if (!default_config) {
+    return false
+  }
+
+  const tools = default_config.run_config_properties?.tools_config?.tools ?? []
+  return tools.length > 0
+}
+
 /**
  * Update a spec's priority via the API
  * @param project_id - The project ID

From 7c2f879942620d11cdb246af7a84b692eae92220 Mon Sep 17 00:00:00 2001
From: Sam Fierro <13154106+sfierro@users.noreply.github.com>
Date: Tue, 20 Jan 2026 20:24:55 -0800
Subject: [PATCH 2/2] feedback

---
 .../routes/(app)/specs/[project_id]/[task_id]/spec_utils.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_utils.ts b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_utils.ts
index c1e991c0c..35922dd48 100644
--- a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_utils.ts
+++ b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/spec_utils.ts
@@ -63,7 +63,11 @@ export async function checkDefaultRunConfigHasTools(
   project_id: string,
   task: Task,
 ): Promise<boolean> {
-  if (!task.id || !task.default_run_config_id) {
+  if (!task.id) {
+    throw new Error("Task ID is required")
+  }
+
+  if (!task.default_run_config_id) {
     return false
   }