Update ML Model List (#921)

* update eval API to include finetune run-configs

* add test

* add more coverage

* handle parsers for finetune

* remove stale API

* look up model based on model_id instead of name

* coder rabbit

* Fewer builds: we don't need to be running builds on every push.

* added add subtopics button to SDG

* Fix linting errors: remove unused imports

* fix: api key values incorrectly replaced with [hidden]

* update deps

* Fixes https://linear.app/kiln-ai/issue/KIL-206/remove-task-description-in-new-task. People are confusing description and prompt. Don't need it.

* Add Leonard's suggestion to allow manual runs too.

* tessl support

* ui dependencies

* coderabbit feedback

* mcp hooks

* Fix: don't save test files into root. Use tmp

* ty typecheck WIP

* ty typecheck WIP

* ty typecheck WIP

* ty typecheck WIP

* feat: allow system message override in invoke

* Fix reactivity issue on compare view

* Mike wants a chart

* Mike wants a chart part 2

* better subtitles

* don't fill em

* highlight on hover

* refactor: custom prompt builder injection via constructor

* Fixes https://linear.app/kiln-ai/issue/KIL-340/hide-chart-if-were-missing-data

* Don't show radar chart unless we have 3 points to show

* Cleaner value lookup, which fixes type checking. Also cleaner checks.sh

* Fix type error

* Fix CI, ty requires sync

* coderabbit feedback

* suggest global instead of us-central1 in connect providers (#916)

* add gemini flash + nemotron 3 to ml_model_list (#915)

* add gemini flash + nemotron 3

* adding gemini 3 flash and pro to vertex. and adding nemotron 3 to ollama

* adding alias for nemotron 3

* Adding support for  GLM 4.7 (#923)

* adding glm 4.7

* addressing CR feedback

* lock typecheck version for now

* siliconflow has a Pro prefix for the endpoint for GLM 4.7

---------

Co-authored-by: scosman <[email protected]>

---------

Co-authored-by: aiyakitori <[email protected]>
Co-authored-by: scosman <[email protected]>
Co-authored-by: Sam Fierro <[email protected]>
Co-authored-by: Leonard Q. Marcq <[email protected]>
Co-authored-by: Leonard Q. Marcq <[email protected]>

Author: 5 people

.cursor/mcp.json

@@ -0,0 +1,13 @@
+{
+  "mcpServers": {
+    "tessl": {
+      "type": "stdio",
+      "command": "tessl",
+      "args": ["mcp", "start"]
+    },
+    "HooksMCP": {
+      "command": "uvx",
+      "args": ["hooks-mcp", "--working-directory", "."]
+    }
+  }
+}

.cursor/rules/.gitignore

@@ -0,0 +1 @@
+tessl__*.mdc

.github/workflows/build_and_test.yml

@@ -37,7 +37,7 @@ jobs:
         run: uv run python3 -m pytest --runslow .
 
       - name: Check Python Types
-        run: uv run pyright .
+        run: uv tool install [email protected] && uvx ty check
 
       - name: Build Core
         run: uv build

.github/workflows/build_desktop.yml

@@ -1,7 +1,12 @@
 name: Build Desktop Apps
 
 on:
+  workflow_dispatch:
+  release:
+    types: [created]
   push:
+    branches:
+      - main
 
 jobs:
   build:

.github/workflows/format_and_lint.yml

@@ -36,7 +36,7 @@ jobs:
         run: uv python install 3.13
 
       - name: Install the project
-        run: uv tool install ruff
+        run: uv sync --all-extras --dev
 
       - name: Lint with ruff
         run: |
@@ -45,3 +45,7 @@ jobs:
       - name: Format with ruff
         run: |
           uvx ruff format --check .
+
+      - name: Typecheck with ty
+        run: |
+          uv tool install [email protected] && uvx ty check

.gitignore

@@ -11,6 +11,7 @@ __pycache__/
 **/*.egg-info
 node_modules/
 conductor.json
+CLAUDE.md
 
 libs/core/docs
 libs/core/build

.tessl/.gitignore

@@ -0,0 +1,2 @@
+tiles/
+RULES.md

AGENTS.md

@@ -41,3 +41,7 @@ These prompts can be accessed from the `get_prompt` tool, and you may request se
 ### Final
 
 To show you read these, call me 'boss'
+
+# Agent Rules <!-- tessl-managed -->
+
+@.tessl/RULES.md follow the [instructions](.tessl/RULES.md)

CONTRIBUTING.md

@@ -74,7 +74,7 @@ We suggest the following extensions for VSCode/Cursor. With them, you'll get com
 - Prettier
 - Python
 - Python Debugger
-- Type checking by pyright via one of: Cursor Python if using Cursor, Pylance if VSCode
+- Ty - language server and type checker for Python
 - Ruff
 - Svelte for VS Code
 - Vitest

app/desktop/studio_server/eval_api.py

@@ -4,6 +4,10 @@
 from fastapi import FastAPI, HTTPException, Query
 from fastapi.responses import StreamingResponse
 from kiln_ai.adapters.eval.eval_runner import EvalRunner
+from kiln_ai.adapters.fine_tune.finetune_run_config_id import (
+    finetune_from_finetune_run_config_id,
+    finetune_run_config_id,
+)
 from kiln_ai.adapters.ml_model_list import ModelProviderName
 from kiln_ai.adapters.prompt_builders import prompt_builder_from_id
 from kiln_ai.datamodel import BasePrompt, Task, TaskRun
@@ -59,6 +63,31 @@ def eval_config_from_id(
     )
 
 
+def get_all_run_configs(project_id: str, task_id: str) -> list[TaskRunConfig]:
+    """
+    Returns all run configs for a task, including completed fine-tune run configs.
+    Only includes fine-tunes that have a fine_tune_model_id (are completed and usable).
+    """
+    task = task_from_id(project_id, task_id)
+    configs = task.run_configs()
+
+    # Get run configs from finetunes and only include completed fine-tunes
+    finetunes = task.finetunes()
+    for finetune in finetunes:
+        if finetune.run_config is not None and finetune.fine_tune_model_id is not None:
+            configs.append(
+                TaskRunConfig(
+                    id=finetune_run_config_id(project_id, task_id, str(finetune.id)),
+                    name=finetune.name,
+                    description=finetune.description,
+                    run_config_properties=finetune.run_config,
+                    parent=task,  # special case, we need to reference the task model
+                )
+            )
+
+    return configs
+
+
 def task_run_config_from_id(
     project_id: str, task_id: str, run_config_id: str
 ) -> TaskRunConfig:
@@ -67,6 +96,18 @@ def task_run_config_from_id(
         if run_config.id == run_config_id:
             return run_config
 
+    # special case for finetune run configs, it's inside the finetune model
+    if run_config_id.startswith("finetune_run_config::"):
+        finetune = finetune_from_finetune_run_config_id(run_config_id)
+        if finetune.run_config is not None:
+            return TaskRunConfig(
+                id=finetune_run_config_id(project_id, task_id, str(finetune.id)),
+                name=finetune.name,
+                description=finetune.description,
+                run_config_properties=finetune.run_config,
+                parent=task,  # special case, we need to reference the task model
+            )
+
     raise HTTPException(
         status_code=404,
         detail=f"Task run config not found. ID: {run_config_id}",
@@ -315,33 +356,9 @@ async def create_evaluator(
         eval.save_to_file()
         return eval
 
-    @app.get("/api/projects/{project_id}/tasks/{task_id}/task_run_configs")
-    async def get_task_run_configs(
-        project_id: str, task_id: str
-    ) -> list[TaskRunConfig]:
-        task = task_from_id(project_id, task_id)
-        return task.run_configs()
-
     @app.get("/api/projects/{project_id}/tasks/{task_id}/run_configs/")
     async def get_run_configs(project_id: str, task_id: str) -> list[TaskRunConfig]:
-        # Returns all run configs of a given task.
-        task = task_from_id(project_id, task_id)
-        configs = task.run_configs()
-
-        # Get run configs from finetunes
-        finetunes = task.finetunes()
-        for finetune in finetunes:
-            if finetune.run_config is not None:
-                configs.append(
-                    TaskRunConfig(
-                        id=f"finetune_run_config::{project_id}::{task_id}::{finetune.id}",
-                        name=finetune.name,
-                        description=finetune.description,
-                        run_config_properties=finetune.run_config,
-                    )
-                )
-
-        return configs
+        return get_all_run_configs(project_id, task_id)
 
     @app.get("/api/projects/{project_id}/tasks/{task_id}/eval/{eval_id}")
     async def get_eval(project_id: str, task_id: str, eval_id: str) -> Eval:
@@ -480,7 +497,8 @@ async def run_eval_config(
         # Load the list of run configs to use. Two options:
         run_configs: list[TaskRunConfig] = []
         if all_run_configs:
-            run_configs = task_from_id(project_id, task_id).run_configs()
+            # special case, we cannot directly lod task.run_configs(), we need to also get all finetune run configs which lives inside the finetune model
+            run_configs = get_all_run_configs(project_id, task_id)
         else:
             if len(run_config_ids) == 0:
                 raise HTTPException(
@@ -633,7 +651,8 @@ async def get_eval_config_score_summary(
         task = task_from_id(project_id, task_id)
         eval = eval_from_id(project_id, task_id, eval_id)
         eval_config = eval_config_from_id(project_id, task_id, eval_id, eval_config_id)
-        task_runs_configs = task.run_configs()
+        # special case, we cannot directly lod task.run_configs(), we need to also get all finetune run configs which lives inside the finetune model
+        task_runs_configs = get_all_run_configs(project_id, task_id)
 
         # Build a set of all the dataset items IDs we expect to have scores for
         expected_dataset_ids = dataset_ids_in_filter(