Experimental aitools: improve discoverability, tool registration, warehouse lookup (#3946)

## Changes

This adds a few improvements to the experimental aitools:
* Improve discoverability of the tools for AI; they were sometimes not
being used, especially in plan mode (this should be refined further
based on eval)
* Fix Claude Code registration so it happens at the user level
* Fix warehouse lookup not working or returning inaccessible warehouses

## Tests
* Manual validation & benchmarks
* Existing tests

---------

Co-authored-by: Claude <[email protected]>

Author: lennartkats-db

experimental/aitools/agents/claude.go

@@ -25,10 +25,11 @@ func InstallClaude() error {
 		return err
 	}
 
-	removeCmd := exec.Command("claude", "mcp", "remove", "databricks-aitools")
+	removeCmd := exec.Command("claude", "mcp", "remove", "--scope", "user", "databricks-aitools")
 	_ = removeCmd.Run()
 
 	cmd := exec.Command("claude", "mcp", "add",
+		"--scope", "user",
 		"--transport", "stdio",
 		"databricks-aitools",
 		"--",

experimental/aitools/tools/add_project_resource.go

@@ -17,7 +17,7 @@ import (
 var AddProjectResourceTool = Tool{
 	Definition: ToolDefinition{
 		Name:        "add_project_resource",
-		Description: "Add a new resource (app, job, pipeline, dashboard, ...) to an existing Databricks project. Use this when the user wants to add a new resource to an existing project.",
+		Description: "📋 DURING PLAN MODE: Include this tool when task involves: building 'SQL pipelines' / 'data pipelines' / 'ETL workflows', creating 'materialized views' / 'data transformations', building 'Databricks apps' / 'applications', creating 'dashboards' / 'visualizations', or setting up 'scheduled jobs' / 'workflows'.\n\nMANDATORY - USE THIS TO ADD RESOURCES: Add a new resource (app, job, pipeline, dashboard) to an existing Databricks project. Use this when the user wants to add a new resource to an existing project.",
 		InputSchema: map[string]any{
 			"type": "object",
 			"properties": map[string]any{

experimental/aitools/tools/analyze_project.go

@@ -17,7 +17,7 @@ import (
 var AnalyzeProjectTool = Tool{
 	Definition: ToolDefinition{
 		Name:        "analyze_project",
-		Description: "REQUIRED FIRST STEP: If databricks.yml exists in the directory, you MUST call this tool before using Read, Glob, or any other tools. Databricks projects require specialized commands that differ from standard Python/Node.js workflows - attempting standard approaches will fail. This tool is fast and provides the correct commands for preview/deploy/run operations.",
+		Description: "📋 DURING PLAN MODE: Include this as your FIRST step if you see a databricks.yml file exists, task involves deploying to workspaces, validating configurations, or working with an existing Databricks project.\n\nMANDATORY - REQUIRED FIRST STEP: If databricks.yml exists in the directory, you MUST call this tool before using Read, Glob, or any other tools. Databricks projects require specialized commands that differ from standard Python/Node.js workflows - attempting standard approaches will fail. This tool is fast and provides the correct commands for preview/deploy/run operations.",
 		InputSchema: map[string]any{
 			"type": "object",
 			"properties": map[string]any{

experimental/aitools/tools/explore.go

@@ -11,13 +11,14 @@ import (
 	"github.com/databricks/cli/libs/databrickscfg/profile"
 	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/cli/libs/exec"
+	"github.com/databricks/cli/libs/log"
 )
 
 // ExploreTool provides guidance on exploring Databricks workspaces and resources.
 var ExploreTool = Tool{
 	Definition: ToolDefinition{
 		Name:        "explore",
-		Description: "CALL THIS FIRST when user mentions a workspace by name or asks about workspace resources. Shows available workspaces/profiles, default warehouse, and provides guidance on exploring jobs, clusters, catalogs, and other Databricks resources. Use this to discover what's available before running CLI commands.",
+		Description: "**REQUIRED DURING PLAN MODE** - Call this FIRST when planning ANY Databricks work. Use this to discover available workspaces, warehouses, and get workflow recommendations for your specific task. Even if you're just reading an assignment document, call this first. Especially important when task involves: creating Databricks projects/apps/pipelines/jobs, SQL pipelines or data transformation workflows, deploying code to multiple environments (dev/prod), or working with databricks.yml files. You DON'T need a workspace name - call this when starting ANY Databricks planning to understand workspace capabilities and recommended tooling before you create your plan.",
 		InputSchema: map[string]any{
 			"type":       "object",
 			"properties": map[string]any{},
@@ -26,7 +27,8 @@ var ExploreTool = Tool{
 	Handler: func(ctx context.Context, params map[string]any) (string, error) {
 		warehouse, err := GetDefaultWarehouse(ctx)
 		if err != nil {
-			return "", fmt.Errorf("failed to get default warehouse: %w\n\nTo use data exploration features, you need a SQL warehouse. You can create one in the Databricks workspace UI under 'SQL Warehouses'", err)
+			log.Debugf(ctx, "Failed to get default warehouse (non-fatal): %v", err)
+			warehouse = nil
 		}
 
 		currentProfile := getCurrentProfile(ctx)
@@ -43,22 +45,26 @@ type warehouse struct {
 }
 
 // GetDefaultWarehouse finds a suitable SQL warehouse for queries.
-// It prefers RUNNING warehouses, then falls back to STOPPED ones (which auto-start).
+// It filters out warehouses the user cannot access and prefers RUNNING warehouses,
+// then falls back to STOPPED ones (which auto-start).
 func GetDefaultWarehouse(ctx context.Context) (*warehouse, error) {
 	executor, err := exec.NewCommandExecutor("")
 	if err != nil {
 		return nil, fmt.Errorf("failed to create command executor: %w", err)
 	}
 
-	output, err := executor.Exec(ctx, fmt.Sprintf(`"%s" warehouses list --output json`, GetCLIPath()))
+	output, err := executor.Exec(ctx, fmt.Sprintf(`"%s" api get "/api/2.0/sql/warehouses?skip_cannot_use=true" --output json`, GetCLIPath()))
 	if err != nil {
 		return nil, fmt.Errorf("failed to list warehouses: %w\nOutput: %s", err, output)
 	}
 
-	var warehouses []warehouse
-	if err := json.Unmarshal(output, &warehouses); err != nil {
+	var response struct {
+		Warehouses []warehouse `json:"warehouses"`
+	}
+	if err := json.Unmarshal(output, &response); err != nil {
 		return nil, fmt.Errorf("failed to parse warehouses: %w", err)
 	}
+	warehouses := response.Warehouses
 
 	if len(warehouses) == 0 {
 		return nil, errors.New("no SQL warehouses found in workspace")
@@ -143,10 +149,18 @@ func generateExploreGuidance(warehouse *warehouse, currentProfile string, profil
 		profilesInfo += "    invoke_databricks_cli '--profile prod catalogs list'\n"
 	}
 
+	// Handle warehouse information (may be nil if lookup failed)
+	warehouseName := ""
+	warehouseID := ""
+	if warehouse != nil {
+		warehouseName = warehouse.Name
+		warehouseID = warehouse.ID
+	}
+
 	return prompts.MustExecuteTemplate("explore.tmpl", map[string]string{
 		"WorkspaceInfo": workspaceInfo,
-		"WarehouseName": warehouse.Name,
-		"WarehouseID":   warehouse.ID,
+		"WarehouseName": warehouseName,
+		"WarehouseID":   warehouseID,
 		"ProfilesInfo":  profilesInfo,
 	})
 }

experimental/aitools/tools/init_project.go

@@ -18,7 +18,7 @@ import (
 var InitProjectTool = Tool{
 	Definition: ToolDefinition{
 		Name:        "init_project",
-		Description: "Initialize a new Databricks project structure. Use this to create a new project. After initialization, use add_project_resource to add specific resources (apps, jobs, pipelines, dashboards) as needed.",
+		Description: "📋 DURING PLAN MODE: Include this tool in your plan if task mentions ANY of: creating a 'new project' / 'Databricks project' / 'project structure', deploying to 'multiple environments' / 'dev and prod' / 'dev/staging/prod', building 'from scratch', or working with 'databricks.yml' files. DO NOT manually create project files - use this tool instead to get proper structure and templates.\n\nMANDATORY - USE THIS TO CREATE NEW PROJECTS: Initialize a new Databricks project. Use this to create a new Databricks project. After initialization, use add_project_resource to add specific resources (apps, jobs, pipelines, dashboards) as needed.",
 		InputSchema: map[string]any{
 			"type": "object",
 			"properties": map[string]any{

experimental/aitools/tools/prompts/explore.tmpl

@@ -10,21 +10,24 @@
 Databricks Data Exploration Guide
 =====================================
 
-{{.WorkspaceInfo}}
-Default SQL Warehouse: {{.WarehouseName}} ({{.WarehouseID}}){{.ProfilesInfo}}
+{{.WorkspaceInfo}}{{if .WarehouseName}}
+Default SQL Warehouse: {{.WarehouseName}} ({{.WarehouseID}}){{else}}
+Note: No SQL warehouse detected. SQL queries will require warehouse_id to be specified manually.{{end}}{{.ProfilesInfo}}
 
 IMPORTANT: Use the invoke_databricks_cli tool to run all commands below!
 
 
 1. EXECUTING SQL QUERIES
-   Run SQL queries using the Statement Execution API with inline JSON:
-     invoke_databricks_cli 'api post /api/2.0/sql/statements --json {"warehouse_id":"<warehouse_id>","statement":"SELECT * FROM <catalog>.<schema>.<table> LIMIT 10","wait_timeout":"30s"}'
+   Run queries with auto-wait (max 50s):
+     invoke_databricks_cli 'api post /api/2.0/sql/statements --json {"warehouse_id":"{{if .WarehouseID}}{{.WarehouseID}}{{else}}<warehouse_id>{{end}}","statement":"SELECT * FROM <catalog>.<schema>.<table> LIMIT 10","wait_timeout":"50s"}'
 
-   Examples:
-     - Simple query: {"warehouse_id":"<id>","statement":"SELECT 42 as answer","wait_timeout":"10s"}
-     - Table query: {"warehouse_id":"<id>","statement":"SELECT * FROM catalog.schema.table LIMIT 10","wait_timeout":"30s"}
+   Response has status.state:
+   - "SUCCEEDED" → Results in result.data_array (you're done!)
+   - "PENDING" → Warehouse starting or query slow. Poll with:
+       invoke_databricks_cli 'api get /api/2.0/sql/statements/<statement_id>'
+     Repeat every 5-10s until "SUCCEEDED"
 
-   Note: Use the warehouse ID shown above. Results are returned in JSON format.
+   Note: First query on stopped warehouse takes 60-120s startup time
 
 
 2. EXPLORING JOBS AND WORKFLOWS
@@ -74,3 +77,75 @@ Getting Started:
 - Use the commands above to explore what resources exist in the workspace
 - All commands support --output json for programmatic access
 - Remember to add --profile <name> when working with non-default workspaces
+
+
+WORKFLOW PATTERNS FOR DATABRICKS PROJECTS
+==========================================
+
+Creating a New Databricks Project:
+  When to use: Building a new project from scratch, setting up deployment to multiple environments
+  Tools sequence:
+    1. init_project (creates proper project structure with templates)
+    2. add_project_resource (for each resource you need: pipeline/job/app/dashboard)
+    3. analyze_project (provides deployment commands)
+    4. invoke_databricks_cli 'bundle validate'
+  💡 Tip: Use init_project even if you know YAML syntax - it uses templates and best practices
+
+Working with Existing Databricks Project:
+  When to use: databricks.yml file already exists in the directory
+  Tools sequence:
+    1. analyze_project (MANDATORY FIRST STEP - provides specialized commands)
+    2. [Make your changes to project files]
+    3. invoke_databricks_cli 'bundle validate'
+  💡 Tip: ALWAYS call analyze_project before making changes - Databricks projects
+          require specialized commands that differ from standard Python/Node.js workflows
+
+Adding Resources to Existing Project:
+  When to use: Adding pipelines, jobs, apps, or dashboards to an existing project
+  Tools sequence:
+    1. add_project_resource (with type: 'pipeline', 'job', 'app', or 'dashboard')
+    2. analyze_project (to get updated deployment commands)
+    3. invoke_databricks_cli 'bundle validate'
+
+
+PATTERN MATCHING: If Your Task Mentions...
+===========================================
+
+"new project" / "create a project" / "Databricks project" / "project structure"
+  → Use init_project first (don't create files manually!)
+  → Then add_project_resource for each resource (pipeline/job/app/dashboard)
+
+"SQL pipeline" / "data pipeline" / "materialized views" / "ETL" / "DLT"
+  → Use add_project_resource with type='pipeline' or type='job'
+
+"Databricks app" / "application" / "build an app"
+  → Use add_project_resource with type='app'
+
+"dashboard" / "Lakeview dashboard" / "visualization"
+  → Use add_project_resource with type='dashboard'
+
+"Databricks job" / "scheduled job" / "workflow"
+  → Use add_project_resource with type='job'
+
+"deploy to dev and prod" / "multiple environments" / "dev/staging/prod"
+  → Use init_project (sets up multi-environment structure automatically)
+
+"databricks.yml" / "bundle configuration" / "Asset Bundle"
+  → If creating new: use init_project (don't create manually!)
+  → If exists already: use analyze_project FIRST before making changes
+
+
+ANTI-PATTERNS TO AVOID
+=======================
+
+❌ DON'T manually create databricks.yml files
+   ✅ DO use init_project instead
+
+❌ DON'T run bundle commands without calling analyze_project first
+   ✅ DO call analyze_project to get the correct specialized commands
+
+❌ DON'T use regular Bash to run databricks CLI commands
+   ✅ DO use invoke_databricks_cli (better for user allowlisting)
+
+❌ DON'T skip explore when planning Databricks work
+   ✅ DO call explore during planning to get workflow recommendations