diff --git a/experimental/apps-mcp/cmd/init_template.go b/experimental/apps-mcp/cmd/init_template.go index 5e4069591f..f6a1651e74 100644 --- a/experimental/apps-mcp/cmd/init_template.go +++ b/experimental/apps-mcp/cmd/init_template.go @@ -336,12 +336,12 @@ After initialization: cmdio.LogString(ctx, fileTree) } - // Try to read and display CLAUDE.md if present - readClaudeMd(ctx, configFile) + // Inject L2 (target-specific guidance for apps) + targetApps := prompts.MustExecuteTemplate("target_apps.tmpl", map[string]any{}) + cmdio.LogString(ctx, targetApps) - // Re-inject app-specific guidance - appsContent := prompts.MustExecuteTemplate("apps.tmpl", map[string]any{}) - cmdio.LogString(ctx, appsContent) + // Inject L3 (template-specific guidance from CLAUDE.md) + readClaudeMd(ctx, configFile) return nil } diff --git a/experimental/apps-mcp/docs/context-management.md b/experimental/apps-mcp/docs/context-management.md new file mode 100644 index 0000000000..e504c1a3df --- /dev/null +++ b/experimental/apps-mcp/docs/context-management.md @@ -0,0 +1,80 @@ + +# Context Management for Databricks MCP + +## Goals + +- Universal MCP for any coding agent (Claude, Cursor, etc.) +- Support multiple target types: apps, jobs, pipelines +- Support multiple templates per target type +- Clean separation of context layers +- Detect existing project context automatically + +## Context Layers + +| Layer | Content | When Injected | +|-------|---------|---------------| +| **L0: Tools** | Databricks MCP tool names and descriptions | Always (MCP protocol) | +| **L1: Flow** | Universal workflow, available tools, CLI patterns | Always (via `databricks_discover`) | +| **L2: Target** | Target-specific: validation, deployment, constraints | When target type detected or after `init-template` | +| **L3: Template** | SDK/language-specific: file structure, commands, patterns | After `init-template`. For existing projects, agent reads CLAUDE.md. | + +L0 is implicit - tool descriptions guide agent behavior before any tool is called (e.g., `databricks_discover` description tells agent to call it first during planning). + +### Examples + +**L1 (universal):** "validate before deploying", "use invoke_databricks_cli for all commands" + +**L2 (apps):** app naming constraints, deployment consent requirement, app-specific validation + +**L3 (appkit-typescript):** npm scripts, tRPC patterns, useAnalyticsQuery usage, TypeScript import rules + +## Flows + +### New Project + +``` +Agent MCP + │ │ + ├─► databricks_discover │ + │ {working_directory: "."} │ + │ ├─► Run detectors (nothing found) + │ ├─► Return L1 only + │◄─────────────────────────────┤ + │ │ + ├─► invoke_databricks_cli │ + │ ["...", "init-template", ...] + │ ├─► Scaffold project + │ ├─► Return L2[apps] + L3 + │◄─────────────────────────────┤ + │ │ + ├─► (agent now has L1 + L2 + L3) +``` + +### Existing Project + +``` +Agent MCP + │ │ + ├─► databricks_discover │ + │ {working_directory: "./my-app"} + │ ├─► BundleDetector: found apps + jobs + │ ├─► Return L1 + L2[apps] + L2[jobs] + │◄─────────────────────────────┤ + │ │ + ├─► Read CLAUDE.md naturally │ + │ (agent learns L3 itself) │ +``` + +### Combined Bundles + +When `databricks.yml` contains multiple resource types (e.g., app + job), all relevant L2 layers are injected together. + +## Extensibility + +New target types can be added by: +1. Creating `target_.tmpl` in `lib/prompts/` +2. Adding detection logic to recognize the target type from `databricks.yml` + +New templates can be added by: +1. Creating template directory with CLAUDE.md (L3 guidance) +2. Adding detection logic to recognize the template from project files diff --git a/experimental/apps-mcp/lib/detector/bundle_detector.go b/experimental/apps-mcp/lib/detector/bundle_detector.go new file mode 100644 index 0000000000..c88c5d19b1 --- /dev/null +++ b/experimental/apps-mcp/lib/detector/bundle_detector.go @@ -0,0 +1,54 @@ +package detector + +import ( + "context" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/libs/logdiag" +) + +// BundleDetector detects Databricks bundle configuration. +type BundleDetector struct{} + +// Detect loads databricks.yml with all includes and extracts target types. +func (d *BundleDetector) Detect(ctx context.Context, workDir string, detected *DetectedContext) error { + bundlePath := filepath.Join(workDir, "databricks.yml") + if _, err := os.Stat(bundlePath); err != nil { + // no bundle file - not an error, just not a bundle project + return nil + } + + // use full bundle loading to get all resources including from includes + ctx = logdiag.InitContext(ctx) + b, err := bundle.Load(ctx, workDir) + if err != nil || b == nil { + return nil + } + + phases.Load(ctx, b) + if logdiag.HasError(ctx) { + return nil + } + + detected.InProject = true + detected.BundleInfo = &BundleInfo{ + Name: b.Config.Bundle.Name, + RootDir: workDir, + } + + // extract target types from fully loaded resources + if len(b.Config.Resources.Apps) > 0 { + detected.TargetTypes = append(detected.TargetTypes, "apps") + } + if len(b.Config.Resources.Jobs) > 0 { + detected.TargetTypes = append(detected.TargetTypes, "jobs") + } + if len(b.Config.Resources.Pipelines) > 0 { + detected.TargetTypes = append(detected.TargetTypes, "pipelines") + } + + return nil +} diff --git a/experimental/apps-mcp/lib/detector/detector.go b/experimental/apps-mcp/lib/detector/detector.go new file mode 100644 index 0000000000..4b00a589ff --- /dev/null +++ b/experimental/apps-mcp/lib/detector/detector.go @@ -0,0 +1,57 @@ +// Package detector provides project context detection for Databricks MCP. +package detector + +import ( + "context" +) + +// BundleInfo contains information about a detected Databricks bundle. +type BundleInfo struct { + Name string + Target string + RootDir string +} + +// DetectedContext represents the detected project context. +type DetectedContext struct { + InProject bool + TargetTypes []string // ["apps", "jobs"] - supports combined bundles + Template string // "appkit-typescript", "python", etc. + BundleInfo *BundleInfo + Metadata map[string]string +} + +// Detector detects project context from a working directory. +type Detector interface { + // Detect examines the working directory and updates the context. + Detect(ctx context.Context, workDir string, detected *DetectedContext) error +} + +// Registry manages a collection of detectors. +type Registry struct { + detectors []Detector +} + +// NewRegistry creates a new detector registry with default detectors. +func NewRegistry() *Registry { + return &Registry{ + detectors: []Detector{ + &BundleDetector{}, + &TemplateDetector{}, + }, + } +} + +// Detect runs all detectors and returns the combined context. +func (r *Registry) Detect(ctx context.Context, workDir string) *DetectedContext { + detected := &DetectedContext{ + Metadata: make(map[string]string), + } + + for _, d := range r.detectors { + // ignore errors - detectors should be resilient + _ = d.Detect(ctx, workDir, detected) + } + + return detected +} diff --git a/experimental/apps-mcp/lib/detector/detector_test.go b/experimental/apps-mcp/lib/detector/detector_test.go new file mode 100644 index 0000000000..fa25b78971 --- /dev/null +++ b/experimental/apps-mcp/lib/detector/detector_test.go @@ -0,0 +1,115 @@ +package detector_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/experimental/apps-mcp/lib/detector" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDetectorRegistry_EmptyDir(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + registry := detector.NewRegistry() + detected := registry.Detect(ctx, dir) + + assert.False(t, detected.InProject) + assert.Empty(t, detected.TargetTypes) + assert.Empty(t, detected.Template) +} + +func TestDetectorRegistry_BundleWithApps(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + bundleYml := `bundle: + name: my-app +resources: + apps: + my_app: {} +` + require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks.yml"), []byte(bundleYml), 0o644)) + + registry := detector.NewRegistry() + detected := registry.Detect(ctx, dir) + + assert.True(t, detected.InProject) + assert.Equal(t, []string{"apps"}, detected.TargetTypes) + assert.Equal(t, "my-app", detected.BundleInfo.Name) +} + +func TestDetectorRegistry_BundleWithJobs(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + bundleYml := `bundle: + name: my-job +resources: + jobs: + daily_job: {} +` + require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks.yml"), []byte(bundleYml), 0o644)) + + registry := detector.NewRegistry() + detected := registry.Detect(ctx, dir) + + assert.True(t, detected.InProject) + assert.Equal(t, []string{"jobs"}, detected.TargetTypes) + assert.Equal(t, "my-job", detected.BundleInfo.Name) +} + +func TestDetectorRegistry_CombinedBundle(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + bundleYml := `bundle: + name: my-project +resources: + apps: + my_app: {} + jobs: + daily_job: {} +` + require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks.yml"), []byte(bundleYml), 0o644)) + + registry := detector.NewRegistry() + detected := registry.Detect(ctx, dir) + + assert.True(t, detected.InProject) + assert.Contains(t, detected.TargetTypes, "apps") + assert.Contains(t, detected.TargetTypes, "jobs") + assert.Equal(t, "my-project", detected.BundleInfo.Name) +} + +func TestDetectorRegistry_AppkitTemplate(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + // bundle + package.json with appkit marker + bundleYml := `bundle: + name: my-app +resources: + apps: + my_app: {} +` + packageJson := `{ + "name": "my-app", + "dependencies": { + "@databricks/sql": "^1.0.0" + } +}` + require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks.yml"), []byte(bundleYml), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "package.json"), []byte(packageJson), 0o644)) + + registry := detector.NewRegistry() + detected := registry.Detect(ctx, dir) + + assert.True(t, detected.InProject) + assert.Equal(t, []string{"apps"}, detected.TargetTypes) + assert.Equal(t, "appkit-typescript", detected.Template) +} diff --git a/experimental/apps-mcp/lib/detector/template_detector.go b/experimental/apps-mcp/lib/detector/template_detector.go new file mode 100644 index 0000000000..cdf8cbf516 --- /dev/null +++ b/experimental/apps-mcp/lib/detector/template_detector.go @@ -0,0 +1,74 @@ +package detector + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" +) + +// TemplateDetector detects the template type from project files. +type TemplateDetector struct{} + +// packageJSON represents relevant parts of package.json. +type packageJSON struct { + Name string `json:"name"` + Dependencies map[string]string `json:"dependencies"` +} + +// Detect identifies the template type from project configuration files. +func (d *TemplateDetector) Detect(ctx context.Context, workDir string, detected *DetectedContext) error { + // check for appkit-typescript (package.json with specific markers) + if template := d.detectFromPackageJSON(workDir); template != "" { + detected.Template = template + return nil + } + + // check for python template (pyproject.toml) + if template := d.detectFromPyproject(workDir); template != "" { + detected.Template = template + return nil + } + + return nil +} + +func (d *TemplateDetector) detectFromPackageJSON(workDir string) string { + pkgPath := filepath.Join(workDir, "package.json") + + data, err := os.ReadFile(pkgPath) + if err != nil { + return "" + } + + var pkg packageJSON + if err := json.Unmarshal(data, &pkg); err != nil { + return "" + } + + // check for appkit markers + if _, hasAppkit := pkg.Dependencies["@databricks/sql"]; hasAppkit { + return "appkit-typescript" + } + + // check for trpc which is used in appkit + for dep := range pkg.Dependencies { + if strings.HasPrefix(dep, "@trpc/") { + return "appkit-typescript" + } + } + + return "" +} + +func (d *TemplateDetector) detectFromPyproject(workDir string) string { + pyprojectPath := filepath.Join(workDir, "pyproject.toml") + + if _, err := os.Stat(pyprojectPath); err == nil { + // pyproject.toml exists - likely python template + return "python" + } + + return "" +} diff --git a/experimental/apps-mcp/lib/middlewares/engine_guide.go b/experimental/apps-mcp/lib/middlewares/engine_guide.go deleted file mode 100644 index b7aa6f3b5b..0000000000 --- a/experimental/apps-mcp/lib/middlewares/engine_guide.go +++ /dev/null @@ -1,31 +0,0 @@ -package middlewares - -import ( - "github.com/databricks/cli/experimental/apps-mcp/lib/mcp" - "github.com/databricks/cli/experimental/apps-mcp/lib/prompts" -) - -// NewEngineGuideMiddleware creates middleware that injects the initialization message on the first tool call. -func NewEngineGuideMiddleware() mcp.Middleware { - return mcp.NewMiddleware(func(ctx *mcp.MiddlewareContext, next mcp.NextFunc) (*mcp.CallToolResult, error) { - isFirst := ctx.Session.GetBool("isFirstToolCall", true) - initializationMessage := prompts.MustExecuteTemplate("initialization_message.tmpl", nil) - - // If this was the first call and execution was successful, prepend the guide - if isFirst { - ctx.Session.Set("isFirstToolCall", false) - result, err := next() - if err != nil { - result = mcp.CreateNewTextContentResultError(err) - } - if result != nil && len(result.Content) > 0 { - if textContent, ok := result.Content[0].(*mcp.TextContent); ok { - textContent.Text = initializationMessage + "\n\n---\n\n" + textContent.Text - } - } - - return result, nil - } - return next() - }) -} diff --git a/experimental/apps-mcp/lib/prompts/apps.tmpl b/experimental/apps-mcp/lib/prompts/apps.tmpl deleted file mode 100644 index 3252bc8252..0000000000 --- a/experimental/apps-mcp/lib/prompts/apps.tmpl +++ /dev/null @@ -1,49 +0,0 @@ -{{- /* - * Guidance for Databricks Apps development. - * - * This guidance provides instructions for initializing and working with - * Databricks Apps using bundle templates. - * - */ -}} - -DATABRICKS APPS DEVELOPMENT -============================ - -When building or planning a new Databricks app, use the following steps: - -1. MUST start by scaffolding a new Databricks app using command: invoke_databricks_cli 'experimental apps-mcp tools init-template --name my-app-name --description "My app description"' - MUST run this command before planning the app. You must understand tge generated file structure and the purpose of the files. -2. MUST read the CLAUDE.md file in the app directory. -3. SHOULD run `npm install`, run `npm run dev` in the background, and open http://localhost:8000 in your browser before making changes to the app. -4. MUST run `cd ` to navigate to the app directory. MUST execute all commands from this directory. -4. If applicable, create SQL query files in the ./config/queries directory. -5. If applicable, create tRPC endpoints in the ./server/trpc.ts file. -6. If applicable, build UI components in the ./client/src directory. -7. MUST update the ./client/src/App.tsx file -8. MUST update smoke test in the ./tests/smoke.spec.ts file -9. MUST run validation using command: invoke_databricks_cli 'experimental apps-mcp tools validate ./' -10. USER CONSENT: Only deploy the app with explicit consent from the user. Deployment using command: invoke_databricks_cli 'experimental apps-mcp tools deploy' - -# Initialization - -⚠️ ALWAYS start by scaffolding a new Databricks app using command: - -invoke_databricks_cli 'experimental apps-mcp tools init-template --name my-app-name --description "My app description"' - -This creates a `my-app-name/` directory in the current working directory containing the app files. - -# Validation - -⚠️ Always validate your app before deploying to production: - -invoke_databricks_cli 'experimental apps-mcp tools validate ./your-app-location' - -# Deployment - -⚠️ (ONLY when user explicitly requested) Use the deploy command which validates, deploys, and runs the app: - -invoke_databricks_cli 'experimental apps-mcp tools deploy' - -# View and manage your app: - -invoke_databricks_cli 'bundle summary' diff --git a/experimental/apps-mcp/lib/prompts/explore.tmpl b/experimental/apps-mcp/lib/prompts/explore.tmpl deleted file mode 100644 index eb364245c9..0000000000 --- a/experimental/apps-mcp/lib/prompts/explore.tmpl +++ /dev/null @@ -1,120 +0,0 @@ -{{- /* - * Guidance for exploring Databricks workspaces and resources. - * - * This guidance is offered by the explore tool to provide comprehensive - * instructions for discovering and querying workspace resources like - * jobs, clusters, catalogs, tables, and SQL warehouses. - * - */ -}} - -{{.WorkspaceInfo}}{{if .WarehouseName}} -Default SQL Warehouse: {{.WarehouseName}} ({{.WarehouseID}}){{else}} -Note: No SQL warehouse detected. SQL queries will require warehouse_id to be specified manually.{{end}}{{.ProfilesInfo}} - -IMPORTANT: Use the invoke_databricks_cli tool to run all commands below! - -1. EXECUTING SQL QUERIES - Execute SQL queries using the query tool (recommended): - invoke_databricks_cli 'experimental apps-mcp tools query "SELECT * FROM catalog.schema.table LIMIT 10"' - -2. EXPLORING JOBS AND WORKFLOWS - List all jobs: - invoke_databricks_cli 'jobs list' - - Get job details: - invoke_databricks_cli 'jobs get ' - - List job runs: - invoke_databricks_cli 'jobs list-runs --job-id ' - - -3. EXPLORING CLUSTERS - List all clusters: - invoke_databricks_cli 'clusters list' - - Get cluster details: - invoke_databricks_cli 'clusters get ' - - -4. EXPLORING UNITY CATALOG DATA - ⚡ EFFICIENT 4-STEP WORKFLOW: - - 1. Find available catalogs: - invoke_databricks_cli 'catalogs list' - - 2. Find schemas in a catalog: - invoke_databricks_cli 'schemas list ' - - 3. Find tables in a schema: - invoke_databricks_cli 'tables list ' - - 4. Batch discover multiple tables (ONE call for efficiency): - invoke_databricks_cli 'experimental apps-mcp tools discover-schema TABLE1 TABLE2 TABLE3' - - ⚡ Always use batch mode: Discover multiple tables in ONE call instead of separate calls - Table format: CATALOG.SCHEMA.TABLE (e.g., samples.nyctaxi.trips) - - QUICK SQL EXECUTION: - Execute SQL and get JSON results: - invoke_databricks_cli 'experimental apps-mcp tools query "SELECT * FROM catalog.schema.table LIMIT 10"' - - ⚠️ COMMON ERRORS: - ❌ Wrong: invoke_databricks_cli 'tables list samples.tpcds_sf1' - ✅ Correct: invoke_databricks_cli 'tables list samples tpcds_sf1' - (Use separate arguments, not dot notation for catalog and schema) - -5. EXPLORING WORKSPACE FILES - List workspace files and notebooks: - invoke_databricks_cli 'workspace list ' - - Export a notebook: - invoke_databricks_cli 'workspace export ' - - -DATABRICKS ASSET BUNDLES (DABs) WORKFLOW -========================================= - -Creating a New Bundle Project: - When to use: Building a new project from scratch with deployment to multiple environments - - 1. Initialize a new bundle (creates proper project structure): - invoke_databricks_cli 'bundle init' - - 2. Validate the bundle configuration: - invoke_databricks_cli 'bundle validate' - - 3. Deploy to a target environment (dev/staging/prod): - invoke_databricks_cli 'bundle deploy --target dev' - -Working with Existing Bundle Project: - When to use: databricks.yml file already exists in the directory - - 1. Validate changes: - invoke_databricks_cli 'bundle validate' - - 2. Deploy to environment: - invoke_databricks_cli 'bundle deploy --target ' - - 3. Run a resource (job/pipeline/app): - invoke_databricks_cli 'bundle run ' - - 4. Destroy deployed resources: - invoke_databricks_cli 'bundle destroy --target ' - - -BEST PRACTICES -============== - -✅ DO use invoke_databricks_cli for all Databricks CLI commands - (Better for user allowlisting and tracking) - -✅ DO use 'experimental apps-mcp tools query' for SQL execution - (Auto-wait, clean JSON output, no manual polling) - -✅ DO use batch discover-schema for multiple tables - (One call instead of multiple: more efficient) - -✅ DO test SQL with query tool before implementing in code - (Verify syntax and results interactively) - -✅ DO call explore during planning to get workspace context diff --git a/experimental/apps-mcp/lib/prompts/flow.tmpl b/experimental/apps-mcp/lib/prompts/flow.tmpl new file mode 100644 index 0000000000..cf75775cff --- /dev/null +++ b/experimental/apps-mcp/lib/prompts/flow.tmpl @@ -0,0 +1,55 @@ +{{- /* + * L1: Universal workflow guidance for Databricks MCP. + * + * This guidance is always provided by databricks_discover. + * Contains: available tools, CLI patterns, best practices. + */ -}} + +## Available Tools +- **databricks_discover**: Discover workspace resources and get workflow recommendations (call this first) +- **databricks_configure_auth**: Switch workspace profile/host +- **invoke_databricks_cli**: Execute any Databricks CLI command + +## Workflow Best Practices +- Use `databricks_discover` at the start of tasks to understand workspace context +- Use `invoke_databricks_cli` for all Databricks CLI operations +- For Databricks Asset Bundles: validate before deploying +- When not sure about the user's intent, ask clarifying questions +- Do NOT create summary files, reports, or README unless explicitly requested + +{{.WorkspaceInfo}}{{if .WarehouseName}} +Default SQL Warehouse: {{.WarehouseName}} ({{.WarehouseID}}){{else}} +Note: No SQL warehouse detected. SQL queries will require warehouse_id to be specified manually.{{end}}{{.ProfilesInfo}} + +IMPORTANT: Use invoke_databricks_cli for all commands below! + +## SQL Queries +Execute SQL using the query tool: + invoke_databricks_cli 'experimental apps-mcp tools query "SELECT * FROM catalog.schema.table LIMIT 10"' + +## Exploring Resources +Jobs: + invoke_databricks_cli 'jobs list' + invoke_databricks_cli 'jobs get ' + +Clusters: + invoke_databricks_cli 'clusters list' + +Unity Catalog: + invoke_databricks_cli 'catalogs list' + invoke_databricks_cli 'schemas list ' + invoke_databricks_cli 'tables list ' + invoke_databricks_cli 'experimental apps-mcp tools discover-schema TABLE1 TABLE2 TABLE3' + +⚠️ Use separate arguments for catalog/schema: 'tables list samples tpcds_sf1' (not dot notation) + +## Databricks Asset Bundles +New project: + invoke_databricks_cli 'bundle init' + invoke_databricks_cli 'bundle validate' + invoke_databricks_cli 'bundle deploy --target dev' + +Existing project: + invoke_databricks_cli 'bundle validate' + invoke_databricks_cli 'bundle deploy --target ' + invoke_databricks_cli 'bundle run ' diff --git a/experimental/apps-mcp/lib/prompts/initialization_message.tmpl b/experimental/apps-mcp/lib/prompts/initialization_message.tmpl deleted file mode 100644 index 4621d24ad8..0000000000 --- a/experimental/apps-mcp/lib/prompts/initialization_message.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -Your session in Databricks MCP has been successfully initialized. - -## Available Tools: -- **explore**: Discover workspace resources, get CLI command examples, and workflow recommendations -- **invoke_databricks_cli**: Execute any Databricks CLI command (bundle, apps, workspace, etc.) - -## Workflow Best Practices: -- Use `explore` at the start of tasks to understand workspace context and get relevant commands -- Use `invoke_databricks_cli` for all Databricks CLI operations (better for tracking and allowlisting) -- For Databricks Asset Bundles: Use `invoke_databricks_cli 'bundle validate'` before deploying -- Always validate before deploying to ensure configuration is correct -- When not sure about the user's intent, ask clarifying questions before proceeding -- Do NOT create summary files, reports, or README unless explicitly requested diff --git a/experimental/apps-mcp/lib/prompts/prompts.go b/experimental/apps-mcp/lib/prompts/prompts.go index 44ab5b7c0a..9304ccd290 100644 --- a/experimental/apps-mcp/lib/prompts/prompts.go +++ b/experimental/apps-mcp/lib/prompts/prompts.go @@ -58,3 +58,9 @@ func MustLoadTemplate(name string) string { } return result } + +// TemplateExists checks if a template with the given name exists. +func TemplateExists(name string) bool { + _, err := promptTemplates.ReadFile(name) + return err == nil +} diff --git a/experimental/apps-mcp/lib/prompts/target_apps.tmpl b/experimental/apps-mcp/lib/prompts/target_apps.tmpl new file mode 100644 index 0000000000..d86f805b09 --- /dev/null +++ b/experimental/apps-mcp/lib/prompts/target_apps.tmpl @@ -0,0 +1,29 @@ +{{- /* + * L2: Target-specific guidance for Databricks Apps. + * + * Injected when: target type "apps" is detected or after init-template. + * Contains: app naming constraints, validation, deployment consent. + */ -}} + +## Databricks Apps Development + +### Building a New App +1. Scaffold: invoke_databricks_cli 'experimental apps-mcp tools init-template --name my-app --description "My app description"' +2. Read CLAUDE.md in the generated directory +3. Navigate to project: cd +4. Run locally: npm install && npm run dev (opens http://localhost:8000) + +### Validation +⚠️ Always validate before deploying: + invoke_databricks_cli 'experimental apps-mcp tools validate ./' + +### Deployment +⚠️ USER CONSENT REQUIRED: Only deploy with explicit user permission. + invoke_databricks_cli 'experimental apps-mcp tools deploy' + +### App Naming +- App name must be ≤26 characters (dev- prefix adds 4 chars, max total 30) +- Use lowercase letters, numbers, and hyphens only + +### View and Manage + invoke_databricks_cli 'bundle summary' diff --git a/experimental/apps-mcp/lib/providers/clitools/explore.go b/experimental/apps-mcp/lib/providers/clitools/discover.go similarity index 53% rename from experimental/apps-mcp/lib/providers/clitools/explore.go rename to experimental/apps-mcp/lib/providers/clitools/discover.go index 7f6bf986ac..3ac1a1d6ab 100644 --- a/experimental/apps-mcp/lib/providers/clitools/explore.go +++ b/experimental/apps-mcp/lib/providers/clitools/discover.go @@ -4,16 +4,17 @@ import ( "context" "fmt" + "github.com/databricks/cli/experimental/apps-mcp/lib/detector" "github.com/databricks/cli/experimental/apps-mcp/lib/middlewares" "github.com/databricks/cli/experimental/apps-mcp/lib/prompts" - "github.com/databricks/cli/experimental/apps-mcp/lib/session" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/service/sql" ) -// Explore provides guidance on exploring Databricks workspaces and resources. -func Explore(ctx context.Context) (string, error) { +// Discover provides workspace context and workflow guidance. +// Returns L1 (flow) always + L2 (target) for detected target types. +func Discover(ctx context.Context, workingDirectory string) (string, error) { warehouse, err := middlewares.GetWarehouseEndpoint(ctx) if err != nil { log.Debugf(ctx, "Failed to get default warehouse (non-fatal): %v", err) @@ -23,15 +24,46 @@ func Explore(ctx context.Context) (string, error) { currentProfile := middlewares.GetDatabricksProfile(ctx) profiles := middlewares.GetAvailableProfiles(ctx) - return generateExploreGuidance(ctx, warehouse, currentProfile, profiles), nil + // run detectors to identify project context + registry := detector.NewRegistry() + detected := registry.Detect(ctx, workingDirectory) + + return generateDiscoverGuidance(ctx, warehouse, currentProfile, profiles, detected), nil } -// generateExploreGuidance creates comprehensive guidance for data exploration. -func generateExploreGuidance(ctx context.Context, warehouse *sql.EndpointInfo, currentProfile string, profiles profile.Profiles) string { - // Build workspace/profile information +// generateDiscoverGuidance creates guidance with L1 (flow) + L2 (target) layers. +func generateDiscoverGuidance(ctx context.Context, warehouse *sql.EndpointInfo, currentProfile string, profiles profile.Profiles, detected *detector.DetectedContext) string { + data := buildTemplateData(warehouse, currentProfile, profiles) + + // L1: always include flow guidance + result := prompts.MustExecuteTemplate("flow.tmpl", data) + + // L2: inject target-specific guidance for detected target types + for _, targetType := range detected.TargetTypes { + templateName := fmt.Sprintf("target_%s.tmpl", targetType) + if prompts.TemplateExists(templateName) { + targetContent := prompts.MustExecuteTemplate(templateName, data) + result += "\n\n" + targetContent + log.Debugf(ctx, "Injected L2 guidance for target type: %s", targetType) + } else { + log.Debugf(ctx, "No L2 template found for target type: %s", targetType) + } + } + + // add project context info if detected + if detected.InProject { + result += "\n\nDetected project: " + detected.BundleInfo.Name + if detected.Template != "" { + result += fmt.Sprintf(" (template: %s)", detected.Template) + } + } + + return result +} + +func buildTemplateData(warehouse *sql.EndpointInfo, currentProfile string, profiles profile.Profiles) map[string]string { workspaceInfo := "Current Workspace Profile: " + currentProfile if len(profiles) > 0 { - // Find current profile details var currentHost string for _, p := range profiles { if p.Name == currentProfile { @@ -47,7 +79,6 @@ func generateExploreGuidance(ctx context.Context, warehouse *sql.EndpointInfo, c } } - // Build available profiles list profilesInfo := "" if len(profiles) > 1 { profilesInfo = "\n\nAvailable Workspace Profiles:\n" @@ -67,7 +98,6 @@ func generateExploreGuidance(ctx context.Context, warehouse *sql.EndpointInfo, c profilesInfo += " invoke_databricks_cli '--profile prod catalogs list'\n" } - // Handle warehouse information (may be nil if lookup failed) warehouseName := "" warehouseID := "" if warehouse != nil { @@ -75,49 +105,11 @@ func generateExploreGuidance(ctx context.Context, warehouse *sql.EndpointInfo, c warehouseID = warehouse.Id } - // Prepare template data - data := map[string]string{ + return map[string]string{ "WorkspaceInfo": workspaceInfo, "WarehouseName": warehouseName, "WarehouseID": warehouseID, "ProfilesInfo": profilesInfo, "Profile": currentProfile, } - - // Render base explore template - result := prompts.MustExecuteTemplate("explore.tmpl", data) - - // Get session and check for enabled capabilities - sess, err := session.GetSession(ctx) - if err != nil { - log.Debugf(ctx, "No session found, skipping capability-based instructions: %v", err) - return result - } - - capabilities, ok := sess.Get(session.CapabilitiesDataKey) - if !ok { - log.Debugf(ctx, "No capabilities set in session") - return result - } - - capList, ok := capabilities.([]string) - if !ok { - log.Warnf(ctx, "Capabilities is not a string slice, skipping") - return result - } - - // Inject additional templates based on enabled capabilities - for _, cap := range capList { - switch cap { - case "apps": - // Render and append apps template - appsContent := prompts.MustExecuteTemplate("apps.tmpl", data) - result = result + "\n\n" + appsContent - log.Debugf(ctx, "Injected apps instructions based on capability") - default: - log.Debugf(ctx, "Unknown capability: %s", cap) - } - } - - return result } diff --git a/experimental/apps-mcp/lib/providers/clitools/provider.go b/experimental/apps-mcp/lib/providers/clitools/provider.go index c04fb5cb10..e9d146f21a 100644 --- a/experimental/apps-mcp/lib/providers/clitools/provider.go +++ b/experimental/apps-mcp/lib/providers/clitools/provider.go @@ -85,15 +85,19 @@ func (p *Provider) RegisterTools(server *mcpsdk.Server) error { }, ) - // Register explore tool + // Register databricks_discover tool + type DiscoverInput struct { + WorkingDirectory string `json:"working_directory" jsonschema:"required" jsonschema_description:"The directory to detect project context from."` + } + mcpsdk.AddTool(server, &mcpsdk.Tool{ - Name: "explore", - Description: "Discover available Databricks workspaces, warehouses, and get workflow recommendations. Call this FIRST when planning ANY Databricks work involving apps, pipelines, jobs, bundles, or SQL workflows. Returns workspace capabilities and recommended tooling.", + Name: "databricks_discover", + Description: "Discover available Databricks workspaces, warehouses, and get workflow recommendations. Call this FIRST when planning ANY Databricks work involving apps, dashboards, pipelines, jobs, bundles, or SQL workflows. Returns workspace capabilities and recommended tooling.", }, - func(ctx context.Context, req *mcpsdk.CallToolRequest, args struct{}) (*mcpsdk.CallToolResult, any, error) { - log.Debug(ctx, "explore called") - result, err := Explore(ctx) + func(ctx context.Context, req *mcpsdk.CallToolRequest, args DiscoverInput) (*mcpsdk.CallToolResult, any, error) { + log.Debugf(ctx, "databricks_discover called: working_directory=%s", args.WorkingDirectory) + result, err := Discover(ctx, args.WorkingDirectory) if err != nil { return nil, nil, err } diff --git a/experimental/apps-mcp/lib/server/server.go b/experimental/apps-mcp/lib/server/server.go index 3b5bf65cb8..9694749d38 100644 --- a/experimental/apps-mcp/lib/server/server.go +++ b/experimental/apps-mcp/lib/server/server.go @@ -42,7 +42,6 @@ func NewServer(ctx context.Context, cfg *mcp.Config) *Server { server.AddMiddleware(middlewares.NewToolCounterMiddleware(sess)) server.AddMiddleware(middlewares.NewDatabricksClientMiddleware([]string{"databricks_configure_auth"})) - server.AddMiddleware(middlewares.NewEngineGuideMiddleware()) server.AddMiddleware(middlewares.NewTrajectoryMiddleware(tracker)) sess.SetTracker(tracker)