diff --git a/acceptance/apps/init-template/app/output.txt b/acceptance/apps/init-template/app/output.txt index a522103bfa..7c1d12ea3e 100644 --- a/acceptance/apps/init-template/app/output.txt +++ b/acceptance/apps/init-template/app/output.txt @@ -1 +1,139 @@ -✓ Template instantiation succeeded +-- +## Databricks Apps Development + +### Validation +⚠️ Always validate before deploying: + invoke_databricks_cli 'experimental aitools tools validate ./' + +This is battle-tested to catch common issues before deployment. Prefer using this over manual checks (e.g. `npm run lint`), as it covers more ground specific to Databricks Apps. + +### Deployment +⚠️ USER CONSENT REQUIRED: Only deploy with explicit user permission. + invoke_databricks_cli 'experimental aitools tools deploy' + +### View and Manage + invoke_databricks_cli 'bundle summary' + +### View App Logs +To troubleshoot deployed apps, view their logs: + invoke_databricks_cli 'apps logs --tail-lines 100' + +### Local Development vs Deployed Apps + +During development: +- Start template-specific dev server (see project's CLAUDE.md for command and port) +- Use localhost URL shown when dev server starts + +After deployment: +- Get URL from: invoke_databricks_cli 'bundle summary' + +Decision tree: +- "open the app" + not deployed → localhost +- "open the app" + deployed → ask which environment +- "localhost"/"local" → always localhost + + +## Skills + +You have access to modular Skills for domain-specific expertise knowledge. + +### Skill Selection & Loading +* When a user request matches a skill's scope description, select that Skill +* Load skills using the MCP tool: `read_skill_file(file_path: "category/skill-name/SKILL.md")` +* Example: `read_skill_file(file_path: "pipelines/materialized-view/SKILL.md")` +* Skills may contain links to sub-sections (e.g., "category/skill-name/file.md") +* If no Skill is suitable, continue with your base capabilities +* Never mention or reference skills to the user, only use them internally + +### Skill Registry (names + brief descriptors) + + +**Note**: The following skills are for other resource types and may not be directly relevant to this project. + +* **pipelines/auto-cdc/SKILL.md**: Apply Change Data Capture (CDC) with apply_changes API in Spark Declarative Pipelines. Use when user needs to process CDC feeds from databases, handle upserts/deletes, maintain slowly changing dimensions (SCD Type 1 and Type 2), synchronize data from operational databases, or process merge operations. + + + +=== CLAUDE.md === +TypeScript full-stack template powered by **Databricks AppKit** with tRPC for additional custom API endpoints. + +- server/: Node.js backend with App Kit and tRPC +- client/: React frontend with App Kit hooks and tRPC client +- config/queries/: SQL query files for analytics +- shared/: Shared TypeScript types +- docs/: Detailed documentation on using App Kit features + +## Quick Start: Your First Query & Chart + +Follow these 3 steps to add data visualization to your app: + +**Step 1: Create a SQL query file** + +```sql +-- config/queries/my_data.sql +SELECT category, COUNT(*) as count, AVG(value) as avg_value +FROM my_table +GROUP BY category +``` + +**Step 2: Define the schema** + +```typescript +// config/queries/schema.ts +export const querySchemas = { + my_data: z.array( + z.object({ + category: z.string(), + count: z.number(), + avg_value: z.number(), + }) + ), +}; +``` + +**Step 3: Add visualization to your app** + +```typescript +// client/src/App.tsx +import { BarChart } from '@databricks/appkit-ui/react'; + + +``` + +**That's it!** The component handles data fetching, loading states, and rendering automatically. + +**To refresh TypeScript types after adding queries:** +- Run `npm run typegen` OR run `npm run dev` - both auto-generate type definitions in `client/src/appKitTypes.d.ts` +- DO NOT manually edit `appKitTypes.d.ts` + +## Installation + +**IMPORTANT**: When running `npm install`, always use `required_permissions: ['all']` to avoid sandbox permission errors. + +## NPM Scripts + +### Development +- `npm run dev` - Start dev server with hot reload (**ALWAYS use during development**) + +### Testing and Code Quality +See the databricks experimental aitools tools validate instead of running these individually. + +### Utility +- `npm run clean` - Remove all build artifacts and node_modules + +**Common workflows:** +- Development: `npm run dev` → make changes → `npm run typecheck` → `npm run lint:fix` +- Pre-deploy: Validate with `databricks experimental aitools tools validate .` + +## Documentation + +**IMPORTANT**: Read the relevant docs below before implementing features. They contain critical information about common pitfalls (e.g., SQL numeric type handling, schema definitions, Radix UI constraints). + +- [SQL Queries](docs/sql-queries.md) - query files, schemas, type handling, parameterization +- [App Kit SDK](docs/appkit-sdk.md) - TypeScript imports, server setup, useAnalyticsQuery hook +- [Frontend](docs/frontend.md) - visualization components, styling, layout, Radix constraints +- [tRPC](docs/trpc.md) - custom endpoints for non-SQL operations (mutations, Databricks APIs) +- [Testing](docs/testing.md) - vitest unit tests, Playwright smoke/E2E tests + +================= + diff --git a/acceptance/apps/init-template/app/script b/acceptance/apps/init-template/app/script index 3f0d48dae6..12955cd313 100644 --- a/acceptance/apps/init-template/app/script +++ b/acceptance/apps/init-template/app/script @@ -1,4 +1,3 @@ #!/bin/bash -$CLI experimental aitools tools init-template app --name test_app --sql-warehouse-id abc123 --output-dir output > /dev/null 2>&1 -echo "✓ Template instantiation succeeded" +$CLI experimental aitools tools init-template app --name test_app --sql-warehouse-id abc123 --output-dir output 2>&1 | grep -A 9999 "^--$" rm -rf output diff --git a/acceptance/apps/init-template/empty/output.txt b/acceptance/apps/init-template/empty/output.txt index a522103bfa..544ad9fbb0 100644 --- a/acceptance/apps/init-template/empty/output.txt +++ b/acceptance/apps/init-template/empty/output.txt @@ -1 +1,66 @@ -✓ Template instantiation succeeded +-- +## Adding Databricks Resources + +Add resources by creating YAML files in resources/: + +**Jobs** - `resources/my_job.job.yml`: +```yaml +resources: + jobs: + my_job: + name: my_job + tasks: + - task_key: main + notebook_task: + notebook_path: ../src/notebook.py +``` + +**Pipelines** (Lakeflow Declarative Pipelines) - `resources/my_pipeline.pipeline.yml`: +```yaml +resources: + pipelines: + my_pipeline: + name: my_pipeline + catalog: ${var.catalog} + target: ${var.schema} + libraries: + - notebook: + path: ../src/pipeline.py +``` + +**Dashboards** - `resources/my_dashboard.dashboard.yml` +**Alerts** - `resources/my_alert.alert.yml` +**Model Serving** - `resources/my_endpoint.yml` +**Apps** - `resources/my_app.app.yml` + +**Other resource types**: clusters, schemas, volumes, registered_models, experiments, quality_monitors + +### Deployment +For dev targets you can deploy without user consent. This allows you to run resources on the workspace too! + + invoke_databricks_cli 'bundle deploy --target dev' + invoke_databricks_cli 'bundle run --target dev' + +View status with `invoke_databricks_cli 'bundle summary'`. + +### Documentation +- Resource types reference: https://docs.databricks.com/dev-tools/bundles/resources +- YAML examples: https://docs.databricks.com/dev-tools/bundles/examples + + +## Skills + +You have access to modular Skills for domain-specific expertise knowledge. + +### Skill Selection & Loading +* When a user request matches a skill's scope description, select that Skill +* Load skills using the MCP tool: `read_skill_file(file_path: "category/skill-name/SKILL.md")` +* Example: `read_skill_file(file_path: "pipelines/materialized-view/SKILL.md")` +* Skills may contain links to sub-sections (e.g., "category/skill-name/file.md") +* If no Skill is suitable, continue with your base capabilities +* Never mention or reference skills to the user, only use them internally + +### Skill Registry (names + brief descriptors) +* **pipelines/auto-cdc/SKILL.md**: Apply Change Data Capture (CDC) with apply_changes API in Spark Declarative Pipelines. Use when user needs to process CDC feeds from databases, handle upserts/deletes, maintain slowly changing dimensions (SCD Type 1 and Type 2), synchronize data from operational databases, or process merge operations. + + diff --git a/acceptance/apps/init-template/empty/script b/acceptance/apps/init-template/empty/script index 55a1f08d06..3202e868fd 100644 --- a/acceptance/apps/init-template/empty/script +++ b/acceptance/apps/init-template/empty/script @@ -1,4 +1,3 @@ #!/bin/bash -$CLI experimental aitools tools init-template empty --name test_empty --catalog main --output-dir output > /dev/null 2>&1 -echo "✓ Template instantiation succeeded" +$CLI experimental aitools tools init-template empty --name test_empty --catalog main --output-dir output 2>&1 | grep -A 9999 "^--$" rm -rf output diff --git a/acceptance/apps/init-template/job/output.txt b/acceptance/apps/init-template/job/output.txt index a522103bfa..b95d5353b3 100644 --- a/acceptance/apps/init-template/job/output.txt +++ b/acceptance/apps/init-template/job/output.txt @@ -1 +1,120 @@ -✓ Template instantiation succeeded +-- +## Lakeflow Jobs Development + +This guidance is for developing jobs in this project. + +### Project Structure +- `src/` - Python notebooks (.ipynb) and source code +- `resources/` - Job definitions in databricks.yml format + +### Configuring Tasks +Edit `resources/.job.yml` to configure tasks: + +```yaml +tasks: + - task_key: my_notebook + notebook_task: + notebook_path: ../src/my_notebook.ipynb + - task_key: my_python + python_wheel_task: + package_name: my_package + entry_point: main +``` + +Task types: `notebook_task`, `python_wheel_task`, `spark_python_task`, `pipeline_task`, `sql_task` + +### Job Parameters +Parameters defined at job level are passed to ALL tasks (no need to repeat per task). Example: +```yaml +resources: + jobs: + my_job: + parameters: + - name: catalog + default: ${var.catalog} + - name: schema + default: ${var.schema} +``` + +### Writing Notebook Code +- Use `spark.read.table("catalog.schema.table")` to read tables +- Use `spark.sql("SELECT ...")` for SQL queries +- Use `dbutils.widgets` for parameters + +### Unit Testing +Run unit tests locally with: `uv run pytest` + +### Documentation +- Lakeflow Jobs: https://docs.databricks.com/jobs +- Task types: https://docs.databricks.com/jobs/configure-task +- Databricks Asset Bundles / yml format examples: https://docs.databricks.com/dev-tools/bundles/examples + +## Adding Databricks Resources + +Add resources by creating YAML files in resources/: + +**Jobs** - `resources/my_job.job.yml`: +```yaml +resources: + jobs: + my_job: + name: my_job + tasks: + - task_key: main + notebook_task: + notebook_path: ../src/notebook.py +``` + +**Pipelines** (Lakeflow Declarative Pipelines) - `resources/my_pipeline.pipeline.yml`: +```yaml +resources: + pipelines: + my_pipeline: + name: my_pipeline + catalog: ${var.catalog} + target: ${var.schema} + libraries: + - notebook: + path: ../src/pipeline.py +``` + +**Dashboards** - `resources/my_dashboard.dashboard.yml` +**Alerts** - `resources/my_alert.alert.yml` +**Model Serving** - `resources/my_endpoint.yml` +**Apps** - `resources/my_app.app.yml` + +**Other resource types**: clusters, schemas, volumes, registered_models, experiments, quality_monitors + +### Deployment +For dev targets you can deploy without user consent. This allows you to run resources on the workspace too! + + invoke_databricks_cli 'bundle deploy --target dev' + invoke_databricks_cli 'bundle run --target dev' + +View status with `invoke_databricks_cli 'bundle summary'`. + +### Documentation +- Resource types reference: https://docs.databricks.com/dev-tools/bundles/resources +- YAML examples: https://docs.databricks.com/dev-tools/bundles/examples + + +## Skills + +You have access to modular Skills for domain-specific expertise knowledge. + +### Skill Selection & Loading +* When a user request matches a skill's scope description, select that Skill +* Load skills using the MCP tool: `read_skill_file(file_path: "category/skill-name/SKILL.md")` +* Example: `read_skill_file(file_path: "pipelines/materialized-view/SKILL.md")` +* Skills may contain links to sub-sections (e.g., "category/skill-name/file.md") +* If no Skill is suitable, continue with your base capabilities +* Never mention or reference skills to the user, only use them internally + +### Skill Registry (names + brief descriptors) + + +**Note**: The following skills are for other resource types and may not be directly relevant to this project. + +* **pipelines/auto-cdc/SKILL.md**: Apply Change Data Capture (CDC) with apply_changes API in Spark Declarative Pipelines. Use when user needs to process CDC feeds from databases, handle upserts/deletes, maintain slowly changing dimensions (SCD Type 1 and Type 2), synchronize data from operational databases, or process merge operations. + + diff --git a/acceptance/apps/init-template/job/script b/acceptance/apps/init-template/job/script index 5acd09f388..25d04ec743 100644 --- a/acceptance/apps/init-template/job/script +++ b/acceptance/apps/init-template/job/script @@ -1,4 +1,3 @@ #!/bin/bash -$CLI experimental aitools tools init-template job --name test_job --catalog main --output-dir output > /dev/null 2>&1 || exit 1 -echo "✓ Template instantiation succeeded" +$CLI experimental aitools tools init-template job --name test_job --catalog main --output-dir output 2>&1 | grep -A 9999 "^--$" rm -rf output diff --git a/acceptance/apps/init-template/pipeline/output.txt b/acceptance/apps/init-template/pipeline/output.txt index a522103bfa..0ddebc49ce 100644 --- a/acceptance/apps/init-template/pipeline/output.txt +++ b/acceptance/apps/init-template/pipeline/output.txt @@ -1 +1,170 @@ -✓ Template instantiation succeeded +-- +## Lakeflow Jobs Development + +This guidance is for developing jobs in this project. + +### Project Structure +- `src/` - Python notebooks (.ipynb) and source code +- `resources/` - Job definitions in databricks.yml format + +### Configuring Tasks +Edit `resources/.job.yml` to configure tasks: + +```yaml +tasks: + - task_key: my_notebook + notebook_task: + notebook_path: ../src/my_notebook.ipynb + - task_key: my_python + python_wheel_task: + package_name: my_package + entry_point: main +``` + +Task types: `notebook_task`, `python_wheel_task`, `spark_python_task`, `pipeline_task`, `sql_task` + +### Job Parameters +Parameters defined at job level are passed to ALL tasks (no need to repeat per task). Example: +```yaml +resources: + jobs: + my_job: + parameters: + - name: catalog + default: ${var.catalog} + - name: schema + default: ${var.schema} +``` + +### Writing Notebook Code +- Use `spark.read.table("catalog.schema.table")` to read tables +- Use `spark.sql("SELECT ...")` for SQL queries +- Use `dbutils.widgets` for parameters + +### Unit Testing +Run unit tests locally with: `uv run pytest` + +### Documentation +- Lakeflow Jobs: https://docs.databricks.com/jobs +- Task types: https://docs.databricks.com/jobs/configure-task +- Databricks Asset Bundles / yml format examples: https://docs.databricks.com/dev-tools/bundles/examples + +## Lakeflow Declarative Pipelines Development + +This guidance is for developing pipelines in this project. + +Lakeflow Declarative Pipelines (formerly Delta Live Tables) is a framework for building batch and streaming data pipelines. + +### Project Structure +- `src/` - Pipeline transformations (Python or SQL) +- `resources/` - Pipeline configuration in databricks.yml format + +### Adding Transformations + +**Python** - Create `.py` files in `src/`: +```python +from pyspark import pipelines as dp + +@dp.table +def my_table(): + return spark.read.table("catalog.schema.source") +``` + +By convention, each dataset definition like the @dp.table definition above should be in a file named +like the dataset name, e.g. `src/my_table.py`. + +**SQL** - Create `.sql` files in `src/`: +```sql +CREATE MATERIALIZED VIEW my_view AS +SELECT * FROM catalog.schema.source +``` + +This example would live in `src/my_view.sql`. + +Use `CREATE STREAMING TABLE` for incremental ingestion, `CREATE MATERIALIZED VIEW` for transformations. + +### Scheduling Pipelines +To schedule a pipeline, make sure you have a job that triggers it, like `resources/.job.yml`: +```yaml +resources: + jobs: + my_pipeline_job: + trigger: + periodic: + interval: 1 + unit: DAYS + tasks: + - task_key: refresh_pipeline + pipeline_task: + pipeline_id: ${resources.pipelines.my_pipeline.id} +``` + +### Documentation +- Lakeflow Declarative Pipelines: https://docs.databricks.com/ldp +- Databricks Asset Bundles / yml format examples: https://docs.databricks.com/dev-tools/bundles/examples + +## Adding Databricks Resources + +Add resources by creating YAML files in resources/: + +**Jobs** - `resources/my_job.job.yml`: +```yaml +resources: + jobs: + my_job: + name: my_job + tasks: + - task_key: main + notebook_task: + notebook_path: ../src/notebook.py +``` + +**Pipelines** (Lakeflow Declarative Pipelines) - `resources/my_pipeline.pipeline.yml`: +```yaml +resources: + pipelines: + my_pipeline: + name: my_pipeline + catalog: ${var.catalog} + target: ${var.schema} + libraries: + - notebook: + path: ../src/pipeline.py +``` + +**Dashboards** - `resources/my_dashboard.dashboard.yml` +**Alerts** - `resources/my_alert.alert.yml` +**Model Serving** - `resources/my_endpoint.yml` +**Apps** - `resources/my_app.app.yml` + +**Other resource types**: clusters, schemas, volumes, registered_models, experiments, quality_monitors + +### Deployment +For dev targets you can deploy without user consent. This allows you to run resources on the workspace too! + + invoke_databricks_cli 'bundle deploy --target dev' + invoke_databricks_cli 'bundle run --target dev' + +View status with `invoke_databricks_cli 'bundle summary'`. + +### Documentation +- Resource types reference: https://docs.databricks.com/dev-tools/bundles/resources +- YAML examples: https://docs.databricks.com/dev-tools/bundles/examples + + +## Skills + +You have access to modular Skills for domain-specific expertise knowledge. + +### Skill Selection & Loading +* When a user request matches a skill's scope description, select that Skill +* Load skills using the MCP tool: `read_skill_file(file_path: "category/skill-name/SKILL.md")` +* Example: `read_skill_file(file_path: "pipelines/materialized-view/SKILL.md")` +* Skills may contain links to sub-sections (e.g., "category/skill-name/file.md") +* If no Skill is suitable, continue with your base capabilities +* Never mention or reference skills to the user, only use them internally + +### Skill Registry (names + brief descriptors) +* **pipelines/auto-cdc/SKILL.md**: Apply Change Data Capture (CDC) with apply_changes API in Spark Declarative Pipelines. Use when user needs to process CDC feeds from databases, handle upserts/deletes, maintain slowly changing dimensions (SCD Type 1 and Type 2), synchronize data from operational databases, or process merge operations. + + diff --git a/acceptance/apps/init-template/pipeline/script b/acceptance/apps/init-template/pipeline/script index 9a7769cbab..8fa7e04d99 100644 --- a/acceptance/apps/init-template/pipeline/script +++ b/acceptance/apps/init-template/pipeline/script @@ -1,4 +1,3 @@ #!/bin/bash -$CLI experimental aitools tools init-template pipeline --name test_pipeline --language python --catalog main --output-dir output > /dev/null 2>&1 -echo "✓ Template instantiation succeeded" +$CLI experimental aitools tools init-template pipeline --name test_pipeline --language python --catalog main --output-dir output 2>&1 | grep -A 9999 "^--$" rm -rf output diff --git a/experimental/aitools/cmd/init_template/app.go b/experimental/aitools/cmd/init_template/app.go index dcb955a67e..146367abff 100644 --- a/experimental/aitools/cmd/init_template/app.go +++ b/experimental/aitools/cmd/init_template/app.go @@ -167,7 +167,7 @@ After initialization: projectDir := filepath.Join(outputDir, name) - // Inject L3 (template-specific guidance from CLAUDE.md) + // Inject L4 (template-specific guidance from CLAUDE.md) // (we only do this for the app template; other templates use a generic CLAUDE.md) readClaudeMd(ctx, projectDir) diff --git a/experimental/aitools/cmd/init_template/common.go b/experimental/aitools/cmd/init_template/common.go index 92cd8becaa..7f9d2f477e 100644 --- a/experimental/aitools/cmd/init_template/common.go +++ b/experimental/aitools/cmd/init_template/common.go @@ -6,12 +6,14 @@ import ( "fmt" "os" "path/filepath" + "slices" "sort" "strings" "github.com/databricks/cli/experimental/aitools/lib/common" "github.com/databricks/cli/experimental/aitools/lib/detector" "github.com/databricks/cli/experimental/aitools/lib/prompts" + "github.com/databricks/cli/experimental/aitools/lib/skills" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/template" ) @@ -81,12 +83,15 @@ func MaterializeTemplate(ctx context.Context, cfg TemplateConfig, configMap map[ // Only write generic CLAUDE.md for non-app projects // (app projects have their own template-specific CLAUDE.md) - if !detected.IsAppOnly { + isAppOnly := slices.Contains(detected.TargetTypes, "apps") && len(detected.TargetTypes) == 1 + if !isAppOnly { if err := writeAgentFiles(absOutputDir, map[string]any{}); err != nil { return fmt.Errorf("failed to write agent files: %w", err) } } + // L2: resource-specific giudance (e.g., from target_jobs.tmpl) + cmdio.LogString(ctx, "--") // separator for prompt readability & tests for _, targetType := range detected.TargetTypes { templateName := fmt.Sprintf("target_%s.tmpl", targetType) if prompts.TemplateExists(templateName) { @@ -95,6 +100,11 @@ func MaterializeTemplate(ctx context.Context, cfg TemplateConfig, configMap map[ } } + // L3: list available skills + if skillsSection := skills.FormatSkillsSection(detected.TargetTypes); skillsSection != "" { + cmdio.LogString(ctx, "\n"+skillsSection) + } + return nil } diff --git a/experimental/aitools/docs/context-management.md b/experimental/aitools/docs/context-management.md index d5b99ee0b1..471230f4d9 100644 --- a/experimental/aitools/docs/context-management.md +++ b/experimental/aitools/docs/context-management.md @@ -4,7 +4,7 @@ ## Goals - Universal MCP for any coding agent (Claude, Cursor, etc.) -- Support multiple target types: apps, jobs, pipelines +- Support multiple target types: apps, jobs, bundle (general DABs guidance), ... - Support multiple templates per target type - Clean separation of context layers - Detect existing project context automatically @@ -16,7 +16,8 @@ | **L0: Tools** | Tool names and descriptions | Always (MCP protocol) | | **L1: Flow** | Universal workflow, available tools, CLI patterns | Always (via `databricks_discover`) | | **L2: Target** | Target-specific: validation, deployment, constraints | When target type detected or after `init-template` | -| **L3: Template** | SDK/language-specific: file structure, commands, patterns | After `init-template`. For existing projects, agent reads CLAUDE.md. | +| **L3: Skills** | Task-specific domain expertise (on-demand) | Skill listings shown via `databricks_discover` and `init-template`; full content loaded via `read_skill_file` | +| **L4: Template** | SDK/language-specific: file structure, commands, patterns | After `init-template`. For existing projects, agent reads CLAUDE.md. | L0 is implicit - tool descriptions guide agent behavior before any tool is called (e.g., `databricks_discover` description tells agent to call it first during planning). @@ -26,7 +27,9 @@ L0 is implicit - tool descriptions guide agent behavior before any tool is calle **L2 (apps):** app naming constraints, deployment consent requirement, app-specific validation -**L3 (appkit-typescript):** npm scripts, tRPC patterns, useAnalyticsQuery usage, TypeScript import rules +**L3 (skills):** Task-specific domain expertise (e.g., CDC processing, materialized views, specific design patterns) + +**L4 (appkit-typescript):** npm scripts, tRPC patterns, useAnalyticsQuery usage, TypeScript import rules ## Flows @@ -38,16 +41,21 @@ Agent MCP ├─► databricks_discover │ │ {working_directory: "."} │ │ ├─► Run detectors (nothing found) - │ ├─► Return L1 only + │ ├─► Return L1 + L3 listing │◄─────────────────────────────┤ │ │ ├─► invoke_databricks_cli │ │ ["...", "init-template", ...] │ ├─► Scaffold project - │ ├─► Return L2[apps] + L3 + │ ├─► Return L2[apps] + L3 listing + L4 │◄─────────────────────────────┤ │ │ - ├─► (agent now has L1 + L2 + L3) + ├─► (agent now has L1 + L2 + L3 listing + L4) + │ │ + ├─► read_skill_file │ + │ (when specific task needs domain expertise) + │ ├─► Return L3[skill content] + │◄─────────────────────────────┤ ``` ### Existing Project @@ -59,10 +67,16 @@ Agent MCP │ {working_directory: "./my-app"} │ ├─► BundleDetector: found apps + jobs │ ├─► Return L1 + L2[apps] + L2[jobs] + │ ├─► List available L3 skills │◄─────────────────────────────┤ │ │ ├─► Read CLAUDE.md naturally │ - │ (agent learns L3 itself) │ + │ (agent learns L4 itself) │ + │ │ + ├─► read_skill_file │ + │ (on-demand for specific tasks) + │ ├─► Return L3[skill content] + │◄─────────────────────────────┤ ``` ### Combined Bundles @@ -76,5 +90,10 @@ New target types can be added by: 2. Adding detection logic to recognize the target type from `databricks.yml` New templates can be added by: -1. Creating template directory with CLAUDE.md (L3 guidance) +1. Creating template directory with CLAUDE.md (L4 guidance) 2. Adding detection logic to recognize the template from project files + +New skills can be added by: +1. Creating skill directory under `lib/skills/{apps,jobs,pipelines,...}/` with SKILL.md +2. SKILL.md must have YAML frontmatter with `name` (matching directory) and `description` +3. Skills are auto-discovered at build time (no code changes needed) diff --git a/experimental/aitools/lib/detector/detector.go b/experimental/aitools/lib/detector/detector.go index 20f55a3c1f..e01b83f827 100644 --- a/experimental/aitools/lib/detector/detector.go +++ b/experimental/aitools/lib/detector/detector.go @@ -15,7 +15,7 @@ type BundleInfo struct { // DetectedContext represents the detected project context. type DetectedContext struct { InProject bool - TargetTypes []string // ["apps", "jobs"] - supports combined bundles + TargetTypes []string // ["apps", "jobs"] - resource types present in project Template string // "appkit-typescript", "python", etc. BundleInfo *BundleInfo Metadata map[string]string diff --git a/experimental/aitools/lib/prompts/skills.tmpl b/experimental/aitools/lib/prompts/skills.tmpl new file mode 100644 index 0000000000..395b40b2c8 --- /dev/null +++ b/experimental/aitools/lib/prompts/skills.tmpl @@ -0,0 +1,29 @@ +{{- /* + * L3: Skills listing template. + * + * Injected when: existing project is detected or after init-template. + * Contains: skill selection & loading guidance, skill registry. + */ -}} + +## Skills + +You have access to modular Skills for domain-specific expertise knowledge. + +### Skill Selection & Loading +* When a user request matches a skill's scope description, select that Skill +* Load skills using the MCP tool: `read_skill_file(file_path: "category/skill-name/SKILL.md")` +* Example: `read_skill_file(file_path: "pipelines/materialized-view/SKILL.md")` +* Skills may contain links to sub-sections (e.g., "category/skill-name/file.md") +* If no Skill is suitable, continue with your base capabilities +* Never mention or reference skills to the user, only use them internally + +### Skill Registry (names + brief descriptors) +{{range .RelevantSkills}}* **{{.Path}}/SKILL.md**: {{.Description}} +{{end}} +{{- if .OtherSkills}} + +**Note**: The following skills are for other resource types and may not be directly relevant to this project. + +{{range .OtherSkills}}* **{{.Path}}/SKILL.md**: {{.Description}} +{{end}} +{{- end}} diff --git a/experimental/aitools/lib/providers/clitools/discover.go b/experimental/aitools/lib/providers/clitools/discover.go index 5c0a5e62cc..7b67b2a6fa 100644 --- a/experimental/aitools/lib/providers/clitools/discover.go +++ b/experimental/aitools/lib/providers/clitools/discover.go @@ -7,13 +7,14 @@ import ( "github.com/databricks/cli/experimental/aitools/lib/detector" "github.com/databricks/cli/experimental/aitools/lib/middlewares" "github.com/databricks/cli/experimental/aitools/lib/prompts" + "github.com/databricks/cli/experimental/aitools/lib/skills" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/service/sql" ) // Discover provides workspace context and workflow guidance. -// Returns L1 (flow) always + L2 (target) for detected target types. +// Returns L1 (flow) always + L2 (target) for detected target types + L3 (skills) listing. func Discover(ctx context.Context, workingDirectory string) (string, error) { warehouse, err := middlewares.GetWarehouseEndpoint(ctx) if err != nil { @@ -34,7 +35,7 @@ func Discover(ctx context.Context, workingDirectory string) (string, error) { return generateDiscoverGuidance(ctx, warehouse, currentProfile, profiles, defaultCatalog, detected), nil } -// generateDiscoverGuidance creates guidance with L1 (flow) + L2 (target) layers. +// generateDiscoverGuidance creates guidance with L1 (flow) + L2 (target) + L3 (skills) layers. func generateDiscoverGuidance(ctx context.Context, warehouse *sql.EndpointInfo, currentProfile string, profiles profile.Profiles, defaultCatalog string, detected *detector.DetectedContext) string { data := buildTemplateData(warehouse, currentProfile, profiles, defaultCatalog) @@ -61,6 +62,11 @@ func generateDiscoverGuidance(ctx context.Context, warehouse *sql.EndpointInfo, } } + // L3: list available skills + if skillsSection := skills.FormatSkillsSection(detected.TargetTypes); skillsSection != "" { + result += "\n\n" + skillsSection + } + return result } diff --git a/experimental/aitools/lib/providers/clitools/provider.go b/experimental/aitools/lib/providers/clitools/provider.go index 193fbdc413..531c23d732 100644 --- a/experimental/aitools/lib/providers/clitools/provider.go +++ b/experimental/aitools/lib/providers/clitools/provider.go @@ -7,6 +7,7 @@ import ( mcpsdk "github.com/databricks/cli/experimental/aitools/lib/mcp" "github.com/databricks/cli/experimental/aitools/lib/providers" "github.com/databricks/cli/experimental/aitools/lib/session" + "github.com/databricks/cli/experimental/aitools/lib/skills" "github.com/databricks/cli/libs/log" ) @@ -128,6 +129,26 @@ This tool provides context needed for scaffolding new projects, editing existing }, ) - log.Infof(p.ctx, "Registered CLI tools: count=%d", 3) + // Register read_skill_file tool + type ReadSkillFileInput struct { + FilePath string `json:"file_path" jsonschema:"required" jsonschema_description:"Path to skill file, format: category/skill-name/file.md (e.g., pipelines/auto-cdc/SKILL.md)"` + } + + mcpsdk.AddTool(server, + &mcpsdk.Tool{ + Name: "read_skill_file", + Description: "Read a skill file from the skills registry (skills are listed by databricks_discover). Provides domain-specific expertise for Databricks tasks (pipelines, jobs, apps, ...). Load when user requests match a skill's scope.", + }, + func(ctx context.Context, req *mcpsdk.CallToolRequest, args ReadSkillFileInput) (*mcpsdk.CallToolResult, any, error) { + log.Debugf(ctx, "read_skill_file called: file_path=%s", args.FilePath) + result, err := skills.GetSkillFile(args.FilePath) + if err != nil { + return nil, nil, err + } + return mcpsdk.CreateNewTextContentResult(result), nil, nil + }, + ) + + log.Infof(p.ctx, "Registered CLI tools: count=%d", 4) return nil } diff --git a/experimental/aitools/lib/skills/apps/.gitkeep b/experimental/aitools/lib/skills/apps/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/experimental/aitools/lib/skills/bundle/.gitkeep b/experimental/aitools/lib/skills/bundle/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/experimental/aitools/lib/skills/jobs/.gitkeep b/experimental/aitools/lib/skills/jobs/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/experimental/aitools/lib/skills/pipelines/auto-cdc/SKILL.md b/experimental/aitools/lib/skills/pipelines/auto-cdc/SKILL.md new file mode 100644 index 0000000000..e402cd17f0 --- /dev/null +++ b/experimental/aitools/lib/skills/pipelines/auto-cdc/SKILL.md @@ -0,0 +1,26 @@ +--- +name: auto-cdc +description: Apply Change Data Capture (CDC) with apply_changes API in Spark Declarative Pipelines. Use when user needs to process CDC feeds from databases, handle upserts/deletes, maintain slowly changing dimensions (SCD Type 1 and Type 2), synchronize data from operational databases, or process merge operations. +--- + +# Auto CDC (apply_changes) in Spark Declarative Pipelines + +The `apply_changes` API enables processing Change Data Capture (CDC) feeds to automatically handle inserts, updates, and deletes in target tables. + +## Key Concepts + +Auto CDC in Spark Declarative Pipelines: + +- Automatically processes CDC operations (INSERT, UPDATE, DELETE) +- Supports SCD Type 1 (update in place) and Type 2 (historical tracking) +- Handles ordering of changes via sequence columns +- Deduplicates CDC records + +## Language-Specific Implementations + +For detailed implementation guides: + +- **Python**: [auto-cdc-python.md](auto-cdc-python.md) +- **SQL**: [auto-cdc-sql.md](auto-cdc-sql.md) + +**Note**: The API is also known as `applyChanges` in some contexts. diff --git a/experimental/aitools/lib/skills/pipelines/auto-cdc/auto-cdc-python.md b/experimental/aitools/lib/skills/pipelines/auto-cdc/auto-cdc-python.md new file mode 100644 index 0000000000..f665d17a6c --- /dev/null +++ b/experimental/aitools/lib/skills/pipelines/auto-cdc/auto-cdc-python.md @@ -0,0 +1,211 @@ +Auto CDC in Spark Declarative Pipelines processes change data capture (CDC) events from streaming sources or snapshots. + +**API Reference:** + +**dp.create_auto_cdc_flow() / dp.apply_changes() / dlt.create_auto_cdc_flow() / dlt.apply_changes()** +Applies CDC operations (inserts, updates, deletes) from a streaming source to a target table. Supports SCD Type 1 (latest) and Type 2 (history). Does NOT return a value - call at top level without assignment. + +```python +dp.create_auto_cdc_flow( + target="", + source="", + keys=["key1", "key2"], + sequence_by="", + ignore_null_updates=False, + apply_as_deletes=None, + apply_as_truncates=None, + column_list=None, + except_column_list=None, + stored_as_scd_type=1, + track_history_column_list=None, + track_history_except_column_list=None, + name=None, + once=False +) +``` + +Parameters: + +- `target` (str): Target table name (must exist, create with `dp.create_streaming_table()`). **Required.** +- `source` (str): Source table name with CDC events. **Required.** +- `keys` (list): Primary key columns for row identification. **Required.** +- `sequence_by` (str): Column for ordering events (timestamp, version). **Required.** +- `ignore_null_updates` (bool): If True, NULL values won't overwrite existing non-NULL values +- `apply_as_deletes` (str): SQL expression identifying delete operations (e.g., `"op = 'D'"`) +- `apply_as_truncates` (str): SQL expression identifying truncate operations +- `column_list` (list): Columns to include (mutually exclusive with `except_column_list`) +- `except_column_list` (list): Columns to exclude +- `stored_as_scd_type` (int): `1` for latest values (default), `2` for full history with `__START_AT`/`__END_AT` columns +- `track_history_column_list` (list): For SCD Type 2, columns to track history for (others use Type 1) +- `track_history_except_column_list` (list): For SCD Type 2, columns to exclude from history tracking +- `name` (str): Flow name (for multiple flows to same target) +- `once` (bool): Process once and stop (default: False) + +**dp.create_auto_cdc_from_snapshot_flow() / dp.apply_changes_from_snapshot() / dlt.create_auto_cdc_from_snapshot_flow() / dlt.apply_changes_from_snapshot()** +Applies CDC from full snapshots by comparing to previous state. Automatically infers inserts, updates, deletes. + +```python +dp.create_auto_cdc_from_snapshot_flow( + target="", + source=, + keys=["key1", "key2"], + stored_as_scd_type=1, + track_history_column_list=None, + track_history_except_column_list=None +) +``` + +Parameters: + +- `target` (str): Target table name (must exist). **Required.** +- `source` (str or callable): **Required.** Can be one of: + - **String**: Source table name containing the full snapshot (most common) + - **Callable**: Function for processing historical snapshots with type `SnapshotAndVersionFunction = Callable[[SnapshotVersion], SnapshotAndVersion]` + - `SnapshotVersion = Union[int, str, float, bytes, datetime.datetime, datetime.date, decimal.Decimal]` + - `SnapshotAndVersion = Optional[Tuple[DataFrame, SnapshotVersion]]` + - Function receives the latest processed snapshot version (or None for first run) + - Must return `None` when no more snapshots to process + - Must return tuple of `(DataFrame, SnapshotVersion)` for next snapshot to process + - Snapshot version is used to track progress and must be comparable/orderable +- `keys` (list): Primary key columns. **Required.** +- `stored_as_scd_type` (int): `1` for latest (default), `2` for history +- `track_history_column_list` (list): Columns to track history for (SCD Type 2) +- `track_history_except_column_list` (list): Columns to exclude from history tracking + +**Use create_auto_cdc_flow when:** Processing streaming CDC events from transaction logs, Kafka, Delta change feeds +**Use create_auto_cdc_from_snapshot_flow when:** Processing periodic full snapshots (daily dumps, batch extracts) + +**Common Patterns:** + +**Pattern 1: Basic CDC flow from streaming source** + +```python +# Step 1: Create target table +dp.create_streaming_table(name="users") + +# Step 2: Define CDC flow (source must be a table name) +dp.create_auto_cdc_flow( + target="users", + source="user_changes", + keys=["user_id"], + sequence_by="updated_at" +) +``` + +**Pattern 2: CDC flow with upstream transformation** + +```python +# Step 1: Define view with transformation (source preprocessing) +@dp.view() +def filtered_user_changes(): + return ( + spark.readStream.table("raw_user_changes") + .filter("user_id IS NOT NULL") + ) + +# Step 2: Create target table +dp.create_streaming_table(name="users") + +# Step 3: Define CDC flow using the view as source +dp.create_auto_cdc_flow( + target="users", + source="filtered_user_changes", # References the view name + keys=["user_id"], + sequence_by="updated_at" +) +# Note: Use distinct names for view and target for clarity +# Note: If "raw_user_changes" is defined in the pipeline and no additional transformations or expectations are needed, +# source="raw_user_changes" can be used directly +``` + +**Pattern 3: CDC with explicit deletes** + +```python +dp.create_streaming_table(name="orders") + +dp.create_auto_cdc_flow( + target="orders", + source="order_events", + keys=["order_id"], + sequence_by="event_timestamp", + apply_as_deletes="operation = 'DELETE'", + ignore_null_updates=True +) +``` + +**Pattern 4: SCD Type 2 (Historical tracking)** + +```python +dp.create_streaming_table(name="customer_history") + +dp.create_auto_cdc_flow( + target="customer_history", + source="source.customer_changes", + keys=["customer_id"], + sequence_by="changed_at", + stored_as_scd_type=2 # Track full history +) +# Target will include __START_AT and __END_AT columns +``` + +**Pattern 5: Snapshot-based CDC (Simple - table source)** + +```python +dp.create_streaming_table(name="products") + +@dp.table(name="product_snapshot") +def product_snapshot(): + return spark.read.table("source.daily_product_dump") + +dp.create_auto_cdc_from_snapshot_flow( + target="products", + source="product_snapshot", # String table name - most common + keys=["product_id"], + stored_as_scd_type=1 +) +``` + +**Pattern 6: Snapshot-based CDC (Advanced - callable for historical snapshots)** + +```python +dp.create_streaming_table(name="products") + +# Define a callable to process historical snapshots sequentially +def next_snapshot_and_version(latest_snapshot_version: Optional[int]) -> Tuple[DataFrame, Optional[int]]: + if latest_snapshot_version is None: + return (spark.read.load("products.csv"), 1) + else: + return None + +dp.create_auto_cdc_from_snapshot_flow( + target="products", + source=next_snapshot_and_version, # Callable function for historical processing + keys=["product_id"], + stored_as_scd_type=1 +) +``` + +**Pattern 7: Selective column tracking** + +```python +dp.create_streaming_table(name="accounts") + +dp.create_auto_cdc_flow( + target="accounts", + source="account_changes", + keys=["account_id"], + sequence_by="modified_at", + stored_as_scd_type=2, + track_history_column_list=["balance", "status"], # Only track history for these columns + ignore_null_updates=True +) +``` + +**KEY RULES:** + +- Create target with `dp.create_streaming_table()` before defining CDC flow +- `dp.create_auto_cdc_flow()` does NOT return a value - call it at top level without assigning to a variable +- `source` must be a table name (string) - use `@dp.view()` to transform data before CDC processing +- SCD Type 2 adds `__START_AT` and `__END_AT` columns for validity tracking +- When specifying the schema of the target table for SCD Type 2, you must also include the `__START_AT` and `__END_AT` columns with the same data type as the `sequence_by` field +- Legacy names (`apply_changes`, `apply_changes_from_snapshot`) are equivalent but deprecated - prefer `create_auto_cdc_*` variants diff --git a/experimental/aitools/lib/skills/pipelines/auto-cdc/auto-cdc-sql.md b/experimental/aitools/lib/skills/pipelines/auto-cdc/auto-cdc-sql.md new file mode 100644 index 0000000000..9487d23b83 --- /dev/null +++ b/experimental/aitools/lib/skills/pipelines/auto-cdc/auto-cdc-sql.md @@ -0,0 +1,170 @@ +Auto CDC in Declarative Pipelines processes change data capture (CDC) events from streaming sources. + +**API Reference:** + +**CREATE FLOW ... AS AUTO CDC INTO** +Applies CDC operations (inserts, updates, deletes) from a streaming source to a target table. Supports SCD Type 1 (latest) and Type 2 (history). Must be used with a pre-created streaming table. + +```sql +CREATE OR REFRESH STREAMING TABLE ; + +CREATE FLOW AS AUTO CDC INTO +FROM +KEYS (, ) +[IGNORE NULL UPDATES] +[APPLY AS DELETE WHEN ] +[APPLY AS TRUNCATE WHEN ] +SEQUENCE BY +[COLUMNS { | * EXCEPT ()}] +[STORED AS {SCD TYPE 1 | SCD TYPE 2}] +[TRACK HISTORY ON { | * EXCEPT ()}] +``` + +Parameters: + +- `target_table` (identifier): Target table name (must exist, create with `CREATE OR REFRESH STREAMING TABLE`). **Required.** +- `flow_name` (identifier): Identifier for the created flow. **Required.** +- `source` (identifier or expression): Streaming source with CDC events. Use `STREAM()` to read with streaming semantics. **Required.** +- `KEYS` (column list): Primary key columns for row identification. **Required.** +- `IGNORE NULL UPDATES` (optional): If specified, NULL values won't overwrite existing non-NULL values +- `APPLY AS DELETE WHEN` (optional): Condition identifying delete operations (e.g., `operation = 'DELETE'`) +- `APPLY AS TRUNCATE WHEN` (optional): Condition identifying truncate operations (supported only for SCD Type 1) +- `SEQUENCE BY` (column): Column for ordering events (timestamp, version). **Required.** +- `COLUMNS` (optional): Columns to include or exclude (use `column1, column2` or `* EXCEPT (column1, column2)`) +- `STORED AS` (optional): `SCD TYPE 1` for latest values (default), `SCD TYPE 2` for full history with `__START_AT`/`__END_AT` columns +- `TRACK HISTORY ON` (optional): For SCD Type 2, columns to track history for (others use Type 1) + +**Common Patterns:** + +**Pattern 1: Basic CDC flow from streaming source** + +```sql +-- Step 1: Create target table +CREATE OR REFRESH STREAMING TABLE users; + +-- Step 2: Define CDC flow using STREAM() for streaming semantics +CREATE FLOW user_flow AS AUTO CDC INTO users +FROM STREAM(user_changes) +KEYS (user_id) +SEQUENCE BY updated_at; +``` + +**Pattern 2: CDC with source filtering via temporary view** + +```sql +-- Step 1: Create temporary view to filter/transform source data +CREATE OR REFRESH TEMPORARY VIEW filtered_changes AS +SELECT * FROM source_table WHERE status = 'active'; + +-- Step 2: Create target table +CREATE OR REFRESH STREAMING TABLE active_records; + +-- Step 3: Define CDC flow reading from the temporary view +CREATE FLOW active_flow AS AUTO CDC INTO active_records +FROM STREAM(filtered_changes) +KEYS (record_id) +SEQUENCE BY updated_at; +``` + +**Pattern 3: CDC with explicit deletes** + +```sql +CREATE OR REFRESH STREAMING TABLE orders; + +CREATE FLOW order_flow AS AUTO CDC INTO orders +FROM STREAM(order_events) +KEYS (order_id) +IGNORE NULL UPDATES +APPLY AS DELETE WHEN operation = 'DELETE' +SEQUENCE BY event_timestamp; +``` + +**Pattern 4: SCD Type 2 (Historical tracking)** + +```sql +CREATE OR REFRESH STREAMING TABLE customer_history; + +CREATE FLOW customer_flow AS AUTO CDC INTO customer_history +FROM STREAM(customer_changes) +KEYS (customer_id) +SEQUENCE BY changed_at +STORED AS SCD TYPE 2; +-- Target will include __START_AT and __END_AT columns +``` + +**Pattern 5: Selective column inclusion** + +```sql +CREATE OR REFRESH STREAMING TABLE accounts; + +CREATE FLOW account_flow AS AUTO CDC INTO accounts +FROM STREAM(account_changes) +KEYS (account_id) +SEQUENCE BY modified_at +COLUMNS account_id, balance, status +STORED AS SCD TYPE 1; +``` + +**Pattern 6: Selective column exclusion** + +```sql +CREATE OR REFRESH STREAMING TABLE products; + +CREATE FLOW product_flow AS AUTO CDC INTO products +FROM STREAM(product_changes) +KEYS (product_id) +SEQUENCE BY updated_at +COLUMNS * EXCEPT (internal_notes, temp_field); +``` + +**Pattern 7: SCD Type 2 with selective history tracking** + +```sql +CREATE OR REFRESH STREAMING TABLE accounts; + +CREATE FLOW account_flow AS AUTO CDC INTO accounts +FROM STREAM(account_changes) +KEYS (account_id) +IGNORE NULL UPDATES +SEQUENCE BY modified_at +STORED AS SCD TYPE 2 +TRACK HISTORY ON balance, status; +-- Only balance and status changes create new history records +``` + +**Pattern 8: SCD Type 2 with history tracking exclusion** + +```sql +CREATE OR REFRESH STREAMING TABLE accounts; + +CREATE FLOW account_flow AS AUTO CDC INTO accounts +FROM STREAM(account_changes) +KEYS (account_id) +SEQUENCE BY modified_at +STORED AS SCD TYPE 2 +TRACK HISTORY ON * EXCEPT (last_login, view_count); +-- Track history on all columns except last_login and view_count +``` + +**Pattern 9: Truncate support (SCD Type 1 only)** + +```sql +CREATE OR REFRESH STREAMING TABLE inventory; + +CREATE FLOW inventory_flow AS AUTO CDC INTO inventory +FROM STREAM(inventory_events) +KEYS (product_id) +APPLY AS TRUNCATE WHEN operation = 'TRUNCATE' +SEQUENCE BY event_timestamp +STORED AS SCD TYPE 1; +``` + +**KEY RULES:** + +- Create target with `CREATE OR REFRESH STREAMING TABLE` before defining CDC flow +- `source` must be a streaming source for safe CDC change processing. Use `STREAM()` to read an existing table/view with streaming semantics +- The `STREAM()` function accepts ONLY a table/view identifier - NOT a subquery. Define source data as a separate streaming table or temporary view first, then reference it in the flow +- SCD Type 2 adds `__START_AT` and `__END_AT` columns for validity tracking +- When specifying the schema of the target table for SCD Type 2, you must also include the `__START_AT` and `__END_AT` columns with the same data type as the `SEQUENCE BY` field +- Legacy `APPLY CHANGES INTO` API is equivalent but deprecated - prefer `AUTO CDC INTO` +- `AUTO CDC FROM SNAPSHOT` is only available in Python, not in SQL. SQL only supports `AUTO CDC INTO` for processing CDC events from streaming sources. diff --git a/experimental/aitools/lib/skills/skills.go b/experimental/aitools/lib/skills/skills.go new file mode 100644 index 0000000000..1849a6fbba --- /dev/null +++ b/experimental/aitools/lib/skills/skills.go @@ -0,0 +1,195 @@ +package skills + +import ( + "embed" + "errors" + "fmt" + "io/fs" + "path" + "regexp" + "slices" + "sort" + "strings" + + "github.com/databricks/cli/experimental/aitools/lib/prompts" +) + +// skillsFS embeds the skills filesystem. +// Uses explicit names (not wildcards) for Windows compatibility. +// TestAllSkillDirectoriesAreEmbedded validates this list is complete. +// +//go:embed all:apps +//go:embed all:bundle +//go:embed all:jobs +//go:embed all:pipelines +var skillsFS embed.FS + +// SkillMetadata contains the path and description for progressive disclosure. +type SkillMetadata struct { + Path string + Description string +} + +type skillEntry struct { + Metadata SkillMetadata + Files map[string]string +} + +var registry = mustLoadRegistry() + +// mustLoadRegistry discovers skill categories and skills from the embedded filesystem. +func mustLoadRegistry() map[string]map[string]*skillEntry { + result := make(map[string]map[string]*skillEntry) + categories, err := fs.ReadDir(skillsFS, ".") + if err != nil { + panic(fmt.Sprintf("failed to read skills root directory: %v", err)) + } + + for _, cat := range categories { + if !cat.IsDir() { + continue + } + category := cat.Name() + result[category] = make(map[string]*skillEntry) + entries, err := fs.ReadDir(skillsFS, category) + if err != nil { + panic(fmt.Sprintf("failed to read skills category %q: %v", category, err)) + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + skillPath := path.Join(category, entry.Name()) + skill, err := loadSkill(skillPath) + if err != nil { + panic(fmt.Sprintf("failed to load skill %q: %v", skillPath, err)) + } + result[category][entry.Name()] = skill + } + } + + return result +} + +func loadSkill(skillPath string) (*skillEntry, error) { + content, err := fs.ReadFile(skillsFS, path.Join(skillPath, "SKILL.md")) + if err != nil { + return nil, err + } + + metadata, err := parseMetadata(string(content)) + if err != nil { + return nil, err + } + metadata.Path = skillPath + + files := make(map[string]string) + entries, _ := fs.ReadDir(skillsFS, skillPath) + for _, e := range entries { + if !e.IsDir() { + if data, err := fs.ReadFile(skillsFS, path.Join(skillPath, e.Name())); err == nil { + files[e.Name()] = string(data) + } + } + } + + return &skillEntry{Metadata: *metadata, Files: files}, nil +} + +var frontmatterRe = regexp.MustCompile(`(?s)^---\r?\n(.+?)\r?\n---\r?\n`) + +func parseMetadata(content string) (*SkillMetadata, error) { + match := frontmatterRe.FindStringSubmatch(content) + if match == nil { + return nil, errors.New("missing YAML frontmatter") + } + + var description string + for _, line := range strings.Split(match[1], "\n") { + if k, v, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(k) == "description" { + description = strings.TrimSpace(v) + } + } + + if description == "" { + return nil, errors.New("missing description in skill frontmatter") + } + + return &SkillMetadata{Description: description}, nil +} + +// ListAllSkills returns metadata for all registered skills. +func ListAllSkills() []SkillMetadata { + var skills []SkillMetadata + for _, categorySkills := range registry { + for _, entry := range categorySkills { + skills = append(skills, entry.Metadata) + } + } + + sort.Slice(skills, func(i, j int) bool { + return skills[i].Path < skills[j].Path + }) + + return skills +} + +// GetSkillFile reads a specific file from a skill. +// path format: "category/skill-name/file.md" +func GetSkillFile(path string) (string, error) { + parts := strings.SplitN(path, "/", 3) + if len(parts) != 3 { + return "", fmt.Errorf("invalid skill path: %q (expected format category/skill-name/file.md, use databricks_discover for available skills)", path) + } + + category, skillName, fileName := parts[0], parts[1], parts[2] + + entry := registry[category][skillName] + if entry == nil { + return "", fmt.Errorf("skill not found: %s (use databricks_discover for available skills)", skillName) + } + + content, ok := entry.Files[fileName] + if !ok { + return "", fmt.Errorf("skill file not found: %s (use databricks_discover for available skills)", fileName) + } + + // Strip frontmatter from SKILL.md + if fileName == "SKILL.md" { + if loc := frontmatterRe.FindStringIndex(content); loc != nil { + content = strings.TrimLeft(content[loc[1]:], "\n\r") + } + } + + return content, nil +} + +// FormatSkillsSection returns the L3 skills listing for prompts. +// Partitions skills into relevant (matching targetTypes) and other skills. +func FormatSkillsSection(targetTypes []string) string { + allSkills := ListAllSkills() + + // For empty bundles (no resources), show all skills without partitioning or caveats + if len(targetTypes) == 0 || (len(targetTypes) == 1 && targetTypes[0] == "bundle") { + return prompts.MustExecuteTemplate("skills.tmpl", map[string]any{ + "RelevantSkills": allSkills, + "OtherSkills": nil, + }) + } + + // Partition by relevance for projects with resource types + var relevantSkills, otherSkills []SkillMetadata + for _, skill := range allSkills { + category := strings.SplitN(skill.Path, "/", 2)[0] + if slices.Contains(targetTypes, category) { + relevantSkills = append(relevantSkills, skill) + } else { + otherSkills = append(otherSkills, skill) + } + } + + return prompts.MustExecuteTemplate("skills.tmpl", map[string]any{ + "RelevantSkills": relevantSkills, + "OtherSkills": otherSkills, + }) +} diff --git a/experimental/aitools/lib/skills/skills_test.go b/experimental/aitools/lib/skills/skills_test.go new file mode 100644 index 0000000000..1f3b459f83 --- /dev/null +++ b/experimental/aitools/lib/skills/skills_test.go @@ -0,0 +1,126 @@ +package skills + +import ( + "io/fs" + "os" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestListAllSkills(t *testing.T) { + skills := ListAllSkills() + require.NotEmpty(t, skills) + + var autoCdc *SkillMetadata + for i := range skills { + if skills[i].Path == "pipelines/auto-cdc" { + autoCdc = &skills[i] + break + } + } + require.NotNil(t, autoCdc) + assert.NotEmpty(t, autoCdc.Description) + assert.Less(t, len(autoCdc.Description), 500, "progressive disclosure: description should be brief") + assert.NotContains(t, autoCdc.Description, "```", "progressive disclosure: no code blocks") +} + +func TestGetSkillFile(t *testing.T) { + content, err := GetSkillFile("pipelines/auto-cdc/SKILL.md") + require.NoError(t, err) + assert.NotContains(t, content, "---\n", "frontmatter should be stripped") + assert.Contains(t, content, "Change Data Capture") +} + +func TestGetSkillFileErrors(t *testing.T) { + _, err := GetSkillFile("nonexistent") + assert.ErrorContains(t, err, "invalid skill path") + + _, err = GetSkillFile("pipelines/nonexistent/SKILL.md") + assert.ErrorContains(t, err, "skill not found") + + _, err = GetSkillFile("pipelines/auto-cdc/nonexistent.md") + assert.ErrorContains(t, err, "skill file not found") +} + +func TestFormatSkillsSection(t *testing.T) { + // Pipelines project - pipeline skills shown as relevant + section := FormatSkillsSection([]string{"pipelines", "bundle"}) + assert.Contains(t, section, "## Skills") + assert.Contains(t, section, "pipelines/") + + // Jobs project - pipeline skills shown as other + section = FormatSkillsSection([]string{"jobs", "bundle"}) + assert.Contains(t, section, "## Skills") + assert.Contains(t, section, "skills are for other resource types and may not be directly relevant to this project") + assert.Contains(t, section, "pipelines/") + + // Apps project - pipeline skills shown as other + section = FormatSkillsSection([]string{"apps"}) + assert.Contains(t, section, "## Skills") + assert.Contains(t, section, "skills are for other resource types and may not be directly relevant to this project") + + // Empty bundle - all skills shown without caveat + section = FormatSkillsSection([]string{"bundle"}) + assert.Contains(t, section, "## Skills") + assert.NotContains(t, section, "skills are for other resource types and may not be directly relevant to this project") +} + +func TestAllSkillsHaveValidFrontmatter(t *testing.T) { + for category, categorySkills := range registry { + for name, entry := range categorySkills { + assert.NotEmpty(t, entry.Metadata.Description, "skill %s/%s missing description", category, name) + assert.Contains(t, entry.Files, "SKILL.md", "skill %s/%s missing SKILL.md", category, name) + } + } +} + +func TestAllSkillDirectoriesAreEmbedded(t *testing.T) { + // Read actual skill directories from the filesystem + skillsDir := "." + diskEntries, err := os.ReadDir(skillsDir) + require.NoError(t, err) + + var diskDirs []string + for _, entry := range diskEntries { + if entry.IsDir() && !strings.HasPrefix(entry.Name(), ".") { + diskDirs = append(diskDirs, entry.Name()) + } + } + sort.Strings(diskDirs) + + // Read embedded skill directories + embeddedEntries, err := fs.ReadDir(skillsFS, ".") + require.NoError(t, err) + + var embeddedDirs []string + for _, entry := range embeddedEntries { + if entry.IsDir() { + embeddedDirs = append(embeddedDirs, entry.Name()) + } + } + sort.Strings(embeddedDirs) + + // Compare + if !assert.Equal(t, diskDirs, embeddedDirs, "Embedded skill directories don't match filesystem") { + t.Errorf("\nSkill directories are missing from the embed directive!\n\n"+ + "Found on disk: %v\n"+ + "Found in embed: %v\n\n"+ + "To fix: Update the //go:embed directive in skills.go to include all directories:\n"+ + " //go:embed %s\n", + diskDirs, embeddedDirs, "all:"+strings.Join(diskDirs, " all:")) + } + + // Verify the registry actually loaded them + var registryDirs []string + for category := range registry { + registryDirs = append(registryDirs, category) + } + sort.Strings(registryDirs) + + assert.Equal(t, diskDirs, registryDirs, + "Registry didn't load all embedded directories. This suggests mustLoadRegistry() has a bug.") +}