diff --git a/.github/workflows/preview-build.yml b/.github/workflows/preview-build.yml index ab8cf2367..41d04b4e0 100644 --- a/.github/workflows/preview-build.yml +++ b/.github/workflows/preview-build.yml @@ -404,6 +404,7 @@ jobs: !cancelled() && github.repository != 'elastic/docs-builder' && steps.docs-build.outputs.skip != 'true' + && steps.docs-build.outcome != 'skipped' && ( steps.deployment.outputs.result || ( @@ -421,6 +422,7 @@ jobs: !cancelled() && github.repository == 'elastic/docs-builder' && steps.docs-build.outputs.skip != 'true' + && steps.docs-build.outcome != 'skipped' && ( steps.deployment.outputs.result || ( @@ -437,6 +439,7 @@ jobs: !cancelled() && steps.docs-build.outputs.skip != 'true' && steps.deployment.outputs.result + && steps.docs-build.outcome == 'success' - name: Upload to S3 id: s3-upload if: > @@ -444,6 +447,7 @@ jobs: && !cancelled() && steps.docs-build.outputs.skip != 'true' && steps.deployment.outputs.result + && steps.docs-build.outcome == 'success' run: | aws s3 sync .artifacts/docs/html "s3://elastic-docs-v3-website-preview${PATH_PREFIX}" --delete --no-follow-symlinks aws cloudfront create-invalidation \ diff --git a/docs/_docset.yml b/docs/_docset.yml index dd11ae52c..e2bd86d51 100644 --- a/docs/_docset.yml +++ b/docs/_docset.yml @@ -46,26 +46,19 @@ toc: - file: branching-strategy.md - file: add-repo.md - file: release-new-version.md - - folder: migration + - folder: building-blocks children: - file: index.md - - folder: freeze - children: - - file: index.md - - file: gh-action.md - - file: syntax.md - - file: ia.md - - file: versioning.md - - file: engineering.md - - folder: guide - children: - - file: index.md - - file: working-in-docs-content.md - - file: automated.md - - file: tooling.md - - file: move-ref-docs.md - - file: mapping.md - - file: how-to-set-up-docs-previews.md + - file: documentation-set.md + - file: assembled-documentation.md + - file: distributed-documentation.md + - file: link-service.md + - file: link-index.md + - file: link-catalog.md + - file: outbound-crosslinks.md + - file: inbound-crosslinks.md + - file: documentation-set-navigation.md + - file: global-navigation.md - folder: configure children: - file: index.md @@ -74,9 +67,9 @@ toc: - file: index.md - file: content.md - file: navigation.md + - file: products.md - file: versions.md - file: legacy-url-mappings.md - - file: products.md - folder: content-set children: - file: index.md @@ -121,6 +114,57 @@ toc: - file: tabs.md - file: tagged_regions.md - file: titles.md + - folder: cli + children: + - file: index.md + - folder: docset + children: + - file: index.md + - file: build.md + - file: diff-validate.md + - file: index-command.md + - file: mv.md + - file: serve.md + - folder: assembler + children: + - file: index.md + - file: assemble.md + - file: assembler-bloom-filter-create.md + - file: assembler-bloom-filter-lookup.md + - file: assembler-build.md + - file: assembler-clone.md + - file: assembler-config-init.md + - file: assembler-content-source-match.md + - file: assembler-content-source-validate.md + - file: assembler-deploy-apply.md + - file: assembler-deploy-plan.md + - file: assembler-deploy-update-redirects.md + - file: assembler-index.md + - file: assembler-navigation-validate.md + - file: assembler-navigation-validate-link-reference.md + - file: assembler-serve.md + - folder: links + children: + - file: index.md + - file: inbound-links-validate.md + - file: inbound-links-validate-all.md + - file: inbound-links-validate-link-reference.md + - folder: migration + children: + - file: index.md + - file: syntax.md + - file: ia.md + - file: versioning.md + - file: engineering.md + - folder: guide + children: + - file: index.md + - file: working-in-docs-content.md + - file: automated.md + - file: tooling.md + - file: move-ref-docs.md + - file: mapping.md + - file: how-to-set-up-docs-previews.md # nested TOCs are only allowed from docset.yml by default # to prevent them from being nested deeply arbitrarily # use max_toc_depth to allow deeper nesting (Expert mode, consult with docs team) diff --git a/docs/_redirects.yml b/docs/_redirects.yml index e3ed6e726..df2983983 100644 --- a/docs/_redirects.yml +++ b/docs/_redirects.yml @@ -1,4 +1,6 @@ redirects: + 'migration/freeze/gh-action.md' : 'index.md' + 'migration/freeze/index.md' : 'index.md' 'testing/redirects/4th-page.md': 'testing/redirects/5th-page.md' 'testing/redirects/9th-page.md': '!testing/redirects/5th-page.md' 'testing/redirects/6th-page.md': diff --git a/docs/building-blocks/assembled-documentation.md b/docs/building-blocks/assembled-documentation.md new file mode 100644 index 000000000..33020aaa1 --- /dev/null +++ b/docs/building-blocks/assembled-documentation.md @@ -0,0 +1,50 @@ +--- +navigation_title: Assembled Documentation +--- + +# Assembled Documentation + +**Assembled documentation** is the product of building many [documentation sets](documentation-set.md) and weaving them into a global navigation to produce the fully assembled documentation site. + +## How it works + +The assembler: + +1. Clones multiple documentation repositories +2. Reads its [configuration files](../configure/site/index.md) and builds [a global navigation](../configure/site/navigation.md) +3. Builds each documentation set independently using the global configuration and navigation to inform path prefixes +4. Produces a unified documentation website + +## Configuration + +Assembled documentation is configured through the site configuration, which defines: + +* [assembler.yml](../configure/site/index.md) Which repositories to include and [their branching strategy](../contribute/branching-strategy.md) +* [navigation.yml](../configure/site/index.md) Navigation and url prefixes for TOC's. +* [versions.yml](../configure/site/versions.md) Defines the various versioning schemes of products/solutions being documented +* [products.yml](../configure/site/products.md) Defines the product catalog (id, name) and ties it to a specific versioning scheme + +See [Site Configuration](../configure/site/index.md) for complete details on configuring assembled documentation. + +Important to note that `docs-builder` makes no assumptions about how repositories, products, solutions and versions tie into each other. + +## Build process + +The typical build process for assembled documentation: + +1. **Clone** - Clone all configured repositories using `docs-builder assembler clone` +2. **Build** - Build all documentation sets using `docs-builder assembler build` +3. **Serve** - Serve the documentation on http://localhost:4000 using `docs-builder assembler serve` + +This uses the embedded configuration files inside the `docs-builder` binary. To build a specific configuration: + +1. **Init Config** - Fetch the latest config to `$(pwd)/config` `docs-builder assembler config init --local` +2. **Clone** - Clone all configured repositories using `docs-builder assembler clone --local` +3. **Build** - Build all documentation sets using `docs-builder assembler build --local` +4. **Serve** - Serve the documentation on http://localhost:4000 using `docs-builder assembler serve` + +## Related concepts + +* [Documentation Set](documentation-set.md) - The individual units being assembled +* [Distributed Documentation](distributed-documentation.md) - How documentation sets work independently +* [Link Catalog](link-catalog.md) - How the assembler knows what to include diff --git a/docs/building-blocks/distributed-documentation.md b/docs/building-blocks/distributed-documentation.md new file mode 100644 index 000000000..8258e004f --- /dev/null +++ b/docs/building-blocks/distributed-documentation.md @@ -0,0 +1,63 @@ +--- +navigation_title: Distributed Documentation +--- + +# Distributed Documentation + +**Distributed documentation** is the architectural approach that allows repositories to build their own documentation independently. + +## Purpose + +The separation between building individual documentation sets and assembling them enables distributed builds, where: + +* Each repository builds its own documentation independently +* Builds don't block each other +* Teams maintain full autonomy over their documentation +* Cross-repository links are validated without requiring synchronized builds + +## How it works + +### Link Index publication + +Each time a documentation set is built successfully on its default integration branch, it produces and publishes a `links.json` file ([Link Index](link-index.md)) to a central location ([Link Service](link-service.md)). + +This Link Index contains all the linkable resources in that repository at that specific commit. + +### Example + +For instance, [Elasticsearch's links.json](https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/elasticsearch/main/links.json) represents all linkable resources in the Elasticsearch repository's main branch. + +## Benefits + +This distributed approach provides several key advantages: + +### Link validation + +* **Outbound links** - Validate links to other repositories ahead of time, even during local `docs-builder` builds +* **Inbound links** - Know when changes to your documentation would break links from other repositories + +### Resilient builds + +* **Isolation** - Documentation errors in one repository won't affect builds of other repositories +* **Fallback mechanism** - When a repository has build failures on its integration branch, the assembler falls back to the last known good commit +* **Snapshot builds** - Assembled builds only use commits that successfully produced a Link Index + +### Independent iteration + +* Teams can iterate on their documentation independently +* No coordination required for routine updates +* Faster feedback loops for documentation changes + +## Architecture components + +The distributed documentation system relies on several key components: + +* [Link Index](link-index.md) - Per-repository file of linkable resources +* [Link Service](link-service.md) - Central storage for Link Index files +* [Link Catalog](link-catalog.md) - Catalog of all available Link Index files +* [Outbound Crosslinks](outbound-crosslinks.md) - Links from one repository to another +* [Inbound Crosslinks](inbound-crosslinks.md) - Links from other repositories to yours + +## Local development + +Even during local development, `docs-builder` can access the [Link Service](link-service.md) to validate cross-repository links without requiring you to clone and build all related repositories. diff --git a/docs/building-blocks/documentation-set-navigation.md b/docs/building-blocks/documentation-set-navigation.md new file mode 100644 index 000000000..3adf88b50 --- /dev/null +++ b/docs/building-blocks/documentation-set-navigation.md @@ -0,0 +1,236 @@ +--- +navigation_title: Documentation Set Navigation +--- + +# Documentation Set Navigation + +**Documentation set navigation** defines how files within a single documentation set are organized and structured. Each documentation set is responsible for its own internal navigation hierarchy. + +## Purpose + +Documentation set navigation allows repository maintainers to: + +* **Organize content** - Define the logical structure of their documentation +* **Control hierarchy** - Determine which pages are nested under others +* **Create sections** - Group related content together +* **Maintain autonomy** - Structure documentation independently of other repositories + +## Basic structure + +Navigation is defined in the `toc` (table of contents) section of the `docset.yml` file: + +```yaml +toc: + - file: index.md + - folder: contribute + children: + - file: index.md + - file: locally.md + children: + - file: page.md +``` + +## TOC node types + +The `toc` section supports several node types: + +### File nodes + +Reference a single markdown file: + +```yaml +- file: getting-started.md +``` + +Files can also have nested children: + +```yaml +- file: locally.md + children: + - file: page.md +``` + +### Folder nodes + +Group related files together: + +```yaml +- folder: configuration + children: + - file: index.md + - file: basic.md +``` + +`children` is optional for `folder` nodes. This will include all files in the folder. +This is especially useful during development when you are unsure how to structure your documentation. + +Once `children` is defined, it must reference all `.md` files in the folder. The build will fail if +it detects any dangling documentation files. + +### Hidden files + +Hide pages from navigation while keeping them accessible: + +```yaml +- hidden: deprecated-page.md +``` + +### Named TOC sections + +For larger documentation sets, create named TOC sections that can be referenced in [global navigation](global-navigation.md): + +```yaml +toc: + - file: index.md + - toc: development +``` + +This will include the `toc` section defined in `development/toc.yml`:` + +## Dedicated toc.yml files + +When a `toc` section becomes too unwieldy, folders can define a dedicated `toc.yml` file to organize their files and link them in their parent `toc.yml` or `docset.yml` file. + +### Example with nested TOC + +In your `docset.yml`: + +```yaml +toc: + - file: index.md + - folder: contribute + children: + - file: index.md + - file: locally.md + children: + - file: page.md + - toc: development +``` + +Then create `development/toc.yml`: + +```yaml +toc: + - file: index.md + - toc: link-validation +``` + +:::{note} +The folder name `development` is not repeated in the `toc.yml` file. This allows for easier renames of the folder itself. +::: + +### Benefits of separate toc.yml files + +* **Modularity** - Each section can be maintained independently +* **Cleaner docset.yml** - Keep the main file focused and readable +* **Easier refactoring** - Rename folders without updating TOC references +* **Team ownership** - Different teams can manage different TOC sections + +## File paths + +All file paths in the `toc` section are relative to the documentation set root (where `docset.yml` is located): + +```yaml +toc: + - file: index.md # docs/index.md + - folder: api + children: + - file: index.md # docs/api/index.md + - file: authentication.md # docs/api/authentication.md +``` + +## Navigation metadata + +You can customize how items appear in the navigation: + +### Custom titles + +The navigation title defaults to a markdown's page first `h1` heading. + +To present the file differently in the navigation, add a `navigation_title` metadata field. + +```markdown +--- +title: Getting Started with the Documentation Builder +navigation_title: Quick Start +--- +``` + +There is no way to set `title` in the `docset.yml` file. This is by design to keep a page's data +contained in its file. + +## Relationship to global navigation + +When building [assembled documentation](assembled-documentation.md), the documentation set navigation becomes a component of the [global navigation](global-navigation.md): + +* **Documentation set navigation** defines the structure **within** a repository +* **Global navigation** defines **how repositories are organized** relative to each other + +Named `toc` sections in `docset.yml` can be referenced and reorganized in the global `navigation.yml` file without affecting the documentation set's internal structure. + +## Best practices + +### Keep it organized + +* Group related content in folders +* Use descriptive folder and file names +* Maintain a logical hierarchy + +The folder names and hierarchy are reflected directly in the URL structure. + +### Use index files + +Always include an `index.md` in folders: + +```yaml +- folder: api + children: + - file: index.md # Overview of API documentation + - file: endpoints.md + - file: authentication.md +``` + +### Limit nesting depth + +Avoid deeply nested structures (more than three to four levels) to maintain navigation clarity. + +### Use toc.yml for large sections + +When a section contains many files or becomes complex, extract it to a dedicated `toc.yml`: + +``` +docs/ +├── docset.yml +├── index.md +└── development/ + ├── toc.yml # Define development section structure here + ├── index.md + └── link-validation/ + └── toc.yml # Nested TOC section +``` + +### Name TOC sections meaningfully + +Use clear, descriptive names for TOC sections: + +**Good:** +```yaml +- toc: api-reference +- toc: getting-started +- toc: troubleshooting +``` + +**Bad:** +```yaml +- toc: section1 +- toc: misc +- toc: other +``` + +These names will end up in the URL structure of the published documentation + +## Related concepts + +* [Global Navigation](global-navigation.md) - How documentation sets are organized in assembled documentation +* [Content Set Configuration](../configure/content-set/index.md) - Complete `docset.yml` reference +* [Navigation Configuration](../configure/content-set/navigation.md) - Detailed navigation options diff --git a/docs/building-blocks/documentation-set.md b/docs/building-blocks/documentation-set.md new file mode 100644 index 000000000..5ed4dc2a5 --- /dev/null +++ b/docs/building-blocks/documentation-set.md @@ -0,0 +1,49 @@ +--- +navigation_title: Documentation Set +--- + +# Documentation Set + +A **documentation set** is a single folder containing the documentation of a single repository. This is the fundamental unit of documentation in the docs-builder system. + +## Minimum requirements + +At a minimum, a documentation set folder must contain: + +* `docset.yml` - The configuration file that defines the structure and metadata of the documentation set +* `index.md` - The entry point or landing page for the documentation set + +## Purpose + +Documentation sets allow each repository to maintain its own documentation independently. Each set can be: + +* Built independently +* Versioned separately +* Maintained by different teams +* Published to its own schedule + +## Structure + +A typical documentation set might look like: + +``` +my-repo/ +└── docs/ + ├── docset.yml + ├── index.md + ├── getting-started.md + ├── configuration/ + │ ├── index.md + │ └── advanced.md + └── reference/ + └── api.md +``` + +## Configuration + +The `docset.yml` file controls how the documentation set is structured and built. See [Content Set Configuration](../configure/content-set/index.md) for complete configuration details. + +## Related concepts + +* [Assembled Documentation](assembled-documentation.md) - How multiple documentation sets are combined +* [Link Index](link-index.md) - How documentation sets publish their linkable resources diff --git a/docs/building-blocks/global-navigation.md b/docs/building-blocks/global-navigation.md new file mode 100644 index 000000000..223bd0ffb --- /dev/null +++ b/docs/building-blocks/global-navigation.md @@ -0,0 +1,193 @@ +--- +navigation_title: Global Navigation +--- + +# Global Navigation + +**Global navigation** defines how multiple documentation sets are organized and presented together in [assembled documentation](assembled-documentation.md). It creates a unified navigation structure across all repositories. + +## Purpose + +Global navigation enables: + +* **Unified experience** - Present documentation from multiple repositories as a cohesive whole +* **Flexible organization** - Arrange documentation by product, feature, or audience rather than by repository +* **Independent evolution** - Reorganize global structure without changing documentation sets +* **Cross-repository grouping** - Combine related content from different repositories + +## Configuration + +Global navigation is defined in the `navigation.yml` file, which is part of the [site configuration](../configure/site/index.md). It follows a very similar `toc` configuration structure to [documentation set navigation](documentation-set-navigation.md). + +## Key differences from documentation set navigation + +Global navigation has specific restrictions: + +* **It may only link to `toc.yml` or `docset.yml` files** - You cannot reference individual markdown files +* **Uses crosslink syntax** - References to other repositories use the `repository://` syntax +* **Requires all TOC sections** - Dangling TOC sections are not allowed + +## Basic structure + +```yaml +toc: + - toc: get-started + - toc: elasticsearch-net:// + - toc: extend + children: + - toc: kibana://extend + path_prefix: extend/kibana + - toc: logstash://extend + path_prefix: extend/logstash + - toc: beats://extend +``` + +## Syntax notes + +### Crosslink syntax + +The TOC uses a similar [cross-link syntax to links](../syntax/links.md): + +```yaml +- toc: elasticsearch-net:// # References entire repository +- toc: kibana://extend # References 'extend' TOC section from kibana +``` + +### Implied suffixes + +The `./docset.yml` or `/toc.yml` suffix is implied - the assembler will find the correct file for you: + +### Special repository handling + +The narrative repository `elastic/docs-content` is 'special', so omitting `scheme://` implies `docs-content://`: + +```yaml +- toc: get-started # Implies docs-content://get-started +- toc: elasticsearch://setup # Explicitly from elasticsearch repository +``` + +## Path prefixes + +You must explicitly provide a URL path prefix when including a `toc`. + +```yaml +- toc: extend + children: + - toc: kibana://extend + path_prefix: extend/kibana # Override default path + - toc: logstash://extend + path_prefix: extend/logstash + - toc: beats://extend + path_prefix: extend/beats +``` + +This allows you to: +* Group content from different repositories under a common path +* Avoid URL conflicts when combining repositories +* Create product-specific URL structures + +## Reorganization independence + +These `toc` sections can be reorganized independently of their position in their origin documentation set navigation. This allows sections from different repositories to be grouped together in the global navigation. + +### Example: Cross-repository organization + +You can create a unified section that combines content from multiple repositories: + +```yaml +toc: + - toc: monitoring + children: + - toc: elasticsearch://monitoring + path_prefix: monitoring/elasticsearch + - toc: kibana://monitoring + path_prefix: monitoring/kibana + - toc: beats://monitoring + path_prefix: monitoring/beats +``` + +Even though each repository defines its own `monitoring` section, the global navigation presents them as a cohesive monitoring guide. + +## Dangling TOC sections + +All `toc` sections must be linked in `navigation.yml`. + +**Dangling `toc` sections are not allowed** and the assembler build will report an error if it finds any. + +This ensures: +* No content is accidentally excluded from the site +* Navigation references are always valid +* Documentation coverage is complete +* Every TOC section defined in a `docset.yml` appears somewhere in the global navigation + +### Example of validation + +If a repository defines: + +```yaml +# my-repo/docs/docset.yml +toc: + - file: index.md + - toc: getting-started + - toc: advanced +``` + +Then `navigation.yml` must reference both `getting-started` and `advanced`: + +```yaml +# navigation.yml +toc: + - toc: my-repo://getting-started + - toc: my-repo://advanced +``` + +If either is missing, the build will fail with an error about dangling TOC sections. + +## Validation + +When building assembled documentation, `docs-builder` validates: + +* All referenced TOC sections exist +* No TOC sections are dangling (unreferenced) +* Path prefixes don't conflict +* Crosslink references resolve correctly + +Validation errors will cause the assembler build to fail. + +## Navigation metadata + +You can customize how sections appear in global navigation: + +### Nested organization + +Create nested navigation structures: + +```yaml +toc: + - toc: getting-started + children: + - toc: elasticsearch://quickstart + - toc: kibana://quickstart + - toc: reference + children: + - toc: elasticsearch://apis + - toc: elasticsearch://settings +``` + +## Build process integration + +During an assembler build: + +1. `docs-builder` reads `navigation.yml` +2. It resolves all TOC section references across repositories +3. It validates that all sections are accounted for (no dangling sections) +4. It builds each documentation set with knowledge of its global path prefix +5. It generates the final site with unified navigation + +## Related concepts + +* [Documentation Set Navigation](documentation-set-navigation.md) - How individual repositories organize their content +* [Assembled Documentation](assembled-documentation.md) - The build process that uses global navigation +* [Site Configuration](../configure/site/index.md) - Complete site configuration reference +* [Navigation Configuration](../configure/site/navigation.md) - Detailed navigation.yml reference +* [Cross-link syntax](../syntax/links.md) - Understanding the repository:// syntax diff --git a/docs/building-blocks/inbound-crosslinks.md b/docs/building-blocks/inbound-crosslinks.md new file mode 100644 index 000000000..e661c7f65 --- /dev/null +++ b/docs/building-blocks/inbound-crosslinks.md @@ -0,0 +1,120 @@ +--- +navigation_title: Inbound Crosslinks +--- + +# Inbound Crosslinks + +**Inbound crosslinks** are links from other documentation sets to yours. Understanding and validating inbound crosslinks helps prevent breaking links in other repositories when you make changes. + +## Purpose + +Inbound crosslink validation allows you to: + +* **Detect breaking changes** - Know when renaming or deleting a file will break links from other repositories +* **Prevent regressions** - Avoid publishing changes that break documentation elsewhere +* **Coordinate changes** - Understand dependencies before making structural changes + +## How it works + +A regular [build](../cli/docset/build.md) of a documentation set won't validate inbound links automatically. + +You have to use the [inbound-links validate-link-reference](../cli/links/inbound-links-validate-link-reference.md) after a build to validate all inbound links. + +The reason for this is that validating all inbound links has to download all published [Link Index](link-index.md) files +for the current [Content Source](../configure/content-sources.md). + +## Validation commands + +### Validate all inbound links + +Check all inbound links for all published [Link Index](link-index.md) files declared in the [Link Catalog](link-catalog.md) + +```bash +docs-builder inbound-links validate-all +``` + +### Validate specific link reference + +Validate a locally built `links.json` against all published Link Index files: + +```bash +docs-builder inbound-links validate-link-reference +docs-builder inbound-links validate-link-reference --file .artifacts/docs/html/links.json +``` + +### Validate with filters + +Check inbound links from specific repositories or to specific resources: + +```bash +docs-builder inbound-links validate --from elasticsearch --to kibana +``` + +## Common scenarios + +### Moving a file + +If you move a file that other repositories link to: + +1. Create a redirect from the old path to the new path +2. Update your documentation's redirect configuration +3. Run inbound link validation to ensure the redirect works +4. Notify teams that maintain repositories with inbound links + +### Deleting a file + +Before deleting a file: + +1. Run inbound link validation to see if other repositories link to it +2. If there are inbound links, coordinate with those teams first +3. Consider leaving a redirect to related content +4. Update the other repositories to remove or update their links + +### Renaming headings + +Heading anchors are part of the Link Index. If other repositories link to specific headings in your documentation: + +1. Validate inbound links before renaming +2. Consider keeping old heading anchors if heavily linked +3. Document the change if coordination is needed + +## Integration with CI/CD + +Our preview CI job will run inbound link validation automatically. + +## Best practices + +### Set up redirects + +When moving or renaming files, always set up redirects: + +```yaml +# In your documentation's configuration +redirects: + - from: /old-path/file.html + to: /new-path/file.html +``` + +### Communicate changes + +If you need to make a breaking change: + +1. Run inbound link validation to identify affected repositories +2. File issues or notify maintainers of affected repositories +3. Coordinate the change timing +4. Provide redirect mappings or alternative URLs + +### Validate before merging + +Make inbound link validation part of your review process: + +* Run validation locally before creating a PR +* Include validation in CI checks +* Review validation results before merging + +## Related concepts + +* [Outbound Crosslinks](outbound-crosslinks.md) - Links from your documentation to others +* [Link Index](link-index.md) - How your linkable resources are tracked +* [Link Service](link-service.md) - Where inbound link information is stored +* [Distributed Documentation](distributed-documentation.md) - The architecture enabling this validation diff --git a/docs/building-blocks/index.md b/docs/building-blocks/index.md new file mode 100644 index 000000000..f571ad0f2 --- /dev/null +++ b/docs/building-blocks/index.md @@ -0,0 +1,64 @@ +--- +navigation_title: Building Blocks +--- + +# Building Blocks + +This section explains the core concepts and building blocks that make up the docs-builder architecture. Understanding these concepts will help you work effectively with distributed documentation and cross-repository linking. + +## Core concepts + +### [Documentation Set](documentation-set.md) + +The fundamental unit of documentation - a single folder containing the documentation for one repository. Each documentation set can be built, versioned, and maintained independently. + +### [Assembled Documentation](assembled-documentation.md) + +The product of combining multiple documentation sets into a unified documentation website with global navigation. This enables a seamless user experience across multiple repositories. + +### [Distributed Documentation](distributed-documentation.md) + +The architectural approach that allows documentation sets to be built independently while maintaining link integrity across repositories. This enables teams to work autonomously without blocking each other. + +## Link management infrastructure + +### [Link Service](link-service.md) + +The central S3-backed service where Link Index files are published and stored. This enables distributed validation and build resilience. + +### [Link Index](link-index.md) + +A JSON file (`links.json`) containing all linkable resources for a specific repository branch. Published to the Link Service after each successful build. + +### [Link Catalog](link-catalog.md) + +A catalog file listing all available Link Index files across all repositories and branches. Used by the assembler to coordinate builds and by documentation builds for validation. + +## Cross-repository linking + +### [Outbound Crosslinks](outbound-crosslinks.md) + +Links from your documentation to other documentation sets. Validated against published Link Index files to ensure they're correct. + +### [Inbound Crosslinks](inbound-crosslinks.md) + +Links from other documentation sets to yours. Validated to prevent breaking changes when you move or delete content. + +## Navigation + +### [Documentation Set Navigation](documentation-set-navigation.md) + +How individual documentation sets organize their content through TOC sections in `docset.yml` and `toc.yml` files. Each repository controls its own internal navigation structure, including file and folder organization. + +### [Global Navigation](global-navigation.md) + +How multiple documentation sets are organized together in assembled documentation through `navigation.yml`. Uses crosslink syntax to reference TOC sections from different repositories and enables cross-repository content organization. + +## How it all works together + +1. Each repository builds its documentation set independently +2. Successful builds publish a Link Index to the Link Service +3. The Link Catalog catalogs all available Link Index files +4. Documentation builds validate crosslinks using these Link Index files +5. The assembler combines documentation sets using the Link Catalog +6. Teams can work independently while maintaining link integrity across repositories diff --git a/docs/building-blocks/link-catalog.md b/docs/building-blocks/link-catalog.md new file mode 100644 index 000000000..a8705be16 --- /dev/null +++ b/docs/building-blocks/link-catalog.md @@ -0,0 +1,78 @@ +--- +navigation_title: Link Catalog +--- + +# Link Catalog + +The **Link Catalog** is a single JSON file that serves as a catalog of all available [Link Index](link-index.md) files across all repositories. + +## Purpose + +The Link Catalog provides: + +* **Discovery** - A single file to query for all available documentation across all repositories and branches +* **Efficiency** - Avoid scanning the entire [Link Service](link-service.md) to find available Link Index files +* **Assembler coordination** - The assembler uses this to determine which repositories and versions are available to build + +## Location + +The Link Catalog is available at: + +``` +https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/link-index.json +``` + +## Structure + +The Link Catalog contains: + +* List of all organizations (e.g., `elastic`) +* Repositories within each organization (e.g., `elasticsearch`, `kibana`) +* Branches for each repository (e.g., `main`, `8.x`, `7.17`) +* Metadata about each Link Index: + * Last updated timestamp + * Commit SHA that produced the Link Index + * ETAG of the Link Index file + * URL to the Link Index file + +## Maintenance + +The Link Catalog is automatically maintained: + +1. A repository's CI/CD pipeline publishes a new `links.json` to the Link Service +2. The S3 bucket triggers an SQS message +3. An AWS Lambda function listens to these SQS messages +4. The Lambda function updates the Link Catalog to include or update the entry for the new Link Index + +This process ensures the catalog stays in sync with published Link Index files without manual intervention. + +## Usage + +### By the assembler + +When running `docs-builder assembler clone` or `docs-builder assembler build`: + +1. The assembler fetches the Link Catalog +2. It determines which repositories and versions to clone/build based on the [site configuration](../configure/site/index.md) +3. It uses the commit SHAs from the catalog to clone specific versions + +### By documentation builds + +During a documentation build: + +1. `docs-builder` fetches the Link Catalog +2. It determines which Link Index files to download for cross-repository validation +3. It validates all cross-links against the appropriate Link Index files + +## Benefits + +* **Single source of truth** - One file to check for all available documentation +* **Performance** - Fast lookup without scanning the entire Link Service +* **Automation** - Maintained automatically via Lambda functions +* **Resilience** - Represents only successful builds with valid Link Indexes + +## Related concepts + +* [Link Service](link-service.md) - Where the Link Catalog is stored +* [Link Index](link-index.md) - The files cataloged by the Link Catalog +* [Assembled Documentation](assembled-documentation.md) - Uses the Link Catalog to coordinate builds diff --git a/docs/building-blocks/link-index.md b/docs/building-blocks/link-index.md new file mode 100644 index 000000000..b12bad7da --- /dev/null +++ b/docs/building-blocks/link-index.md @@ -0,0 +1,86 @@ +--- +navigation_title: Link Index +--- + +# Link Index + +A **Link Index** is a JSON file (`links.json`) that contains all the linkable resources for a specific repository branch. + +## Purpose + +The Link Index enables: + +* **Cross-repository linking** - Other documentation sets can link to your content +* **Link validation** - Validate that links to your content are correct +* **Inbound link detection** - Know when other repositories link to your content +* **Distributed builds** - Build documentation independently while maintaining link integrity + +## Structure + +Each repository branch gets its own Link Index file in the [Link Service](link-service.md), organized by: + +* **Organization** - e.g., `elastic` +* **Repository** - e.g., `elasticsearch` +* **Branch** - e.g., `main`, `8.x`, `7.17` + +## Example + +View [Elasticsearch's main branch Link Index](https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/elasticsearch/main/links.json) to see a real example. + +The Link Index contains: + +* All documentation pages in the repository +* Headings within those pages +* Anchors and linkable elements +* Version information +* Metadata about the build + +## Generation + +The Link Index is automatically generated during a documentation build: + +1. `docs-builder` builds the documentation set +2. During the build, all linkable resources are tracked +3. After a successful build, a `links.json` file is written to `.artifacts/docs/html/links.json` +4. CI/CD publishes this file to the [Link Service](link-service.md) + +## Usage + +### Resolving outbound crosslinks + +When you use a crosslink like `elasticsearch://reference/api/search.md`, `docs-builder`: + +1. Fetches the Elasticsearch Link Index from the Link Service +2. Looks up the path in the index +3. Validates the link exists +4. Resolves it to the correct URL + +### Validating inbound crosslinks + +When building your documentation, `docs-builder` can: + +1. Fetch your repository's Link Index from previous builds +2. Compare with your current local changes +3. Detect if you've moved or deleted files that other repositories link to +4. Warn about breaking changes + +## File location + +During a build, the Link Index is written to: + +``` +.artifacts/docs/html/links.json +``` + +After publishing, it's available at: + +``` +https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{org}/{repo}/{branch}/links.json +``` + +## Related concepts + +* [Link Service](link-service.md) - Where Link Index files are stored +* [Link Catalog](link-catalog.md) - Catalog of all Link Index files +* [Outbound Crosslinks](outbound-crosslinks.md) - Links that use the Link Index +* [Inbound Crosslinks](inbound-crosslinks.md) - Links to resources in the Link Index diff --git a/docs/building-blocks/link-service.md b/docs/building-blocks/link-service.md new file mode 100644 index 000000000..e45ab097f --- /dev/null +++ b/docs/building-blocks/link-service.md @@ -0,0 +1,66 @@ +--- +navigation_title: Link Service +--- + +# Link Service + +The **Link Service** is the central location that stores: + +* All [Link Index](link-index.md) files for all the repositories and branches that are published. +* The [Link Catalog](link-catalog.md), a single JSON file that contains references to all the `Link Index` files. + +We only have one link service today for all public documentation. + +* https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/ + +## Architecture + +The Link Service is implemented as: + +* **Storage** - An S3 bucket +* **CDN** - CloudFront fronting the S3 bucket for fast global access +* **Access** - Publicly accessible for read operations + +## Purpose + +The Link Service enables: + +* **Distributed validation** - Any documentation build can validate cross-repository links without cloning all repositories +* **Link discovery** - Find what resources are available in other repositories +* **Build resilience** - Assembler builds can reference the last known good state of each repository +* **Decentralized publishing** - Each repository publishes its own Link Index independently + +## URL structure + +Link Index files are organized by repository and branch: + +``` +https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{org}/{repo}/{branch}/links.json +``` + +For example: +* [Elasticsearch main branch](https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/elasticsearch/main/links.json) +* [Kibana main branch](https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/kibana/main/links.json) + +## Publishing process + +When a documentation build completes successfully on a default integration branch: + +1. The build generates a `links.json` file +2. The CI/CD pipeline publishes the file to the Link Service +3. An AWS Lambda function triggers on the S3 event +4. The Lambda updates the [Link Catalog](link-catalog.md) to include the new Link Index + +## Access during builds + +During both local and CI builds, `docs-builder`: + +* Fetches relevant Link Index files from the Link Service +* Validates outbound crosslinks against these indexes +* Validates that inbound crosslinks won't be broken by local changes + +## Related concepts + +* [Link Index](link-index.md) - The files stored in the Link Service +* [Link Catalog](link-catalog.md) - The catalog of all Link Index files +* [Distributed Documentation](distributed-documentation.md) - Why the Link Service exists diff --git a/docs/building-blocks/outbound-crosslinks.md b/docs/building-blocks/outbound-crosslinks.md new file mode 100644 index 000000000..0fb913880 --- /dev/null +++ b/docs/building-blocks/outbound-crosslinks.md @@ -0,0 +1,105 @@ +--- +navigation_title: Outbound Crosslinks +--- + +# Outbound Crosslinks + +**Outbound crosslinks** are links from your documentation set to other documentation sets in different repositories. + +## Purpose + +Outbound crosslinks allow you to: + +* Link to documentation in other repositories +* Maintain those links even as the target repository evolves +* Validate links during local builds +* Get warnings if target content is moved or deleted + +## Syntax + +If both repositories publish to the same [Link Service](link-service.md), they can link to each other using the crosslink syntax: + +```markdown +[Link text](repository-name://path/to/file.md) +``` + +For example: + +```markdown +See the [Search API documentation](elasticsearch://reference/api/search.md) +``` + +## How it works + +You have to explicitly opt in to another repository's `Link Index` by adding it to your `docset.yml` file: + +```yaml +cross_links: + - docs-content +``` + + +When `docs-builder` encounters a crosslink: + +1. **Parse** - Extracts the repository name and path from the link +3. **Resolve** - Looks up the path in the locally cached [Link Index](link-index.md) to get the actual URL +4. **Validate** - Verifies the link exists and generates an error if not +5. **Transform** - Replaces the crosslink with the fully resolved URL in the output + +## Validation + +During a build, `docs-builder`: + +* **Validates immediately** - Checks all outbound cross-links against locally fetched [Link Index](link-index.md) files +* **Reports errors** - Reports errors about broken links before you publish + +## Configuration + +To enable cross-links to a repository, add it to your `docset.yml`: + +```yaml +cross_links: + - elasticsearch + - kibana + - fleet +``` + +This instructs `docs-builder` to fetch the `Link Index` from the [Link Service](link-service.md) during the build process which are then cached locally. +`docs-builder` will validate locally cached `Link Index` files against the remote `Link Index` files on each build fetching updates as needed. + +Now you can create crosslinks e.g `elasticsearch://path/to/file.md` + +The explicit opt-in prevents each repository build having the fetch all the links for all the repositories in the [`Link Catalog`](link-catalog.md) of which there may be many. + +## Best practices + +### Link to files, not URLs + +**Good:** +```markdown +[Search API](elasticsearch://reference/api/search.md) +``` + +**Bad:** +```markdown +[Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search.html) +``` + +The crosslink syntax is resilient to: +* URL structure changes +* File moves (if redirects are configured) +* Version differences + +### Link to headings + +You can link to specific headings within a page: + +```markdown +[Query DSL](elasticsearch://reference/query-dsl.md#match-query) +``` + +## Related concepts + +* [Inbound Crosslinks](inbound-crosslinks.md) - Links from other repositories to yours +* [Link Index](link-index.md) - How crosslinks are resolved +* [Links syntax](../syntax/links.md) - Complete link syntax documentation diff --git a/docs/cli/assembler/assemble.md b/docs/cli/assembler/assemble.md new file mode 100644 index 000000000..e8c5bcc8a --- /dev/null +++ b/docs/cli/assembler/assemble.md @@ -0,0 +1,91 @@ +# assemble + +Do a full assembler clone, build and optional serving of the full documentation in one swoop + +## Usage + +``` +docs-builder assemble [options...] [-h|--help] [--version] +``` + + + +## Usage examples + +The following will clone the repository, build the documentation and serve it on port 4000 using the embedded configuration inside the `docs-builder` binary. + +```bash +docs-builder assemble --serve +``` + +This single command is equivalent to the following commands: + +```bash +docs-builder assembler clone +docs-builder assembler build +docs-builder assembler serve +``` + +### Using a local workspace for assembler builds + +Where this command really shines is when you want to create a temporary workspace folder to validate: + +* changes to [site wide configuration](../../configure/site/index.md) +* changes to one or more repositories and their effect on the assembler build. + +To do that inside an empty folder, call: + +```bash +docs-builder assembler config init --local +docs-builder assemble --serve +``` + +This will source the latest configuration from [The `config` folder on the `main` branch of `docs-builder`](https://github.com/elastic/docs-builder/tree/main/config) +and place them inside the `$(pwd)/config` folder. + +Now when you call `docs-builder assemble` rather than using the embedded configuration, it will use local one that one you just created. +You can be explicit about the configuration source to use: + +```bash +docs-builder assembler config init --local +docs-builder assemble --serve -c local +``` + +## Options + +`--strict` `` +: Treat warnings as errors and fail the build on warnings (optional) + +`--environment` `` +: The environment to build (optional) defaults to 'dev' + +`--fetch-latest` `` +: If true, fetch the latest commit of the branch instead of the link registry entry ref (optional) + +`--assume-cloned` `` +: If true, assume the repository folder already exists on disk assume it's cloned already, primarily used for testing (optional) + +`--metadata-only` `` +: Only emit documentation metadata to output, ignored if 'exporters' is also set (optional) + +`--show-hints` `` +: Show hints from all documentation sets during assembler build (optional) + +`--exporters` `` +: Set available exporters: + + * html + * es, + * config, + * links, + * state, + * llm, + * redirect, + * metadata, + * default + * none. + + Defaults to (html, llm, config, links, state, redirect) or 'default'. (optional) + +`--serve` +: Serve the documentation on port 4000 after successful build (Optional) \ No newline at end of file diff --git a/docs/cli/assembler/assembler-bloom-filter-create.md b/docs/cli/assembler/assembler-bloom-filter-create.md new file mode 100644 index 000000000..cda2ec4a2 --- /dev/null +++ b/docs/cli/assembler/assembler-bloom-filter-create.md @@ -0,0 +1,24 @@ +--- +navigation_title: "bloom-filter create" +--- + +# assembler bloom-filter create + +Generates a bloom filter that gets embedded into the `docs-builder` binary. + +This bloom filter is used to determine whether a document's `mapped_page` in the frontmatter exists in + +the project of [legacy-url-mappings](../../configure/site/legacy-url-mappings.md) + +The existence determines how the document history selector should be populated. + +## Usage + +``` +docs-builder assembler bloom-filter create [options...] [-h|--help] [--version] +``` + +## Options + +`--built-docs-dir` `` +: The local dir of local elastic/built-docs repository (Required) \ No newline at end of file diff --git a/docs/cli/assembler/assembler-bloom-filter-lookup.md b/docs/cli/assembler/assembler-bloom-filter-lookup.md new file mode 100644 index 000000000..bdb302712 --- /dev/null +++ b/docs/cli/assembler/assembler-bloom-filter-lookup.md @@ -0,0 +1,17 @@ +--- +navigation_title: "bloom-filter lookup" +--- +# assembler bloom-filter lookup + +Test command to assert if an old V2 url matches with our bloom filter + +## Usage + +``` +docs-builder assembler bloom-filter lookup [options...] [-h|--help] [--version] +``` + +## Options + +`--path` `` +: The local dir of local elastic/built-docs repository (Required) \ No newline at end of file diff --git a/docs/cli/assembler/assembler-build.md b/docs/cli/assembler/assembler-build.md new file mode 100644 index 000000000..6134277a1 --- /dev/null +++ b/docs/cli/assembler/assembler-build.md @@ -0,0 +1,51 @@ +--- +navigation_title: "build" +--- + +# assembler build + +:::note +This command requires that you've previously ran `docs-builder assembler clone` to clone the documentation sets. +If you clone using a certain `--environment` you must also use that same `--environment` when building. +::: + +Builds all the documentation sets and assembles them into an assembled complete documentation site that's ready to be deployed. + +It uses [the site configuration files](../../configure/site/index.md) to direct how the documentation sets should be assembled. + +## Usage + +``` +docs-builder assembler build [options...] [-h|--help] [--version] +``` + +## Options + +`--strict` `` +: Treat warnings as errors and fail the build on warnings (optional) + +`--environment` `` +: The environment to build (optional) + +`--metadata-only` `` +: Only emit documentation metadata to output, ignored if 'exporters' is also set (optional) + +`--show-hints` `` +: Show hints from all documentation sets during assembler build (optional) + +`--exporters` `` +: Set available exporters: + + * html + * es, + * config, + * links, + * state, + * llm, + * redirect, + * metadata, + * default + * none. + + Defaults to (html, llm, config, links, state, redirect) or 'default'. (optional) + diff --git a/docs/cli/assembler/assembler-clone.md b/docs/cli/assembler/assembler-clone.md new file mode 100644 index 000000000..0a555990f --- /dev/null +++ b/docs/cli/assembler/assembler-clone.md @@ -0,0 +1,29 @@ +--- +navigation_title: "clone" +--- + +# assembler clone + +Clones all repositories. Defaults to `$(pwd)/.artifacts/checkouts/{content_source}`. + +The `content_source` is the `content_source` of the `--environment` option as configured in `assembly.yaml` + +## Usage + +``` +docs-builder assembler clone [options...] [-h|--help] [--version] +``` + +## Options + +`--strict` `` +: Treat warnings as errors and fail the build on warnings (optional) + +`--environment` `` +: The environment to build (optional) + +`--fetch-latest` `` +: If true, fetch the latest commit of the branch instead of the link registry entry ref (optional) + +`--assume-cloned` `` +: If true, assume the repository folder already exists on disk assume it's cloned already, primarily used for testing (optional) \ No newline at end of file diff --git a/docs/cli/assembler/assembler-config-init.md b/docs/cli/assembler/assembler-config-init.md new file mode 100644 index 000000000..cc23be77a --- /dev/null +++ b/docs/cli/assembler/assembler-config-init.md @@ -0,0 +1,31 @@ +--- +navigation_title: "config init" +--- + +# assembler config init + +Sources the configuration from [The `config` folder on the `main` branch of `docs-builder`](https://github.com/elastic/docs-builder/tree/main/config) + +By default, the configuration is placed in a special application folder as its main usages is to be used by CI environments. + +* OSX: `~/Library/Application Support/docs-builder` [NSApplicationSupportDirectory](https://developer.apple.com/documentation/foundation/filemanager/searchpathdirectory/applicationsupportdirectory) +* Linux: `~/.config/docs-builder` +* {icon}`logo_windows` Windows: `%APPDATA%\docs-builder` + +You can also use the `--local` option to save the configuration locally in the current working directory. This exposes a great way to assemble the full documentation locally in an empty directory. + +See [using assemble to create local workspaces](assemble.md#using-a-local-workspace-for-assembler-builds) for more information. + +## Usage + +``` +docs-builder assembler config init [options...] [-h|--help] [--version] +``` + +## Options + +`--git-ref` `` +: The git reference of the config, defaults to 'main' (optional) + +`--local` +: Save the remote configuration locally in the pwd so later commands can pick it up as local (Optional) \ No newline at end of file diff --git a/docs/cli/assembler/assembler-content-source-match.md b/docs/cli/assembler/assembler-content-source-match.md new file mode 100644 index 000000000..67781a42a --- /dev/null +++ b/docs/cli/assembler/assembler-content-source-match.md @@ -0,0 +1,33 @@ +--- +navigation_title: "content-source match" +--- + +# assembler content-source match + +This command is used to match a repository and branch to a content source it will emit the following `$GITHUB_OUTPUT`: + +* `content-source-match` - whether the branch is a configured content source +* `content-source-next` - whether the branch is the next content source +* `content-source-current` - whether the branch is the current content source +* `content-source-speculative` - whether the branch is a speculative content source. + +#### Speculative builds + +If branches follow semantic versioning, if a branch is cut that is greater than the current version, it will be considered a speculative build. +`docs-builer`'s shared workflow will trigger even if it's not specified as a content source in `assembler.yml`. + +This allows a branch `links.json` to be published to the `Link Service` a head of time before it's configured as a content source. + +## Usage + +``` +docs-builder assembler content-source match [-h|--help] [--version] +``` + +## Arguments + +` +: The name of the `elastic/` repository you want to match if it should be build on CI + +` +: The branch you want to match if it should be build on CI` \ No newline at end of file diff --git a/docs/cli/assembler/assembler-content-source-validate.md b/docs/cli/assembler/assembler-content-source-validate.md new file mode 100644 index 000000000..661269a81 --- /dev/null +++ b/docs/cli/assembler/assembler-content-source-validate.md @@ -0,0 +1,13 @@ +--- +navigation_title: "content-source validate" +--- + +# assembler content-source validate + +Validates that the configured content source branches are publishing succesfully to the `Links Service`. + +## Usage + +``` +docs-builder assembler content-source validate [-h|--help] [--version] +``` \ No newline at end of file diff --git a/docs/cli/assembler/assembler-deploy-apply.md b/docs/cli/assembler/assembler-deploy-apply.md new file mode 100644 index 000000000..ce1c0df27 --- /dev/null +++ b/docs/cli/assembler/assembler-deploy-apply.md @@ -0,0 +1,24 @@ +--- +navigation_title: "deploy apply" +--- + +# assembler deploy apply + +Applies an incremental synchronization plan created by [`docs-builder assembler deploy plan`](./assembler-deploy-plan). + +## Usage + +``` +docs-builder assembler deploy apply [options...] [-h|--help] [--version] +``` + +## Options + +`--environment` `` +: The environment to build (Required) + +`--s3-bucket-name` `` +: The S3 bucket name to deploy to (Required) + +`--plan-file` `` +: The file path to the plan file to apply (Required) \ No newline at end of file diff --git a/docs/cli/assembler/assembler-deploy-plan.md b/docs/cli/assembler/assembler-deploy-plan.md new file mode 100644 index 000000000..cf4e5c5e7 --- /dev/null +++ b/docs/cli/assembler/assembler-deploy-plan.md @@ -0,0 +1,27 @@ +--- +navigation_title: "deploy plan" +--- + +# assembler deploy plan + +Creates an incremental synchronization plan by comparing the reote `--s3-bucket-name` with the local output of the build. + +## Usage + +``` +docs-builder assembler deploy plan [options...] [-h|--help] [--version] +``` + +## Options + +`--environment` `` +: The environment to build (Required) + +`--s3-bucket-name` `` +: The S3 bucket name to deploy to (Required) + +`--out` `` +: The file to write the plan to (Default: "") + +`--delete-threshold` `` +: The percentage of deletions allowed in the plan as float (optional) \ No newline at end of file diff --git a/docs/cli/assembler/assembler-deploy-update-redirects.md b/docs/cli/assembler/assembler-deploy-update-redirects.md new file mode 100644 index 000000000..ec0255d5d --- /dev/null +++ b/docs/cli/assembler/assembler-deploy-update-redirects.md @@ -0,0 +1,21 @@ +--- +navigation_title: "deploy update-redirects" +--- + +# assembler deploy update-redirects + +Refreshes the redirects mapping in Cloudfront's KeyValueStore + +## Usage + +``` +docs-builder assembler deploy update-redirects [options...] [-h|--help] [--version] +``` + +## Options + +`--environment` `` +: The environment to build (Required) + +`--redirects-file` `` +: Path to the redirects mapping pre-generated by docs-assembler (optional) \ No newline at end of file diff --git a/docs/cli/assembler/assembler-index.md b/docs/cli/assembler/assembler-index.md new file mode 100644 index 000000000..ea8d6947c --- /dev/null +++ b/docs/cli/assembler/assembler-index.md @@ -0,0 +1,75 @@ +--- +navigation_title: "index" +--- + +# assembler index + +Index documentation to Elasticsearch, calls `docs-builder assembler build --exporters elasticsearch`. Exposes more options + +## Usage + +``` +docs-builder assembler index [options...] [-h|--help] [--version] +``` + +## Options + +`-es|--endpoint ` +: Elasticsearch endpoint, alternatively set env DOCUMENTATION_ELASTIC_URL (optional) + +`--environment` `` +: The --environment used to clone ends up being part of the index name (optional) + +`--api-key` `` +: Elasticsearch API key, alternatively set env DOCUMENTATION_ELASTIC_APIKEY (optional) + +`--username` `` +: Elasticsearch username (basic auth), alternatively set env DOCUMENTATION_ELASTIC_USERNAME (optional) + +`--password` `` +: Elasticsearch password (basic auth), alternatively set env DOCUMENTATION_ELASTIC_PASSWORD (optional) + +`--no-semantic` `` +: Index without semantic fields (optional) + +`--search-num-threads` `` +: The number of search threads the inference endpoint should use. Defaults: 8 (optional) + +`--index-num-threads` `` +: The number of index threads the inference endpoint should use. Defaults: 8 (optional) + +`--bootstrap-timeout` `` +: Timeout in minutes for the inference endpoint creation. Defaults: 4 (optional) + +`--index-name-prefix` `` +: The prefix for the computed index/alias names. Defaults: semantic-docs (optional) + +`--buffer-size` `` +: The number of documents to send to ES as part of the bulk. Defaults: 100 (optional) + +`--max-retries` `` +: The number of times failed bulk items should be retried. Defaults: 3 (optional) + +`--debug-mode` `` +: Buffer ES request/responses for better error messages and pass ?pretty to all requests (optional) + +`--proxy-address` `` +: Route requests through a proxy server (optional) + +`--proxy-password` `` +: Proxy server password (optional) + +`--proxy-username` `` +: Proxy server username (optional) + +`--disable-ssl-verification` `` +: Disable SSL certificate validation (EXPERT OPTION) (optional) + +`--certificate-fingerprint` `` +: Pass a self-signed certificate fingerprint to validate the SSL connection (optional) + +`--certificate-path` `` +: Pass a self-signed certificate to validate the SSL connection (optional) + +`--certificate-not-root` `` +: If the certificate is not root but only part of the validation chain pass this (optional) \ No newline at end of file diff --git a/docs/cli/assembler/assembler-navigation-validate-link-reference.md b/docs/cli/assembler/assembler-navigation-validate-link-reference.md new file mode 100644 index 000000000..678724217 --- /dev/null +++ b/docs/cli/assembler/assembler-navigation-validate-link-reference.md @@ -0,0 +1,20 @@ +--- +navigation_title: "navigation validate-link-reference" +--- + +# assembler navigation validate-link-reference + +Validate all published links in links.json do not collide with navigation path_prefixes and all urls are unique. + +Read more about [navigation](../../configure/site/navigation.md). + +## Usage + +``` +docs-builder assembler navigation validate-link-reference [arguments...] [-h|--help] [--version] +``` + +## Arguments + +`[0] ` +: Path to `links.json` defaults to '.artifacts/docs/html/links.json' \ No newline at end of file diff --git a/docs/cli/assembler/assembler-navigation-validate.md b/docs/cli/assembler/assembler-navigation-validate.md new file mode 100644 index 000000000..4e897d41e --- /dev/null +++ b/docs/cli/assembler/assembler-navigation-validate.md @@ -0,0 +1,13 @@ +--- +navigation_title: "navigation validate" +--- + +# assembler navigation validate + +Validates [navigation.yml](../../configure/site/navigation.md) does not contain colliding path prefixes and all urls are unique + +## Usage + +``` +docs-builder assembler navigation validate [-h|--help] [--version] +``` \ No newline at end of file diff --git a/docs/cli/assembler/assembler-serve.md b/docs/cli/assembler/assembler-serve.md new file mode 100644 index 000000000..805d1b438 --- /dev/null +++ b/docs/cli/assembler/assembler-serve.md @@ -0,0 +1,21 @@ +--- +navigation_title: "serve" +--- + +# assembler serve + +Serve the output of an assembler build on `http://localhost:4000/` + +## Usage + +``` +docs-builder assembler serve [options...] [-h|--help] [--version] +``` + +## Options + +`--port` `` +: Port to serve the documentation. (Default: 4000) + +`--path` `` +: (optional) \ No newline at end of file diff --git a/docs/cli/assembler/index.md b/docs/cli/assembler/index.md new file mode 100644 index 000000000..6c92bac9e --- /dev/null +++ b/docs/cli/assembler/index.md @@ -0,0 +1,64 @@ +--- +navigation_title: "assembler" +--- + +# Assembler commands + +Assembler builds bring together all isolated builds and turn them into the overall documentation that gets published. + +If you want to build the latest documentation, you can do so using the following commands + +:::{note} +When assembling using the `config init --local` option, it's advised to create an empty directory to run these commands in. +This creates a dedicated workspace for the assembler build and any local changes that you might want to test. +::: + +```bash +docs-builder assembler config init --local +docs-builder assemble --serve +``` + +The full assembled documentation should now be running at http://localhost:4000. + +The [assemble](assemble.md) command is syntactic sugar over the following commands: + +```bash +docs-builder assembler config init --local +docs-builder assembler clone +docs-builder assembler build +docs-builder assembler serve +``` + +Which may be more appropriate to call in isolation depending on the workflow you are going for. + +All `assembler` commans take an `--environment ` argument that defaults to 'dev' but can be set e.g to 'prod' to +build the production documentation. See [assembler.yml](../../configure/site/index.md) configuration for which environments are +available + +## Build commands + +- [assemble](assemble.md) +- [assembler build](assembler-build.md) +- [assembler clone](assembler-clone.md) +- [assembler config init](assembler-config-init.md) +- [assembler index](assembler-index.md) +- [assembler serve](assembler-serve.md) + +## Specialized build commands + +- [assembler bloom-filter create](assembler-bloom-filter-create.md) +- [assembler bloom-filter lookup](assembler-bloom-filter-lookup.md) + +## Validation commands + +- [assembler content-source match](assembler-content-source-match.md) +- [assembler content-source validate](assembler-content-source-validate.md) +- [assembler navigation validate](assembler-navigation-validate.md) +- [assembler navigation validate-link-reference](assembler-navigation-validate-link-reference.md) + +## Deploy commands + +- [assembler deploy apply](assembler-deploy-apply.md) +- [assembler deploy plan](assembler-deploy-plan.md) +- [assembler deploy update-redirects](assembler-deploy-update-redirects.md) + diff --git a/docs/cli/docset/build.md b/docs/cli/docset/build.md new file mode 100644 index 000000000..d300c6ee9 --- /dev/null +++ b/docs/cli/docset/build.md @@ -0,0 +1,62 @@ +# build + +Builds a local documentation set folder. + +Repeated invocations will do incremental builds of only the changed files unless: + +* The base branch has changed +* The state file in the output folder has been removed +* The `--force` option is specified. + +## Usage + +``` +docs-builder [command] [options...] [-h|--help] [--version] +``` + +## Global Options + +- `--log-level` `level` + +## Options + +`-p|--path ` +: Defaults to the`{pwd}/docs` folder (optional) + +`-o|--output ` +: Defaults to `.artifacts/html` (optional) + +`--path-prefix` `` +: Specifies the path prefix for urls (optional) + +`--force` `` +: Force a full rebuild of the destination folder (optional) + +`--strict` `` +: Treat warnings as errors and fail the build on warnings (optional) + +`--allow-indexing` `` +: Allow indexing and following of HTML files (optional) + +`--metadata-only` `` +: Only emit documentation metadata to output, ignored if 'exporters' is also set (optional) + +`--exporters` `` +: Set available exporters: + + * html + * es, + * config, + * links, + * state, + * llm, + * redirect, + * metadata, + * default + * none. + + Defaults to (html, llm, config, links, state, redirect) or 'default'. (optional) + + +`--canonical-base-url` `` +: The base URL for the canonical url tag (optional) \ No newline at end of file diff --git a/docs/cli/docset/diff-validate.md b/docs/cli/docset/diff-validate.md new file mode 100644 index 000000000..02c97d84b --- /dev/null +++ b/docs/cli/docset/diff-validate.md @@ -0,0 +1,18 @@ +# diff validate + +Gathers the local changes by inspecting the git log, stashed and unstashed changes. + +It currently validates the following: + +* Ensures that renames and deletions are reflected in [redirects.yml](../../contribute/redirects.md) + +## Usage + +``` +docs-builder diff validate [options...] [-h|--help] [--version] +``` + +## Options + +`-p|--path ` +: Defaults to the`{pwd}/docs` folder (optional) \ No newline at end of file diff --git a/docs/cli/docset/index-command.md b/docs/cli/docset/index-command.md new file mode 100644 index 000000000..833e6a8a6 --- /dev/null +++ b/docs/cli/docset/index-command.md @@ -0,0 +1,71 @@ +# index + +Index a single documentation set to Elasticsearch, calls `docs-builder --exporters elasticsearch`. Exposes more options + +## Usage + +``` +docs-builder index [options...] [-h|--help] [--version] +``` + +## Options + +`-es|--endpoint ` +: Elasticsearch endpoint, alternatively set env DOCUMENTATION_ELASTIC_URL (optional) + +`--path` `` +: path to the documentation folder, defaults to pwd. (optional) + +`--api-key` `` +: Elasticsearch API key, alternatively set env DOCUMENTATION_ELASTIC_APIKEY (optional) + +`--username` `` +: Elasticsearch username (basic auth), alternatively set env DOCUMENTATION_ELASTIC_USERNAME (optional) + +`--password` `` +: Elasticsearch password (basic auth), alternatively set env DOCUMENTATION_ELASTIC_PASSWORD (optional) + +`--no-semantic` `` +: Index without semantic fields (optional) + +`--search-num-threads` `` +: The number of search threads the inference endpoint should use. Defaults: 8 (optional) + +`--index-num-threads` `` +: The number of index threads the inference endpoint should use. Defaults: 8 (optional) + +`--bootstrap-timeout` `` +: Timeout in minutes for the inference endpoint creation. Defaults: 4 (optional) + +`--index-name-prefix` `` +: The prefix for the computed index/alias names. Defaults: semantic-docs (optional) + +`--buffer-size` `` +: The number of documents to send to ES as part of the bulk. Defaults: 100 (optional) + +`--max-retries` `` +: The number of times failed bulk items should be retried. Defaults: 3 (optional) + +`--debug-mode` `` +: Buffer ES request/responses for better error messages and pass ?pretty to all requests (optional) + +`--proxy-address` `` +: Route requests through a proxy server (optional) + +`--proxy-password` `` +: Proxy server password (optional) + +`--proxy-username` `` +: Proxy server username (optional) + +`--disable-ssl-verification` `` +: Disable SSL certificate validation (EXPERT OPTION) (optional) + +`--certificate-fingerprint` `` +: Pass a self-signed certificate fingerprint to validate the SSL connection (optional) + +`--certificate-path` `` +: Pass a self-signed certificate to validate the SSL connection (optional) + +`--certificate-not-root` `` +: If the certificate is not root but only part of the validation chain pass this (optional) \ No newline at end of file diff --git a/docs/cli/docset/index.md b/docs/cli/docset/index.md new file mode 100644 index 000000000..fc11c183c --- /dev/null +++ b/docs/cli/docset/index.md @@ -0,0 +1,26 @@ +--- +navigation_title: "documentation set" +--- + +# Documentation Set Commands + +An isolated build means building a single documentation set. + +A `Documentation Set` is defined as a folder containing a [docset.yml](../configure/content-set/index.md) file. + +These commands are typically what you interface with when you are working on the documentation of a single repository locally. + +## Isolated build commands + +`build` is the default command so you can just run `docs-builder` to build a single documentation set. `docs-builder` will +locate the `docset.yml` anywhere in the directory tree automatically and build the documentation. + +- [build](docset/build.md) - build a single documentation set (incrementally) +- [serve](docset/serve.md) - partial build and serve documentation as needed at http://localhost:3000 +- [index](docset/index-command.md) - ingest a single documentation set to an Elasticsearch index. + +## Refactor commands + +- [mv](docset/mv.md) - move a file or folder to a new location. This will rewrite all links in all files too. +- [diff validate](docset/diff-validate.md) - validate that local changes are reflected in [redirects.yml](../contribute/redirects.md) + diff --git a/docs/cli/docset/mv.md b/docs/cli/docset/mv.md new file mode 100644 index 000000000..173563b1a --- /dev/null +++ b/docs/cli/docset/mv.md @@ -0,0 +1,25 @@ +# mv + +Move a file or folder from one location to another and update all links in the documentation + +## Usage + +``` +docs-builder mv [arguments...] [options...] [-h|--help] [--version] +``` + +## Arguments + +`[0] ` +: The source file or folder path to move from (required) + +`[1] ` +: The target file or folder path to move to (required) + +## Options + +`--dry-run` `` +: Dry run the move operation (optional) + +`-p|--path ` +: Defaults to the`{pwd}` folder (optional) \ No newline at end of file diff --git a/docs/cli/docset/serve.md b/docs/cli/docset/serve.md new file mode 100644 index 000000000..018b995fe --- /dev/null +++ b/docs/cli/docset/serve.md @@ -0,0 +1,23 @@ +# serve + +Continuously serve a documentation folder at http://localhost:3000. + +When running `docs-builder serve`, the documentation is not built in full. +Each page will be build on the fly continuously when requested in the browser. + +The `serve` command is also `live reload` enabled so that file systems changes will be reflected without having to restart the server. +This includes changes to the documentation files, the navigation, or the configuration files. + +## Usage + +``` +docs-builder serve [options...] [-h|--help] [--version] +``` + +## Options + +`-p|--path ` +: Path to serve the documentation. Defaults to the`{pwd}/docs` folder (optional) + +`--port` `` +: Port to serve the documentation. (Default: 3000) \ No newline at end of file diff --git a/docs/cli/index.md b/docs/cli/index.md new file mode 100644 index 000000000..f9d9410a3 --- /dev/null +++ b/docs/cli/index.md @@ -0,0 +1,47 @@ +--- +navigation_title: CLI (docs-builder) +--- + +# Command line interface + +`docs-builder` is the binary used to invoke various commands. +These commands can be roughly grouped into three main categories + +- [Documentation Set commands](#documentation-set-commands) +- [Link commands](#link-commands) +- [Assembler commands](#assembler-commands) + +### Global options + +The following options are available for all commands: + +`--log-level ` +: Change the log level one of ( `trace`, `debug`, `info`, `warn`, `error`, `critical`). Defaults to `info` + +`--config-source` or `-c` +: Explicitly set the configuration source one of `local`, `remote` or `embedded`. Defaults to `local` if available + other wise `embedded` + +## Documentation Set Commands + +Commands that operate over a single documentation set. + +A `Documentation Set` is defined as a folder containing a [docset.yml](../configure/content-set/index.md) file. + +These commands are typically what you interface with when you are working on the documentation of a single repository locally. + +[See available CLI commands for documentation sets](docset/index.md) + +## Link Commands + +Outbound links, those going from the documentation set to other sources, are validated as part of the build process. + +Inbound links, those going from other sources to the documentation set, are validated using specialized commands. + +[See available CLI commands for inbound links](links/index.md) + +## Assembler Commands + +Assembler builds bring together all isolated documentation set builds and turn them into the overall documentation that gets published. + +[See available CLI commands for assembler](assembler/index.md) diff --git a/docs/cli/links/inbound-links-validate-all.md b/docs/cli/links/inbound-links-validate-all.md new file mode 100644 index 000000000..3563db79d --- /dev/null +++ b/docs/cli/links/inbound-links-validate-all.md @@ -0,0 +1,9 @@ +# inbound-links validate-all + +Validate all published cross_links in all published links.json files. + +## Usage + +``` +docs-builder inbound-links validate-all [-h|--help] [--version] +``` \ No newline at end of file diff --git a/docs/cli/links/inbound-links-validate-link-reference.md b/docs/cli/links/inbound-links-validate-link-reference.md new file mode 100644 index 000000000..b9eaca3c2 --- /dev/null +++ b/docs/cli/links/inbound-links-validate-link-reference.md @@ -0,0 +1,17 @@ +# inbound-links validate-link-reference + +Validate a locally published links.json file against all published links.json files in the registry + +## Usage + +``` +docs-builder inbound-links validate-link-reference [options...] [-h|--help] [--version] +``` + +## Options + +`--file` `` +: Path to `links.json` defaults to '.artifacts/docs/html/links.json' (optional) + +`-p|--path ` +: Defaults to the `{pwd}` folder (optional) \ No newline at end of file diff --git a/docs/cli/links/inbound-links-validate.md b/docs/cli/links/inbound-links-validate.md new file mode 100644 index 000000000..80036eec2 --- /dev/null +++ b/docs/cli/links/inbound-links-validate.md @@ -0,0 +1,17 @@ +# inbound-links validate + +Validate all published cross_links in all published links.json files. + +## Usage + +``` +docs-builder inbound-links validate [options...] [-h|--help] [--version] +``` + +## Options + +`--from` `` +: (optional) + +`--to` `` +: (optional) \ No newline at end of file diff --git a/docs/cli/links/index.md b/docs/cli/links/index.md new file mode 100644 index 000000000..60f838ba3 --- /dev/null +++ b/docs/cli/links/index.md @@ -0,0 +1,15 @@ +--- +navigation_title: links +--- + +# Inbound Links + +Outbound links, those going from the documentation set to other sources, are validated as part of the build process. + +Inbound links, those going from other sources to the documentation set, are validated using specialized commands. + +### Inbound link validation commands + +- [inbound-links validate-all](links/inbound-links-validate-all.md) - validate all inbounds links as published to the links registry. +- [inbound-links validate](links/inbound-links-validate.md) - validate inbound links from and to specific repositories +- [inbound-links validate-link-reference](links/inbound-links-validate-link-reference.md) - validate a local link reference artifact from [build](docset/build.md) with the published registry diff --git a/docs/configure/site/legacy-url-mappings.md b/docs/configure/site/legacy-url-mappings.md index 0d72bb433..294e2cc81 100644 --- a/docs/configure/site/legacy-url-mappings.md +++ b/docs/configure/site/legacy-url-mappings.md @@ -1,3 +1,6 @@ +--- +navigation_title: 'legacy-url-mappings.yml' +--- # Legacy URL mappings This [`legacy-url-mappings.yml`](https://github.com/elastic/docs-builder/blob/main/config/legacy-url-mappings.yml) file manages legacy URL patterns for Elastic documentation, mapping the path of each legacy build URL to a list of documentation versions. It ensures that users can easily find previous versions of our documentation. diff --git a/docs/migration/freeze/gh-action.md b/docs/migration/freeze/gh-action.md deleted file mode 100644 index 8c32e3f1e..000000000 --- a/docs/migration/freeze/gh-action.md +++ /dev/null @@ -1,32 +0,0 @@ -# GH Action - -## Overview - -The documentation team will use a GitHub Action to enforce the content freeze by adding comments to pull requests that modify `.asciidoc` files. It complements the use of `CODEOWNERS` to ensure changes during a freeze period are reviewed and approved by the `@docs-freeze-team`. - -## How It Works -1. **Trigger**: The Action is triggered on pull request events (`opened`, `reopened`, or `synchronize`). -2. **Check Changes**: It checks the diff between the latest commits to detect modifications to `.asciidoc` files. -3. **Add Comment**: If changes are detected, the Action posts a comment in the pull request, reminding the contributor of the freeze. - -```yaml -name: Comment on PR for .asciidoc changes - -on: - pull_request: - types: - - synchronize - - opened - - reopened - branches: - - main - - master - - "9.0" - -jobs: - comment-on-asciidoc-change: - permissions: - contents: read - pull-requests: write - uses: elastic/docs-builder/.github/workflows/comment-on-asciidoc-changes.yml@main -``` diff --git a/docs/migration/freeze/index.md b/docs/migration/freeze/index.md deleted file mode 100644 index b4d03a011..000000000 --- a/docs/migration/freeze/index.md +++ /dev/null @@ -1,40 +0,0 @@ -# Documentation Freeze - -:::{important} -During the documentation freeze, maintaining consistency and avoiding conflicts is key. To discourage documentation changes from merging during the documentation freeze, a [GH Action](./gh-action.md) is being added to all repositories with public-facing documentation. -::: - -During the documentation freeze, the Docs team will focus almost entirely on migration tasks to ensure all content is successfully migrated and will handle only emergency documentation requests and release-related activities. When the migration is complete, the docs team will assess any documentation requests that were submitted during the documentation freeze and ensure that they're still relevant in the new information architecture and format. - -:::{note} -The documentation freeze does not block you from merging asciidoc changes. However, you are strongly discouraged from merging changes to these files as any changes will not be carried forward in the migration and will be lost forever. -::: - -## Timeline - -* **29-Jan**: Merge all open Docs PRs by 12AM PST -* **30-Jan**: Documentation freeze begins for all public-facing Docs on main/master -* **20-Feb**: Documentation freeze ends -* **25-Mar**: 9.0.0 Docs + Elastic Docs v3 GA - -### Details - -:::{important} -We are freezing only the main/master public-facing Docs. You can continue to make changes to the 8.x Docs by creating PRs in the 8.x branches. -::: - -Before we migrate on 30-Jan, we will close all unmerged Docs PRs targeting main/master. When the freeze ends, you can open a new PR if the changes are still needed. - -We will not close PRs targeting main/master that also include code changes. After the freeze begins, merged PRs targeting main/master branches that include AsciiDoc changes will not be migrated. When the freeze ends, all 9.0.0 Docs changes will be made to the migrated Markdown Docs files. - -### During the freeze: - -* If you make Docs changes to the main/master branches, GitHub Actions will warn against merging the changes. -* You can make 8.x Docs changes by creating PRs in the relevant 8.x branches. -* For 9.0.0 Docs changes, [open an issue](https://github.com/elastic/docs-content/issues/new?template=internal-request.yaml) in [elastic/docs-content](https://github.com/elastic/docs-content/issues) and we’ll incorporate the changes post migration. -* The Docs Team will focus exclusively on migrating content, with the exception of the following: - * Stack-versioned release notes, including 8.18.0, 9.0.0-beta1, -rc1, and -rc2 - * Serverless changelog - * Cloud Hosted and ECE release notes - -For any questions and emergency Docs requests, post in the [#docs](https://elastic.slack.com/archives/C0JF80CJZ) Slack channel. diff --git a/docs/migration/index.md b/docs/migration/index.md index d9749743a..f385921da 100644 --- a/docs/migration/index.md +++ b/docs/migration/index.md @@ -1,11 +1,11 @@ --- -navigation_title: Migration +navigation_title: Asciidoc Migration --- # Migration to docs-builder :::{important} -We are enforcing a [Documentation Freeze](./freeze/index.md) while we migrate docs between our two build systems. +All repositories have been migrated to `docs-builder` however older branches may still be using `asciidoc`. ::: Migrating to Elastic Docs V3 is more than just moving to a new documentation build system. This migration includes: diff --git a/src/tooling/docs-builder/Commands/IsolatedBuildCommand.cs b/src/tooling/docs-builder/Commands/IsolatedBuildCommand.cs index 4673fc5d0..0dd4514d6 100644 --- a/src/tooling/docs-builder/Commands/IsolatedBuildCommand.cs +++ b/src/tooling/docs-builder/Commands/IsolatedBuildCommand.cs @@ -23,7 +23,7 @@ IConfigurationContext configurationContext ) { /// - /// Converts a source Markdown folder or file to an output folder + /// Builds a source documentation set folder. /// global options: /// --log-level level ///