diff --git a/.github/workflows/test-docs.yml b/.github/workflows/test-docs.yml index 1051a8a17d..7aab441434 100644 --- a/.github/workflows/test-docs.yml +++ b/.github/workflows/test-docs.yml @@ -11,6 +11,7 @@ jobs: runs-on: ubuntu-latest outputs: console: ${{ steps.filter.outputs.console }} + quickstart: ${{ steps.filter.outputs.quickstart }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -21,22 +22,39 @@ jobs: filters: | console: - 'modules/console/**' + quickstart: + - 'modules/get-started/pages/quick-start.adoc' run-tests: needs: setup permissions: contents: write pull-requests: write issues: write + id-token: write strategy: matrix: os: [ubuntu-latest] # Only using Linux for now since macOS takes a long time runs-on: ${{ matrix.os }} steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.RP_AWS_CRED_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.RP_AWS_CRED_ACCOUNT_ID }}:role/${{ vars.RP_AWS_CRED_BASE_ROLE_NAME }}${{ github.event.repository.name }} + - uses: aws-actions/aws-secretsmanager-get-secrets@v2 + with: + secret-ids: | + ,sdlc/prod/github/actions_bot_token + parse-json-secrets: true - uses: actions/checkout@v4 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ env.ACTIONS_BOT_TOKEN }} path: redpanda-docs - - name: Test docs + + - name: Set GitHub token + run: | + echo "REDPANDA_GITHUB_TOKEN=${{ env.ACTIONS_BOT_TOKEN }}" >> $GITHUB_ENV + + - name: Run all tests if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} uses: doc-detective/github-action@v1 with: @@ -46,11 +64,32 @@ jobs: # create a PR/issue only if the workflow wasn't already triggered by a PR create_pr_on_change: true create_issue_on_fail: true + token: ${{ env.ACTIONS_BOT_TOKEN }} - - name: Test Console docs + - name: Test Redpanda Console docs if: needs.setup.outputs.console == 'true' uses: doc-detective/github-action@v1 with: input: ../modules/console working_directory: redpanda-docs/setup-tests exit_on_fail: true + env: + REDPANDA_GITHUB_TOKEN: ${{ env.ACTIONS_BOT_TOKEN }} + + - name: Test Redpanda Self-Managed quickstart + if: needs.setup.outputs.quickstart == 'true' + uses: doc-detective/github-action@v1 + with: + input: ../modules/get-started/pages/quick-start.adoc + working_directory: redpanda-docs/setup-tests + exit_on_fail: true + env: + REDPANDA_GITHUB_TOKEN: ${{ env.ACTIONS_BOT_TOKEN }} + - name: Upload debug artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: doc-detective-output + path: /home/runner/work/_temp/doc-detective-output.json + env: + REDPANDA_GITHUB_TOKEN: ${{ env.ACTIONS_BOT_TOKEN }} diff --git a/antora.yml b/antora.yml index 4bf3598f1c..d9d52c97d5 100644 --- a/antora.yml +++ b/antora.yml @@ -1,6 +1,6 @@ name: ROOT title: Self-Managed -version: 24.2 +version: 24.3 start_page: home:index.adoc nav: - modules/ROOT/nav.adoc @@ -15,11 +15,16 @@ asciidoc: # Fallback versions # We try to fetch the latest from GitHub at build time # -- - full-version: 24.2.2 - latest-release-commit: '72ba3d3' + full-version: 24.3.1 + latest-redpanda-tag: 'v24.3.1' + latest-console-tag: 'v2.7.2' + latest-release-commit: 'afe1a3f' latest-operator-version: 'v2.2.0-24.2.2' latest-redpanda-helm-chart-version: 5.8.3 - redpanda-beta-version: 24.2.2-rc5 + redpanda-beta-version: '24.3.1-rc2' + redpanda-beta-tag: 'v24.3.1-rc2' + console-beta-version: '2.8.0-beta.1' + console-beta-tag: 'v2.8.0-beta.1' # -- supported-kubernetes-version: 1.25.0-0 supported-helm-version: 3.10.0 diff --git a/local-antora-playbook.yml b/local-antora-playbook.yml index cc6752aef9..66ce9062d9 100644 --- a/local-antora-playbook.yml +++ b/local-antora-playbook.yml @@ -40,6 +40,7 @@ antora: extensions: - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-info' - require: '@redpanda-data/docs-extensions-and-macros/extensions/unpublish-pages' + - require: '@redpanda-data/docs-extensions-and-macros/extensions/collect-bloblang-samples' - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-categories' - require: '@redpanda-data/docs-extensions-and-macros/extensions/modify-redirects' - require: '@redpanda-data/docs-extensions-and-macros/extensions/unlisted-pages' diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 1f2a472244..67463c3ab8 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -3,9 +3,19 @@ ** xref:get-started:whats-new.adoc[] ** xref:get-started:intro-to-events.adoc[Introduction to Redpanda] ** xref:get-started:architecture.adoc[How Redpanda Works] +** xref:console:index.adoc[Introduction to Redpanda Console] ** xref:get-started:install-beta.adoc[Install Beta] -** xref:get-started:quick-start.adoc[Quickstart] -** xref:get-started:licenses.adoc[Redpanda Licensing] +** xref:get-started:quickstarts.adoc[Quickstarts] +*** xref:get-started:quick-start.adoc[Redpanda Self-Managed] +*** xref:console:quickstart.adoc[Redpanda Console] +** xref:get-started:licensing/index.adoc[Redpanda Licensing] +*** xref:get-started:licensing/overview.adoc[Editions and Enterprise Features] +*** xref:get-started:licensing/add-license-redpanda/index.adoc[Add License] +**** xref:get-started:licensing/add-license-redpanda/linux.adoc[Linux] +**** xref:get-started:licensing/add-license-redpanda/kubernetes.adoc[Kubernetes] +**** xref:console:ui/add-license.adoc[Redpanda Console] +*** xref:get-started:licensing/monitor-license-status.adoc[Check Status] +*** xref:get-started:licensing/disable-enterprise-features.adoc[Disable Enterprise Features] ** xref:get-started:rpk/index.adoc[Redpanda CLI] *** xref:get-started:intro-to-rpk.adoc[Introduction to rpk] *** xref:get-started:rpk-install.adoc[] @@ -21,6 +31,7 @@ ** xref:develop:produce-data/index.adoc[Produce Data] *** xref:develop:produce-data/configure-producers.adoc[] *** xref:develop:produce-data/idempotent-producers.adoc[Idempotent Producers] +*** xref:develop:produce-data/leader-pinning.adoc[] ** xref:develop:consume-data/index.adoc[Consume Data] *** xref:develop:consume-data/consumer-offsets.adoc[Consumer Offsets] *** xref:develop:consume-data/follower-fetching.adoc[Follower Fetching] @@ -34,6 +45,7 @@ *** xref:develop:data-transforms/deploy.adoc[Deploy] *** xref:develop:data-transforms/test.adoc[Test] *** xref:develop:data-transforms/monitor.adoc[Monitor] +*** xref:console:ui/data-transforms.adoc[Manage in Redpanda Console] *** xref:develop:data-transforms/upgrade.adoc[Upgrade] *** xref:develop:data-transforms/versioning-compatibility.adoc[Versioning and Compatibility] *** xref:develop:data-transforms/labs.adoc[Examples] @@ -123,9 +135,6 @@ **** xref:manage:kubernetes/monitoring/k-monitor-connectors.adoc[Connectors] *** xref:manage:kubernetes/k-rolling-restart.adoc[Rolling Restart] *** xref:manage:kubernetes/k-resilience-testing.adoc[Resilience Testing] -*** xref:manage:kubernetes/troubleshooting/index.adoc[Troubleshooting] -**** xref:manage:kubernetes/troubleshooting/k-troubleshoot.adoc[] -**** xref:manage:kubernetes/troubleshooting/k-diagnostics-bundle.adoc[Diagnostics Bundle] ** xref:manage:cluster-maintenance/index.adoc[Cluster Maintenance] *** xref:manage:cluster-maintenance/cluster-property-configuration.adoc[] *** xref:manage:cluster-maintenance/node-property-configuration.adoc[] @@ -140,7 +149,6 @@ *** xref:manage:cluster-maintenance/manage-throughput.adoc[Manage Throughput] *** xref:manage:cluster-maintenance/compaction-settings.adoc[Compaction Settings] *** xref:manage:cluster-maintenance/configure-availability.adoc[Configure Availability] -*** xref:manage:cluster-maintenance/cluster-diagnostics.adoc[Cluster Diagnostics] *** xref:manage:cluster-maintenance/partition-recovery.adoc[Forced Partition Recovery] *** xref:manage:cluster-maintenance/nodewise-partition-recovery.adoc[Node-wise Partition Recovery] ** xref:manage:security/index.adoc[Security] @@ -154,19 +162,24 @@ *** xref:manage:security/iam-roles.adoc[] ** xref:manage:tiered-storage-linux/index.adoc[Tiered Storage] *** xref:manage:tiered-storage.adoc[] +*** xref:manage:topic-iceberg-integration.adoc[Iceberg topics] *** xref:manage:fast-commission-decommission.adoc[] +*** xref:manage:mountable-topics.adoc[] *** xref:manage:remote-read-replicas.adoc[Remote Read Replicas] *** xref:manage:topic-recovery.adoc[Topic Recovery] *** xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] ** xref:manage:schema-reg/index.adoc[Schema Registry] -*** xref:manage:schema-reg/schema-reg-overview.adoc[] -*** xref:manage:schema-reg/schema-reg-api.adoc[] +*** xref:manage:schema-reg/schema-reg-overview.adoc[Overview] +*** xref:manage:schema-reg/manage-schema-reg.adoc[] +**** xref:manage:schema-reg/schema-reg-api.adoc[API] +**** xref:console:ui/schema-reg.adoc[Redpanda Console] +**** xref:manage:kubernetes/k-schema-controller.adoc[Kubernetes] *** xref:manage:schema-reg/schema-id-validation.adoc[] +*** xref:console:ui/schema-reg.adoc[Manage in Redpanda Console] ** xref:manage:console/index.adoc[Redpanda Console] -*** xref:console:index.adoc[Overview] -*** xref:console:quickstart.adoc[Quickstart] *** xref:console:config/index.adoc[Configuration] **** xref:console:config/configure-console.adoc[Configure Console] +**** xref:console:config/enterprise-license.adoc[Add an Enterprise License] **** xref:console:config/connect-to-redpanda.adoc[Connect to Redpanda] **** xref:console:config/security/index.adoc[Security] ***** xref:console:config/security/authentication.adoc[Authentication] @@ -183,8 +196,6 @@ **** xref:console:config/deserialization.adoc[Deserialization] **** xref:console:config/kafka-connect.adoc[Kafka Connect] **** xref:console:config/topic-documentation.adoc[Topic Documentation] -*** xref:console:ui/schema-reg.adoc[Schema Registry] -*** xref:console:ui/data-transforms.adoc[Data Transforms] *** xref:console:ui/programmable-push-filters.adoc[Filter Messages] *** xref:console:ui/record-deserialization.adoc[Deserialize Messages] *** xref:console:ui/edit-topic-configuration.adoc[Edit Topic Configuration] @@ -195,6 +206,17 @@ ** xref:manage:monitoring.adoc[] ** xref:manage:io-optimization.adoc[] ** xref:manage:raft-group-reconfiguration.adoc[Raft Group Reconfiguration] +* xref:troubleshoot:index.adoc[Troubleshoot] +** xref:troubleshoot:cluster-diagnostics/index.adoc[Cluster Diagnostics] +*** xref:troubleshoot:cluster-diagnostics/diagnose-issues.adoc[Linux] +*** xref:troubleshoot:cluster-diagnostics/k-diagnose-issues.adoc[Kubernetes] +** xref:troubleshoot:debug-bundle/index.adoc[Generate Debug Bundle] +*** xref:troubleshoot:debug-bundle/generate-debug-bundle.adoc[Linux] +*** xref:troubleshoot:debug-bundle/k-generate-debug-bundle.adoc[Kubernetes] +*** xref:console:ui/generate-bundle.adoc[Redpanda Console] +** xref:troubleshoot:errors-solutions/index.adoc[Resolve Errors] +*** xref:troubleshoot:errors-solutions/resolve-errors.adoc[Linux] +*** xref:troubleshoot:errors-solutions/k-resolve-errors.adoc[Kubernetes] * xref:reference:index.adoc[Reference] ** xref:reference:properties/index.adoc[] *** xref:reference:properties/broker-properties.adoc[] @@ -271,9 +293,16 @@ ***** xref:reference:rpk/rpk-cluster/rpk-cluster-self-test-status.adoc[] ***** xref:reference:rpk/rpk-cluster/rpk-cluster-self-test-stop.adoc[] **** xref:reference:rpk/rpk-cluster/rpk-cluster-storage.adoc[] +***** rpk cluster storage mount/unmount +****** xref:reference:rpk/rpk-cluster/rpk-cluster-storage-cancel-mount.adoc[] +****** xref:reference:rpk/rpk-cluster/rpk-cluster-storage-list-mount.adoc[] +****** xref:reference:rpk/rpk-cluster/rpk-cluster-storage-list-mountable.adoc[] +****** xref:reference:rpk/rpk-cluster/rpk-cluster-storage-mount.adoc[] +****** xref:reference:rpk/rpk-cluster/rpk-cluster-storage-status-mount.adoc[] +****** xref:reference:rpk/rpk-cluster/rpk-cluster-storage-unmount.adoc[] ***** xref:reference:rpk/rpk-cluster/rpk-cluster-storage-restore.adoc[] -****** xref:reference:rpk/rpk-cluster/rpk-cluster-storage-restore-start.adoc[] -****** xref:reference:rpk/rpk-cluster/rpk-cluster-storage-restore-status.adoc[] +***** xref:reference:rpk/rpk-cluster/rpk-cluster-storage-restore-start.adoc[] +***** xref:reference:rpk/rpk-cluster/rpk-cluster-storage-restore-status.adoc[] **** xref:reference:rpk/rpk-cluster/rpk-cluster-txn.adoc[] ***** xref:reference:rpk/rpk-cluster/rpk-cluster-txn-describe.adoc[] ***** xref:reference:rpk/rpk-cluster/rpk-cluster-txn-describe-producers.adoc[] diff --git a/modules/console/images/broker-overview.png b/modules/console/images/broker-overview.png index 79281acacf..ad52e5b5c9 100644 Binary files a/modules/console/images/broker-overview.png and b/modules/console/images/broker-overview.png differ diff --git a/modules/console/images/js-filter.png b/modules/console/images/js-filter.png index 0af427dfad..23b627ac05 100644 Binary files a/modules/console/images/js-filter.png and b/modules/console/images/js-filter.png differ diff --git a/modules/console/images/license.png b/modules/console/images/license.png new file mode 100644 index 0000000000..da720acc75 Binary files /dev/null and b/modules/console/images/license.png differ diff --git a/modules/console/images/overview.png b/modules/console/images/overview.png index 740b83c397..96042370c5 100644 Binary files a/modules/console/images/overview.png and b/modules/console/images/overview.png differ diff --git a/modules/console/images/topic-documentation.png b/modules/console/images/topic-documentation.png index d2311749bd..513fe90a92 100644 Binary files a/modules/console/images/topic-documentation.png and b/modules/console/images/topic-documentation.png differ diff --git a/modules/console/images/topic.png b/modules/console/images/topic.png index 05a2e23f3b..a914096438 100644 Binary files a/modules/console/images/topic.png and b/modules/console/images/topic.png differ diff --git a/modules/console/images/user.png b/modules/console/images/user.png index 568a8c0922..65b1046cec 100644 Binary files a/modules/console/images/user.png and b/modules/console/images/user.png differ diff --git a/modules/console/pages/config/configure-console.adoc b/modules/console/pages/config/configure-console.adoc index 0f2a562e5d..b5ba9a5075 100644 --- a/modules/console/pages/config/configure-console.adoc +++ b/modules/console/pages/config/configure-console.adoc @@ -53,7 +53,7 @@ The following YAML file contains a complete list of all Redpanda Console configu [CAUTION] ==== * Where necessary, ensure that values are enclosed in quotes and escaped. For example, put passwords with special characters in single quotes. -* This configuration file contains both Redpanda Enterprise and Redpanda Community Edition configurations. If you don't provide an Enterprise license, Redpanda Console ignores configurations for enterprise features. +* This configuration file contains both Redpanda Enterprise and Redpanda Community Edition configurations. If you don't provide an enterprise license, Redpanda Console ignores configurations for enterprise features. ==== xref:shared:attachment$redpanda-console-config.yaml[Download the sample file]. diff --git a/modules/console/pages/config/connect-to-redpanda.adoc b/modules/console/pages/config/connect-to-redpanda.adoc index dafc393fc0..0e0ff7828d 100644 --- a/modules/console/pages/config/connect-to-redpanda.adoc +++ b/modules/console/pages/config/connect-to-redpanda.adoc @@ -63,9 +63,10 @@ kafka: enableFast: true ---- +[[admin]] == Configure access to the Redpanda Admin API -Configuring a connection to the Redpanda Admin API enables additional Redpanda-specific features in Redpanda Console, such as viewing the Redpanda version, managing data transforms, and SASL-SCRAM users. +Configuring a connection to the Redpanda Admin API enables additional Redpanda-specific features in Redpanda Console, such as viewing the Redpanda version, managing data transforms and SASL-SCRAM users, and generating debug bundles. [,yaml] ---- @@ -84,6 +85,8 @@ redpanda: # insecureSkipTlsVerify: false ---- +NOTE: Make sure to include the URLs of _all_ brokers in the `redpanda.adminApi.urls` array. + == Suggested reading - xref:console:config/deserialization.adoc[] diff --git a/modules/console/pages/config/enterprise-license.adoc b/modules/console/pages/config/enterprise-license.adoc new file mode 100644 index 0000000000..d2dca6c15d --- /dev/null +++ b/modules/console/pages/config/enterprise-license.adoc @@ -0,0 +1,72 @@ += Add a License Key to Redpanda Console +:description: Learn how to apply or update a license key to Redpanda Console. + +To enable xref:get-started:licensing/overview.adoc#console[enterprise features for Redpanda Console], you must have an Enterprise Edition license to load at startup. This guide explains how to configure Redpanda Console to load the license key from its local configuration. + +TIP: Redpanda Console can also load the license key from a connected Redpanda cluster. To add a license key to Redpanda, see xref:get-started:licensing/add-license-redpanda/index.adoc[]. + +== Prerequisites + +You must have an Enterprise Edition license. To get a trial license key or extend your trial period, https://redpanda.com/try-enterprise[generate a new trial license key^]. To purchase a license, contact https://redpanda.com/upgrade[Redpanda Sales^]. + +If Redpanda Console has enterprise features enabled and cannot find a valid license locally or in the connected Redpanda cluster, it shuts down. See xref:get-started:licensing/overview.adoc[]. + +== Add a new license to Redpanda Console + +To add a new license to Redpanda Console, you have two options: + +- <>. +- <>. + +[[file]] +=== Use a license file + +Specify the path to the license file in one of the following ways: + +- Set the `licenseFilepath` property in the `/etc/redpanda/redpanda-console-config.yaml` configuration file: ++ +```yaml +licenseFilepath: +``` +- Set the `REDPANDA_LICENSE_FILEPATH` environment variable: ++ +```bash +export REDPANDA_LICENSE_FILEPATH= +``` + +[[inline]] +=== Use the license key contents directly + +If you don't want to provide a path to the license file, you can use the contents of the license key directly in one of the following ways: + +- Set the `license` property in the `/etc/redpanda/redpanda-console-config.yaml` configuration file: ++ +```yaml +license: +``` + +- Set the `REDPANDA_LICENSE` environment variable: ++ +```yaml +export REDPANDA_LICENSE= +``` + +== Update an existing license + +To update an existing license: + +. Update your configuration file or environment variables with one of the following: + +- <> +- <> + +. Restart Redpanda Console to make the changes take effect. + +== Next steps + +xref:get-started:licensing/monitor-license-status.adoc[]. + +== Suggested reading + +- xref:get-started:licensing/index.adoc[] +- xref:console:ui/add-license.adoc[] diff --git a/modules/console/pages/config/security/authorization.adoc b/modules/console/pages/config/security/authorization.adoc index 7493c0f999..b8385851d8 100644 --- a/modules/console/pages/config/security/authorization.adoc +++ b/modules/console/pages/config/security/authorization.adoc @@ -47,6 +47,9 @@ It does not include permission to create/remove ACLs or to create or remove a se The `admin` role grants all permissions that come with the `editor` role and additionally includes: +* Access to the *Admin* page that includes: +** Details about all users and ACLs +** The ability to xref:console:ui/generate-bundle.adoc[generate debug bundles] * Managing all service account aspects (create/remove service accounts) * Managing all ACL aspects (create/remove ACLs) diff --git a/modules/console/pages/config/topic-documentation.adoc b/modules/console/pages/config/topic-documentation.adoc index 0e8eb21820..9359c8f5e3 100644 --- a/modules/console/pages/config/topic-documentation.adoc +++ b/modules/console/pages/config/topic-documentation.adoc @@ -4,7 +4,7 @@ You can embed your topic's documentation into the Redpanda Console user interface by providing access to a public or private Git repository that hosts your documentation files in Markdown format. -//image::topic-documentation.png[] +image::topic-documentation.png[] Redpanda Console clones the provided Git repository and stores all Markdown files it finds in memory. The *Documentation* tab in the frontend displays the content of the Markdown file that matches the name of the Kafka topic. diff --git a/modules/console/pages/index.adoc b/modules/console/pages/index.adoc index 7eac7749f0..630bcf18bb 100644 --- a/modules/console/pages/index.adoc +++ b/modules/console/pages/index.adoc @@ -1,4 +1,4 @@ -= Overview of Redpanda Console += Introduction to Redpanda Console :description: Learn about Redpanda Console: a web interface for managing and interacting with Redpanda clusters. :page-aliases: console:index/index.adoc, console:features/index.adoc, reference:console/index.adoc diff --git a/modules/console/pages/quickstart.adoc b/modules/console/pages/quickstart.adoc index 7230491e1a..7913c36421 100644 --- a/modules/console/pages/quickstart.adoc +++ b/modules/console/pages/quickstart.adoc @@ -77,7 +77,7 @@ include::console:attachment$transactions-schema.json[] ```bash docker compose up -d --wait ``` -// (step {"action":"runShell", "command": "docker pull docker.redpanda.com/redpandadata/redpanda:v${REDPANDA_VERSION:?Set a Redpanda version} && docker pull docker.redpanda.com/redpandadata/console:v${REDPANDA_CONSOLE_VERSION:?Set a Redpanda Console version} && docker pull docker.redpanda.com/redpandadata/connect:latest", "workingDirectory": "../test-resources", "timeout": 100000, "exitCodes": [0,1]}) +// (step {"action":"runShell", "command": "docker pull docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest} && docker pull docker.redpanda.com/redpandadata/${CONSOLE_DOCKER_REPO:-console}:${REDPANDA_CONSOLE_VERSION:-latest} && docker pull docker.redpanda.com/redpandadata/connect:latest", "workingDirectory": "../test-resources", "timeout": 100000, "exitCodes": [0,1]}) // (step {"action":"runShell", "command": "docker compose up -d --wait", "workingDirectory": "../test-resources", "timeout": 50000, "exitCodes": [0,1]}) // (step {"action":"wait", "duration": 10000}) + @@ -141,15 +141,17 @@ Suppose you're asked to find all transactions related to the `.edu` domain. You // (step {"action":"find", "selector": "[data-testid='add-topic-filter-javascript']", "matchText": "JavaScript Filter", "click": true, "timeout": 10000}) . Give your filter a name such as 'Find .edu domains'. // (step {"action":"find", "selector": "[data-testid='add-javascript-filter-name']", "typeKeys": "Find .edu domains", "click": true, "timeout": 10000}) -// (step {"action":"saveScreenshot", "path": "js-filter.png", "directory": "../images", "overwrite": "byVariation"}) +// (step {"action":"saveScreenshot", "path": "js-filter.png", "directory": "../images", "overwrite": "byVariation", "maxVariation": 10}) + image::js-filter.png[] + . Replace the default JavaScript code with the following: + [,js] ---- return value.email.includes(".edu"); ---- + . Click *Save* to apply the filter. // (step {"action":"find", "selector": "[data-testid='add-javascript-filter-save']", "matchText": "Save", "click": true}) + @@ -286,6 +288,24 @@ See also: - xref:console:ui/data-transforms.adoc[] - xref:develop:data-transforms/index.adoc[] +// (step {"action":"goTo", "url": "http://localhost:8080/admin/upload-license"}) +// (step {"action":"wait"}) +// (step {"action":"saveScreenshot", "path": "license.png", "directory": "../images", "overwrite": "byVariation"}) + +== Clean up + +If you don't want to continue experimenting with Redpanda, you can shut it down and delete the containers: + +```bash +docker compose down +``` + +To delete the volumes along with all your cluster data: + +```bash +docker compose down -v +``` + // (step {"action":"runShell", "command": "docker compose down -v", "workingDirectory": "../test-resources"}) // (test end) diff --git a/modules/console/pages/ui/add-license.adoc b/modules/console/pages/ui/add-license.adoc new file mode 100644 index 0000000000..dc84f6e14c --- /dev/null +++ b/modules/console/pages/ui/add-license.adoc @@ -0,0 +1,59 @@ += Manage Enterprise Edition Licenses through Redpanda Console +:description: Learn how to manage Enterprise Edition licenses in Redpanda Console. + +You can add, update and check your xref:get-started:licensing/overview.adoc#console[Enterprise Edition license] for both Redpanda and Redpanda Console directly through the Redpanda Console UI. + +== Prerequisites + +- You must have an Enterprise Edition license. https://www.redpanda.com/contact[Request a license^] if you don't have one already. +- Redpanda Console must be xref:console:config/connect-to-redpanda.adoc[connected to a Redpanda cluster]. +- Redpanda Console must be xref:console:config/connect-to-redpanda.adoc#admin[configured to connect to the Redpanda Admin API]. + +TIP: You can also xref:console:config/enterprise-license.adoc[configure Redpanda Console to load the license key from its local configuration]. + +== Upload a license + +When a license is uploaded through Redpanda Console, it is replicated across the cluster and stored persistently in Redpanda's internal metadata, ensuring it is retained across restarts. + +[CAUTION] +==== +include::console:partial$licensing/kubernetes-note.adoc[] +==== + +To upload a new license directly through the Console UI: + +. Open the *Upload License* page, using one of the following methods: ++ +- *Cluster Overview* page: ++ +Navigate to the *Cluster Overview* page in Redpanda Console. +Under the *Licensing* section, click on the *Upload new license* link to upload a new license key. + +- Expiration warning banner: ++ +If the existing license expires soon, you can click the *Upload license* button in the expiration warning banner. + +. Upload your license. You can drag and drop a license file into the box or copy and paste the license string into the text input. ++ +image::console:license.png[] ++ +When a new license is uploaded, enterprise features in Redpanda Self-Managed are unlocked immediately without requiring a cluster restart. However, to unlock enterprise features in Redpanda Console, you must restart the Redpanda Console instance. + +After restarting Redpanda Console, enterprise features such as RBAC are unlocked. However, to enable and use these features, you must configure them. See xref:console:config/index.adoc[]. + +== Check the license status in Redpanda Console + +You can check the expiration date of a license on the **Cluster Overview** page in Redpanda Console, under the **Licensing** section. + +If the license is due to expire within 30 days, a warning banner is displayed on all pages of Redpanda Console. + +See also: xref:get-started:licensing/monitor-license-status.adoc[]. + +== Next steps + +xref:get-started:licensing/monitor-license-status.adoc[]. + +== Suggested reading + +- xref:get-started:licensing/index.adoc[] +- xref:get-started:licensing/overview.adoc[] \ No newline at end of file diff --git a/modules/console/pages/ui/generate-bundle.adoc b/modules/console/pages/ui/generate-bundle.adoc new file mode 100644 index 0000000000..3f19863b4d --- /dev/null +++ b/modules/console/pages/ui/generate-bundle.adoc @@ -0,0 +1,49 @@ += Manage Debug Bundles in Redpanda Console +:description: Learn how to generate, download, and delete debug bundles in Redpanda Console for comprehensive cluster diagnostics. + +[NOTE] +==== +include::shared:partial$enterprise-and-console.adoc[] +==== + +{description} + +== Prerequisites + +- Redpanda Console must be xref:console:config/connect-to-redpanda.adoc[connected to a Redpanda cluster] and xref:console:config/connect-to-redpanda.adoc#admin[configured to connect to the Redpanda Admin API]. +- Redpanda Console must be xref:console:config/security/authentication.adoc[configured with an authentication provider] and have at least one admin user. + +== Generate a debug bundle + +You can generate a debug bundle for all brokers in the cluster and download it onto your local computer for inspection. + +NOTE: Only one debug bundle can exist at a time. If you generate a new debug bundle, any existing bundle from a previous run will be automatically deleted. + +. Log in as an admin user. +. Click *Admin* in the sidebar. ++ +This option is visible only when logged in as an admin user. +. Go to *Debug bundle*. +. Click *Generate new*. +. Wait until the process is complete. +. Click *debug-bundle.zip* to download the bundle on your local computer. +. Unzip the file to inspect the contents. + +include::troubleshoot:partial$inspect-bundle.adoc[] + +== Delete a debug bundle + +To manually delete a debug bundle: + +. Log in as an admin user. +. Click *Admin* in the sidebar. ++ +This option is visible only when logged in as an admin user. +. Go to *Debug bundle*. +. Click the trash icon next to *debug-bundle.zip* to delete the bundle. + +== Next steps + +- xref:troubleshoot:cluster-diagnostics/index.adoc[] +- xref:troubleshoot:errors-solutions/index.adoc[] + diff --git a/modules/console/partials/licensing/kubernetes-note.adoc b/modules/console/partials/licensing/kubernetes-note.adoc new file mode 100644 index 0000000000..040ab531d9 --- /dev/null +++ b/modules/console/partials/licensing/kubernetes-note.adoc @@ -0,0 +1,3 @@ +ifndef::env-kubernetes[If you use Kubernetes to deploy Redpanda, do] +ifdef::env-kubernetes[Do] +not use Redpanda Console to update the license if it's already set in your Kubernetes resources. During upgrades or redeployments, license values in your Kubernetes resources will override the license set using Redpanda Console. For consistent license management, set the license using either Redpanda Console or Kubernetes resources, but not both. \ No newline at end of file diff --git a/modules/console/test-resources/docker-compose.yml b/modules/console/test-resources/docker-compose.yml index 204699c358..9dcf0a0ff1 100644 --- a/modules/console/test-resources/docker-compose.yml +++ b/modules/console/test-resources/docker-compose.yml @@ -33,7 +33,7 @@ services: # Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system. - --smp 1 - --default-log-level=info - image: docker.redpanda.com/redpandadata/redpanda:v${REDPANDA_VERSION:?Set a Redpanda version} + image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest} container_name: redpanda-0 # Sets the username and password of the SCRAM superuser environment: @@ -68,7 +68,7 @@ services: - --smp 1 - --default-log-level=info - --seeds redpanda-0:33145 - image: docker.redpanda.com/redpandadata/redpanda:v${REDPANDA_VERSION:?Set a Redpanda version} + image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest} container_name: redpanda-1 environment: RP_BOOTSTRAP_USER: "superuser:secretpassword" @@ -99,7 +99,7 @@ services: - --smp 1 - --default-log-level=info - --seeds redpanda-0:33145 - image: docker.redpanda.com/redpandadata/redpanda:v${REDPANDA_VERSION:?Set a Redpanda version} + image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:?Set a Redpanda version} container_name: redpanda-2 environment: RP_BOOTSTRAP_USER: "superuser:secretpassword" @@ -120,7 +120,7 @@ services: #################### console: container_name: redpanda-console - image: docker.redpanda.com/redpandadata/console:v${REDPANDA_CONSOLE_VERSION:?Set a Redpanda Console version} + image: docker.redpanda.com/redpandadata/${CONSOLE_DOCKER_REPO:-console}:${REDPANDA_CONSOLE_VERSION:-latest} networks: - redpanda_network entrypoint: /bin/sh @@ -235,7 +235,7 @@ services: - -X user=superuser - -X pass=secretpassword - -X registry.hosts=redpanda-0:8081 - image: docker.redpanda.com/redpandadata/redpanda:v${REDPANDA_VERSION:?Set a Redpanda version} + image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest} # mount the local directory that contains your schema to the container. volumes: - ./transactions-schema.json:/etc/redpanda/transactions-schema.json diff --git a/modules/deploy/pages/deployment-option/self-hosted/kubernetes/k-deployment-overview.adoc b/modules/deploy/pages/deployment-option/self-hosted/kubernetes/k-deployment-overview.adoc index 2673266fd9..2f16e31e3d 100644 --- a/modules/deploy/pages/deployment-option/self-hosted/kubernetes/k-deployment-overview.adoc +++ b/modules/deploy/pages/deployment-option/self-hosted/kubernetes/k-deployment-overview.adoc @@ -5,106 +5,120 @@ :env-kubernetes: true :page-categories: Deployment, GitOps -Kubernetes is a container orchestration tool that helps you manage Redpanda deployments using declarative configuration files called _manifests_. -Kubernetes provides a standardized way of achieving the following: +Kubernetes is a container orchestration tool that helps you manage Redpanda deployments using declarative configuration files called _manifests_. Kubernetes provides a standardized way of achieving the following: * High availability * Disaster recovery * Scalability -== Redpanda Helm chart vs Redpanda Operator +== Deployment tools -To deploy Redpanda in Kubernetes, you can choose to use Helm for its simplicity, or combine Helm with the Redpanda Operator to leverage custom resource definitions (CRDs) for a more GitOps-friendly deployment process. Redpanda Operator is the recommended deployment option. +To deploy Redpanda in Kubernetes, you can choose to use Helm for its simplicity or the Redpanda Operator to leverage custom resource definitions (CRDs) for a more GitOps-friendly deployment process. The Redpanda Operator is the recommended option for production deployments. |=== -| Feature | Helm | Helm + Redpanda Operator +| Feature | Helm | Redpanda Operator | *Version management* -| Directly supported through Helm, enabling updates and rollbacks. -| Managed through Helm. +| Supported through Helm, enabling updates and rollbacks. +| Managed through Redpanda custom resources, with optional integration with Flux for GitOps workflows. | *Multi-tenant Kubernetes clusters* -| Supported. Helm allows you to deploy multiple independent Redpanda clusters by defining different Helm release names. -| Supported. The Redpanda Operator enables a more declarative approach for managing multi-tenant clusters using the RedpandaList custom resource. +| Supported. Helm allows deployment of multiple independent Redpanda clusters by defining different Helm release names. +| Supported. The Redpanda Operator provides a declarative approach for managing multi-tenant clusters using the RedpandaList custom resource. | *Dependency and configuration management* -| Managed directly through the Redpanda Helm chart. -| Configurations defined in custom resources are passed to Helm, which then manages the dependencies and configurations. +| Managed through the Redpanda Helm chart. +| Configurations are defined in custom resources. |=== -TIP: If you are already using the Redpanda Helm chart and want to migrate to the latest Redpanda Operator to manage your deployment, see xref:upgrade:migrate/kubernetes/helm-to-operator.adoc[Migrate from the Redpanda Helm chart]. +TIP: If you are already using the Redpanda Helm chart and want to migrate to the latest Redpanda Operator for better lifecycle management, see xref:upgrade:migrate/kubernetes/helm-to-operator.adoc[Migrate from the Redpanda Helm chart]. === Helm -https://helm.sh/docs[Helm^] is a package manager for Kubernetes, which simplifies the process of defining, installing, and upgrading Kubernetes applications. Helm uses charts, a collection of files that describe a related set of Kubernetes resources, to deploy applications in a Kubernetes cluster. +https://helm.sh/docs[Helm^] is a package manager for Kubernetes that simplifies defining, installing, and upgrading Kubernetes applications. Helm uses charts, a collection of files describing Kubernetes resources, to deploy applications in a Kubernetes cluster. -The Redpanda Helm chart generates and applies all the manifest files you need for deploying Redpanda in Kubernetes, including: +The Redpanda Helm chart provides all the manifest files required to deploy Redpanda in Kubernetes, including: -- The StatefulSet to manage the Redpanda brokers. -- A Headless ClusterIP Service for internal communication with the Redpanda cluster. -- A NodePort Service for external communication with the Redpanda cluster. +- A StatefulSet to manage Redpanda brokers +- A Headless ClusterIP Service for internal communication with the Redpanda cluster +- A NodePort Service for external communication with the Redpanda cluster -The Redpanda Helm chart comes with default settings that enable a straightforward deployment out of the box. However, Helm also offers customization by allowing you to override these default values. -You can override the defaults in your own YAML files using the `--values` option or directly in the command line with the `--set` option. These changes in Helm chart values not only influence the configuration of the Redpanda cluster, but also determine what Kubernetes resources are deployed and how these resources are configured. +[[helm-and-redpanda-operator]] +=== Redpanda Operator -=== Helm and Redpanda Operator +The Redpanda Operator provides two deployment modes controlled by the `useFlux` flag. The modes differ in how reconciliation is handled: -The recommended option for deploying Redpanda in Kubernetes is a combination of <> and the Redpanda Operator. While the Helm chart facilitates the deployment of Redpanda, the Redpanda Operator manages the lifecycle of Redpanda clusters. The operator uses https://fluxcd.io/flux/concepts/[Flux^] to automate the deployment of the Redpanda Helm chart through the HelmRelease resource. +- *Flux-managed mode (`useFlux: true`)*: +When `useFlux` is set to `true` (default), the Redpanda Operator delegates reconciliation to https://fluxcd.io/flux/concepts/[Flux^] controllers. The workflow is as follows: ++ +. The Redpanda Operator defines the necessary CRDs for Redpanda. A Redpanda resource represents a Redpanda cluster, configured through the Redpanda Helm chart values. +. When the Redpanda Operator detects changes to Redpanda resources, it triggers Flux to reconcile the corresponding HelmRelease resource. +. Flux uses the HelmRelease resource to manage the lifecycle of the Redpanda Helm chart, applying updates as needed. -Flux, when integrated with the Redpanda Operator, uses the HelmRelease resource to manage the lifecycle of the Redpanda Helm chart in the Kubernetes cluster. This integration streamlines the creation and management of Redpanda clusters, ensuring they are always in the desired state as defined by the operator's custom resource definitions (CRDs). +- *Redpanda-managed mode (`useFlux: false`)*: +When `useFlux` is set to `false`, the Redpanda Operator manages and reconciles Redpanda resources directly, without involving Flux. This mode works as follows: ++ +. The Redpanda Operator creates both `HelmRelease` and `HelmRepository` resources but suspends them, keeping Flux inactive. ++ +NOTE: Although these resources are created in a suspended state, the operator still requires the Flux CRDs to be installed. +. The operator directly reconciles Redpanda resources, performing tasks such as installations, updates, and cleanup, normally handled by Flux. +. If `useFlux` is later re-enabled, the operator unsuspends the Flux resources, and Flux resumes lifecycle management. -image::shared:operator-flux-relationship.png[CRDs for the Redpanda Operator include Flux CRDs. Flux manages the Redpanda Helm chart through the HelmRelease resource.] - -Here's how it works with Flux: +Example configuration to specify the deployment mode in Redpanda's custom resource: -. The Redpanda Operator defines the necessary CRDs for Redpanda. The Redpanda resource represents a Redpanda cluster that's configured using values from the Redpanda Helm chart. -. When the Redpanda Operator detects changes to these Redpanda resources, it triggers Flux to reconcile the corresponding HelmRelease resource. -. Flux uses the HelmRelease resource to manage the lifecycle of the Redpanda Helm chart in the cluster. +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + clusterSpec: + useFlux: true # or false, depending on your desired deployment mode +---- -If you already have Flux installed in your environment, you can disable the Flux HelmReleaseController and SourceController in the Redpanda Operator to prevent redundant reconciliations of HelmRelease resources. For details, see xref:./k-production-deployment.adoc#config-operator[Deploy a Redpanda cluster]. +image::shared:operator-flux-relationship.png[CRDs for the Redpanda Operator include Flux CRDs. Flux manages the Redpanda Helm chart through the HelmRelease resource.] -== Managed vs bare-metal +== Kubernetes deployment environments -You can run Redpanda on managed Kubernetes services as well as in bare-metal environments. Managed Kubernetes services offer simpler deployment and maintenance, while bare-metal environments provide complete control and potential cost efficiencies. +You can run Redpanda on managed Kubernetes services as well as in bare-metal environments. Managed Kubernetes services offer simpler deployment and maintenance, while bare-metal environments provide complete control and cost efficiencies. === Managed Kubernetes -Managed Kubernetes services manage one or more components of a Kubernetes cluster for you. Several cloud computing vendors provide this service, such as Google Cloud's Google Kubernetes Engine (GKE) and Amazon Web Services' Elastic Kubernetes Service (EKS). - -Managed Kubernetes platforms provide the following benefits: +Managed Kubernetes services, such as Google Kubernetes Engine (GKE) and Amazon Elastic Kubernetes Service (EKS), handle core components of a Kubernetes cluster, offering benefits such as: -* *Ease of deployment*: Managed Kubernetes platforms allow you to provision cloud instances to serve as worker nodes. These instances are pre-configured with Kubernetes agent software and automatically join your Kubernetes cluster, making the process of deploying Redpanda simpler and more efficient. -* *Control plane maintenance*: The managed service provider maintains and updates the control plane software, ensuring that it remains secure, reliable, and up-to-date. -* *Health monitoring and repairs*: The health of the master nodes is continuously monitored, and repairs are made as necessary. This provides an additional level of confidence in the reliability of the platform. +* *Ease of deployment*: Pre-configured instances join your Kubernetes cluster automatically. +* *Control plane maintenance*: The provider maintains the control plane, ensuring security and reliability. +* *Health monitoring and repairs*: The provider monitors master nodes and repairs them as needed. -However, you are still responsible for deploying and maintaining your Redpanda instances on the worker nodes. +You remain responsible for deploying and maintaining Redpanda instances on worker nodes. === Bare-metal Kubernetes environments -Bare-metal Kubernetes environments refer to any deployments where you are responsible for both the control plane and the worker nodes. Running Redpanda on bare-metal environments offers several advantages: +Bare-metal Kubernetes environments give you complete control over both the control plane and the worker nodes, which can be advantageous when you want the following: -* *Complete control*: With bare-metal Kubernetes, you have full control over every aspect of the deployment. Bare-metal deployments may be beneficial when your needs aren't addressed by managed Kubernetes services. -* *Custom configuration*: Bare-metal allows for granular control over the Kubernetes configuration, meaning you can fine-tune the environment. -* *Cost efficiency*: Owning and operating your own hardware may prove to be more cost-effective. +* *Complete control*: Bare-metal Kubernetes offers control over every aspect of deployment, suited for highly customized environments. +* *Custom configurations*: You have granular control to fine-tune the Kubernetes setup. +* *Cost efficiency*: Owning and operating your hardware may be more economical over time. == Documentation conventions -The Kubernetes documentation follows these conventions: +This documentation follows conventions to help users easily identify Kubernetes resource types and Helm values: -* Resource names: Kubernetes resources names, such as Service or PersistentVolume, are distinguished by the use of Pascal case. These are the names of resources when specified as a kind in manifest files. -* Helm values: Helm values, such as `storage.persistentVolume.enabled`, are rendered in monospace font and written according to the JSON path specification. +* *Resource names*: Kubernetes resource names, such as Service or PersistentVolume, are capitalized and in Pascal case to match the manifest files. +* *Helm values*: Helm values, like `storage.persistentVolume.enabled`, are displayed in monospace font. == Next steps -Whether you're deploying locally or in the cloud, choose one of the following guides to get you started: +Whether you're deploying locally or in the cloud, choose one of the following guides to get started: -* xref:./local-guide.adoc[Local] (kind and minikube) -* xref:./aks-guide.adoc[Azure Kubernetes Service] (AKS) -* xref:./eks-guide.adoc[Elastic Kubernetes Service] (EKS) -* xref:./gke-guide.adoc[Google Kubernetes Engine] (GKE) +* xref:./local-guide.adoc[Local Deployment Guide] (kind and minikube) +* xref:./aks-guide.adoc[Azure Kubernetes Service Guide] (AKS) +* xref:./eks-guide.adoc[Elastic Kubernetes Service Guide] (EKS) +* xref:./gke-guide.adoc[Google Kubernetes Engine Guide] (GKE) -Or, you can explore our xref:./k-production-workflow.adoc[production workflow] to learn more about the requirements and best practices. +Or, explore our xref:./k-production-workflow.adoc[production workflow] to learn about requirements and best practices. include::shared:partial$suggested-reading.adoc[] diff --git a/modules/deploy/pages/deployment-option/self-hosted/kubernetes/k-production-deployment.adoc b/modules/deploy/pages/deployment-option/self-hosted/kubernetes/k-production-deployment.adoc index 2fef4d7e5d..948c747165 100644 --- a/modules/deploy/pages/deployment-option/self-hosted/kubernetes/k-production-deployment.adoc +++ b/modules/deploy/pages/deployment-option/self-hosted/kubernetes/k-production-deployment.adoc @@ -7,7 +7,7 @@ :env-kubernetes: true :page-categories: Deployment, GitOps -This topic describes how to configure and deploy one or more Redpanda clusters in Kubernetes. +This topic describes how to configure and deploy one or more Redpanda clusters and Redpanda Console in Kubernetes. == Prerequisites @@ -17,11 +17,9 @@ You must already have a ConfigMap that stores your `io-config.yaml` file. See xr == Deploy a Redpanda cluster -To deploy Redpanda, you can use the following methods. Regardless of the method you choose to deploy Redpanda, you'll deploy the Redpanda Helm chart, which includes Redpanda and Redpanda Console. Redpanda Console comes bundled as a subchart within the Redpanda Helm chart. +To deploy Redpanda and Redpanda Console, you can use the following tools: - *Helm and the Redpanda Operator*: The Redpanda Operator extends Kubernetes with custom resource definitions (CRDs), allowing you to define Redpanda clusters as native Kubernetes resources. The resource that the Redpanda Operator uses to represent a Redpanda cluster is the Redpanda resource. -+ -The Redpanda Operator handles the deployment and management of the Redpanda Helm chart for you by using https://fluxcd.io/flux/concepts/[Flux^]. When you deploy a Redpanda resource, the Redpanda Operator takes that configuration and passes it to Flux. Flux, in turn, interacts with Helm, by creating the necessary HelmRepository and HelmRelease resources to deploy and manage the Redpanda Helm chart. - *Helm*: https://helm.sh/docs[Helm^] is a package manager for Kubernetes, which simplifies the process of defining, installing, and upgrading Kubernetes applications. Helm uses charts, a collection of files that describe a related set of Kubernetes resources, to deploy applications in a Kubernetes cluster. TIP: For more details about the differences between these two methods, see xref:./k-deployment-overview.adoc[]. @@ -69,22 +67,6 @@ include::deploy:partial$kubernetes/install-crds.adoc[] image: tag: {latest-operator-version} ---- -+ -[NOTE] -===== -If you already have Flux installed and you want it to continue managing resources across the entire cluster, use the following: - -.`redpanda-operator-values.yaml` -[,yaml] ----- -additionalCmdFlags: - - --enable-helm-controllers=false ----- - -This flag prevents the Redpanda Operator from deploying its own set of Flux controllers that may conflict with existing ones. - -WARNING: Your existing versions of Flux controllers and Flux CRDs must match the versions that come with the Redpanda Operator. You can find these versions on https://github.com/redpanda-data/redpanda-operator/blob/main/src/go/k8s/go.mod#L12-L17[GitHub]. -===== . Deploy the Redpanda Operator. + @@ -121,6 +103,11 @@ spec: chartRef: chartVersion: {latest-redpanda-helm-chart-version} clusterSpec: + #useFlux: true + #enterprise: + #licenseSecretRef: + #name: + #key: statefulset: extraVolumes: |- - name: redpanda-io-config @@ -137,6 +124,7 @@ spec: - xref:reference:k-crd.adoc#k8s-api-github-com-redpanda-data-redpanda-operator-api-redpanda-v1alpha2-chartref[`spec.chartRef`]: Information about the Helm chart that will be used to deploy Redpanda. - `spec.chartRef.chartVersion`: This field specifies the exact version of the Redpanda Helm chart to use for deployment. By setting this value, you <>, which prevents automatic updates that might introduce breaking changes or new features that have not been tested in your environment. - xref:reference:k-crd.adoc#k8s-api-github-com-redpanda-data-redpanda-operator-api-redpanda-v1alpha2-redpandaclusterspec[`spec.clusterSpec`]: This is where you can override default values in the Redpanda Helm chart. Here, you mount the <> to the Pods that run Redpanda. For other configuration details, see <>. +- `spec.clusterSpec.useFlux`: By default, the Redpanda Operator uses Flux controllers to deploy and manage the Redpanda resource. Set this to `false` to disable Flux and instead use the Redpanda Operator controllers. - `spec.clusterSpec.enterprise`: If you want to use enterprise features in Redpanda, uncomment this section and add the details of a Secret that stores your Enterprise Edition license key. For details, see xref:get-started:licenses.adoc[]. - `spec.clusterSpec.statefulset`: Here, you mount the <> to the Pods that run Redpanda. For other configuration details, see <>. @@ -221,6 +209,19 @@ statefulset: ---- + Redpanda reads from this file at startup to optimize itself for the given I/O parameters. ++ +If you want to use enterprise features in Redpanda, add the details of a Secret that stores your Enterprise Edition license key. ++ +.`redpanda-values.yaml` +[,yaml] +---- +enterprise: + licenseSecretRef: + name: + key: +---- ++ +For details, see xref:get-started:licensing/add-license-redpanda/kubernetes.adoc[]. . Install the Redpanda Helm chart to deploy a Redpanda cluster and Redpanda Console. + @@ -452,7 +453,7 @@ If you're using a private repository, always ensure your nodes have the necessar ---- image: repository: docker.redpanda.com/redpandadata/redpanda - tag: "v{full-version}" + tag: "{latest-redpanda-tag}" imagePullSecrets: [] ---- diff --git a/modules/deploy/partials/kubernetes/guides/troubleshoot.adoc b/modules/deploy/partials/kubernetes/guides/troubleshoot.adoc index e1013cbdfc..1f0aa789e4 100644 --- a/modules/deploy/partials/kubernetes/guides/troubleshoot.adoc +++ b/modules/deploy/partials/kubernetes/guides/troubleshoot.adoc @@ -2,6 +2,6 @@ Before troubleshooting your cluster, make sure that you have all the <>. -include::manage:kubernetes/troubleshooting/k-troubleshoot.adoc[tags=deployment,leveloffset=-1] +include::troubleshoot:partial$errors-and-solutions.adoc[tags=deployment,leveloffset=-1] For more troubleshooting steps, see xref:manage:kubernetes/troubleshooting/k-troubleshoot.adoc[Troubleshoot Redpanda in Kubernetes]. \ No newline at end of file diff --git a/modules/develop/pages/kafka-clients.adoc b/modules/develop/pages/kafka-clients.adoc index c149222f80..7294e33e49 100644 --- a/modules/develop/pages/kafka-clients.adoc +++ b/modules/develop/pages/kafka-clients.adoc @@ -54,7 +54,6 @@ ifdef::env-cloud[] endif::[] ifndef::env-cloud[] + -* The `delete.retention.ms` topic configuration in Kafka is not supported. Tombstone markers are not removed for topics with a `compact` xref:develop:config-topics.adoc#change-the-cleanup-policy[cleanup policy]. Redpanda only deletes tombstone markers when topics with a cleanup policy of `compact,delete` have reached their xref:manage:cluster-maintenance/disk-utilization.adoc#configure-message-retention[retention limits]. * Quotas per user for bandwidth and API request rates. However, xref:manage:cluster-maintenance/manage-throughput.adoc#client-throughput-limits[quotas per client and per client group] using AlterClientQuotas and DescribeClientQuotas APIs are supported. endif::[] diff --git a/modules/develop/pages/produce-data/leader-pinning.adoc b/modules/develop/pages/produce-data/leader-pinning.adoc new file mode 100644 index 0000000000..06dd7ebbd6 --- /dev/null +++ b/modules/develop/pages/produce-data/leader-pinning.adoc @@ -0,0 +1,52 @@ += Leader Pinning +// tag::single-source[] +:description: Learn about Leader Pinning and how to configure a preferred partition leader location based on cloud availability zones or regions. + +[NOTE] +==== +include::shared:partial$enterprise-license.adoc[] +==== + +Produce requests that write data to Redpanda topics go through the topic partition leader, which syncs messages across its follower replicas. For a Redpanda cluster deployed across multiple availability zones (AZs), Leader Pinning ensures that a topic's partition leaders are geographically closer to clients, which helps decrease networking costs and guarantees lower latency. + +If consumers are located in the same preferred region or AZ for Leader Pinning, and you have not set up xref:develop:consume-data/follower-fetching.adoc[follower fetching], Leader Pinning can also help reduce networking costs on consume requests. + +== Prerequisites + +Before you can enable Leader Pinning, you must xref:manage:rack-awareness.adoc#configure-rack-awareness[configure rack awareness] on the cluster. If the config_ref:enable_rack_awareness,true,properties/cluster-properties[] cluster configuration property is set to `false`, Leader Pinning is disabled across the cluster. + +== Configure Leader Pinning + +You can use both a topic configuration property and a cluster configuration property to configure Leader Pinning. + +You can set the topic configuration property for individual topics only, or set the cluster-wide configuration property that will enable Leader Pinning by default for all topics. You can also use a combination in which a default setting applies across the cluster, and you toggle the setting on or off for specific topics. + + +This configuration is based on the following scenario: you have Redpanda deployed in a multi-AZ or multi-region cluster, and you have configured each broker so that the config_ref:rack,true,properties/broker-properties[] configuration property contains rack IDs corresponding to the AZ IDs: + +* Set the topic configuration property xref:reference:properties/topic-properties.adoc#redpandaleaderspreference[`redpanda.leaders.preference`]. The property accepts the following string values: ++ +-- +** `none`: Opt out the topic from Leader Pinning. +** `racks:[,,...]`: Specify the preferred location (rack) of all topic partition leaders. The list can contain one or more rack IDs, and you can list the IDs in any order. Spaces in the list are ignored, for example: `racks:rack1,rack2` and `racks: rack1, rack2` are equivalent. You cannot specify empty rack IDs, for example: `racks: rack1,,rack2`. If you specify multiple IDs, Redpanda tries to distribute the partition leader locations equally across brokers in these racks. +-- ++ +This property inherits the default value from the cluster property `default_leaders_preference`. + +* Set the cluster configuration property config_ref:default_leaders_preference,true,properties/cluster-properties[], which specifies the default Leader Pinning configuration for all topics that don’t have `redpanda.leaders.preference` explicitly set. It accepts values in the same format as `redpanda.leaders.preference`. Default: `none` ++ +This property also affects internal topics, such as `__consumer_offsets` and transaction coordinators. All offset tracking and transaction coordination requests get placed within the preferred regions or AZs for all clients, so you see end-to-end latency and networking cost benefits. + +If there is more than one broker in the preferred AZ (or AZs), Leader Pinning distributes partition leaders uniformly across brokers in the AZ. + +== Leader Pinning failover across availability zones + +If there are three AZs: A, B, and C, and A becomes unavailable, the failover behavior is as follows: + +* A topic with "A" as the preferred leader AZ will have its partition leaders uniformly distributed across B and C. +* A topic with "A,B" as the preferred leader AZs will have its partition leaders in B. +* A topic with “B” as the preferred leader AZ will have its partition leaders in B as well. + +== Suggested reading + +* xref:develop:consume-data/follower-fetching.adoc[] diff --git a/modules/get-started/attachments/single-broker b/modules/get-started/attachments/single-broker new file mode 120000 index 0000000000..f41c920cf9 --- /dev/null +++ b/modules/get-started/attachments/single-broker @@ -0,0 +1 @@ +../test-resources/single-broker \ No newline at end of file diff --git a/modules/get-started/attachments/three-brokers b/modules/get-started/attachments/three-brokers new file mode 120000 index 0000000000..aac706a552 --- /dev/null +++ b/modules/get-started/attachments/three-brokers @@ -0,0 +1 @@ +../test-resources/three-brokers \ No newline at end of file diff --git a/modules/get-started/pages/install-beta.adoc b/modules/get-started/pages/install-beta.adoc index 07f22dd71b..4cb70201b6 100644 --- a/modules/get-started/pages/install-beta.adoc +++ b/modules/get-started/pages/install-beta.adoc @@ -2,7 +2,7 @@ :description: Learn how to install the beta version. :publish-only-during-beta: true -Redpanda beta versions provide users the opportunity to test and share feedback on new features before they're finalized in general availability. Beta versions, like `v{redpanda-beta-version}`, are published to `redpanda-unstable` as release candidate (RC) builds. RC builds are not recommended for production use. +Redpanda beta versions provide users the opportunity to test and share feedback on new features before they're finalized in general availability. Beta versions, like `{redpanda-beta-tag}`, are published to `redpanda-unstable` as release candidate (RC) builds. RC builds are not recommended for production use. To install the beta version, select your environment. @@ -16,7 +16,7 @@ Docker:: + [source,bash,subs="attributes+"] ---- -docker pull docker.redpanda.com/redpandadata/redpanda-unstable:v{redpanda-beta-version} +docker pull docker.redpanda.com/redpandadata/redpanda-unstable:{redpanda-beta-tag} ---- . Create a Docker Compose file with the beta version: @@ -59,7 +59,7 @@ services: - --mode dev-container # enable logs for debugging. - --default-log-level=debug - image: docker.redpanda.com/redpandadata/redpanda-unstable:v{redpanda-beta-version} + image: docker.redpanda.com/redpandadata/redpanda-unstable:{redpanda-beta-tag} container_name: redpanda-0 volumes: - redpanda-0:/var/lib/redpanda/data @@ -72,7 +72,7 @@ services: - 19644:9644 console: container_name: redpanda-console - image: docker.redpanda.com/redpandadata/console:v{latest-console-version} + image: docker.redpanda.com/redpandadata/console:{latest-console-tag} networks: - redpanda_network entrypoint: /bin/sh @@ -158,7 +158,7 @@ Install Redpanda with Helm from the RC build in https://hub.docker.com/r/redpand helm repo add redpanda https://charts.redpanda.com helm install redpanda redpanda/redpanda \ --namespace \ - --create-namespace --set image.repository=docker.redpanda.com/redpandadata/redpanda-unstable --set image.tag=v{redpanda-beta-version} + --create-namespace --set image.repository=docker.redpanda.com/redpandadata/redpanda-unstable --set image.tag={redpanda-beta-tag} ---- -- diff --git a/modules/get-started/pages/licenses.adoc b/modules/get-started/pages/licenses.adoc deleted file mode 100644 index aee2e4aeb7..0000000000 --- a/modules/get-started/pages/licenses.adoc +++ /dev/null @@ -1,163 +0,0 @@ -= Redpanda Licensing -:description: Redpanda Self-Managed is free and source-available at the Redpanda GitHub repo. Additional features are included with an Enterprise license. -:page-aliases: introduction:licenses.adoc - -You can deploy Redpanda in a self-hosted environment (Redpanda Self-Managed) or as a fully-managed cloud service (Redpanda Cloud). For Redpanda Self-Managed, select either the Community Edition or the Enterprise Edition: - -* Redpanda Community Edition is free and source-available at the https://github.com/redpanda-data/redpanda[Redpanda GitHub repository^]. -* Redpanda Enterprise Edition requires a license key and includes additional features. - -[NOTE] -==== -To learn about Redpanda Cloud deployments, see the xref:redpanda-cloud:get-started:cloud-overview.adoc[]. -==== - -== Redpanda Community Edition - -Redpanda Community Edition is licensed with the Redpanda https://github.com/redpanda-data/redpanda/blob/dev/licenses/bsl.md[Business Source License^] (BSL). These core features are free and source-available. - -[NOTE] -==== -* Users cannot provide Redpanda as a commercial streaming or queuing service to others. -* The BSL code converts to Apache 2.0 licensing four years after each code merge. -==== - -== Redpanda Enterprise Edition - -Redpanda Enterprise Edition is licensed with the https://github.com/redpanda-data/redpanda/blob/dev/licenses/rcl.md[Redpanda Community License^] (RCL). It includes the free features licensed under the Redpanda BSL, as well as the following features: - -* xref:manage:tiered-storage.adoc[Tiered Storage] -* xref:manage:cluster-maintenance/continuous-data-balancing.adoc[Continuous Data Balancing] -* xref:manage:remote-read-replicas.adoc[Remote Read Replicas] -* xref:manage:audit-logging.adoc[Audit Logging] -* xref:manage:security/fips-compliance.adoc[FIPS Compliance] -* xref:manage:schema-reg/schema-id-validation.adoc[Server-side Schema ID Validation] -* xref:manage:security/authentication.adoc#enable-kerberos[Kerberos Authentication] -* xref:manage:security/authentication.adoc#oidc [OAUTHBEARER/OIDC Authentication] -* xref:manage:security/authorization/rbac.adoc[Redpanda Role-Based Access Control (RBAC)] -* xref:manage:security/console/authorization.adoc[Redpanda Console Authorization (RBAC)] -* xref:manage:security/console/authentication.adoc[Redpanda Console Authentication] -* xref:redpanda-connect:components:outputs/snowflake_put.adoc[Snowflake connector for Redpanda Connect] -* xref:redpanda-connect:components:outputs/splunk_hec.adoc[Splunk connector for Redpanda Connect] - -Enterprise features require a license key. You can evaluate enterprise features with a free 30-day trial. Contact https://redpanda.com/try-redpanda?section=enterprise-trial[Redpanda Sales^] to request a trial license, to extend your trial period, or to purchase an Enterprise Edition license. - -=== Apply a license key to Redpanda - -A license key unlocks Redpanda Enterprise Edition features. - -[tabs] -====== -Linux or Docker:: -+ --- - -To apply the license key to your cluster, run: - -`rpk cluster license set` - -Either provide a path to a file containing the license or provide the license string inline. For example, assuming you use the default admin host/port of `10.0.0.1:9644`, run: - -```bash -rpk cluster license set --path -X admin.hosts=10.0.0.1:9644 -``` - -Or: - -```bash -rpk cluster license set -``` - -If neither the path nor the license string are provided, `rpk` looks for the license in `/etc/redpanda/redpanda.license`. - --- -Kubernetes:: -+ --- - -To apply the license key to your cluster using the Helm chart, -either provide a secret that contains the license or provide the license string inline: - -- Use a secret: -+ -[tabs] -==== ---values:: -+ -.`license-key.yaml` -[,yaml] ----- -enterprise: - licenseSecretRef: - name: - key: ----- -+ -```bash -helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ ---values license-key.yaml -``` - ---set:: -+ -```bash -helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ - --set enterprise.licenseSecretRef.name= \ - --set enterprise.licenseSecretRef.key= -``` -==== -+ -helm_ref:enterprise.licenseSecretRef[] - -- Use an inline string: -+ -[tabs] -==== ---values:: -+ -.`license-key.yaml` -[,yaml] ----- -enterprise: - license: ----- -+ -```bash -helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ ---values license-key.yaml -``` - ---set:: -+ -```bash -helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ - --set enterprise.license= -``` -==== -+ -helm_ref:enterprise.license[] - -If neither the path nor the license string are provided, Redpanda looks for the license in `/etc/redpanda/redpanda.license`. - --- -====== - -=== Check the status of a license - -To check the status of your current license, run: - -`rpk cluster license info -X admin.hosts=10.0.0.1:9644` - -Redpanda sends warning messages in the cluster logs if you enable enterprise features without a valid license key. - -=== Apply a license key to Redpanda Console - -To use an enterprise feature with Redpanda Console, you must provide Redpanda Console with a copy of your license key. -You have two options for providing the license: - -. Specify the path to the license key file either in the `redpanda.licenseFilepath` property of the `/etc/redpanda/redpanda-console-config.yaml` file or in the `REDPANDA_LICENSE_FILEPATH` environment variable. -. Specify the license key file contents directly either in the `redpanda.license` property of the YAML file or in the `REDPANDA_LICENSE` environment variable. - -Redpanda Console checks the license key status on startup and warns you 30 days before the license expires. You can view the license key's expiration date in the startup logs. -If the license key expires at runtime, Redpanda Console shuts down. If the license has already -expired at startup, Redpanda Console prints an error message and exits. diff --git a/modules/get-started/pages/licensing/add-license-redpanda/index.adoc b/modules/get-started/pages/licensing/add-license-redpanda/index.adoc new file mode 100644 index 0000000000..2675ea38a1 --- /dev/null +++ b/modules/get-started/pages/licensing/add-license-redpanda/index.adoc @@ -0,0 +1,3 @@ += Add an Enterprise Edition License to Redpanda +:page-layout: index +:description: Learn how to add an Enterprise Edition license to Redpanda Self-Managed. diff --git a/modules/get-started/pages/licensing/add-license-redpanda/kubernetes.adoc b/modules/get-started/pages/licensing/add-license-redpanda/kubernetes.adoc new file mode 100644 index 0000000000..6ec1546b88 --- /dev/null +++ b/modules/get-started/pages/licensing/add-license-redpanda/kubernetes.adoc @@ -0,0 +1,324 @@ += Add an Enterprise Edition License to Redpanda in Kubernetes +:description: Learn how to add or update a Redpanda Enterprise Edition license in a Kubernetes environment. +:env-kubernetes: true + +To enable xref:get-started:licensing/overview.adoc[enterprise features for Redpanda Self-Managed], you must have an Enterprise Edition license. This guide outlines how to apply or update an Enterprise Edition license for Redpanda Self-Managed in a Kubernetes environment. + +== Prerequisites + +You must have an Enterprise Edition license. To get a trial license key or extend your trial period, https://redpanda.com/try-enterprise[generate a new trial license key^]. To purchase a license, contact https://redpanda.com/upgrade[Redpanda Sales^]. + +== Add a new license + +Redpanda supports the following ways to apply a new license: + +- <>. +- <>. +- <>. + +[[secret]] +=== Use a Kubernetes Secret + +You can store the license in a Kubernetes Secret and reference it in your Helm values or manifest file. + +. Download your license file (`redpanda.license`) and create a Kubernetes Secret: ++ +[,bash] +---- +kubectl create secret generic redpanda-license +--from-file=license=./redpanda.license +--namespace +---- ++ +This command creates a Kubernetes Secret named `redpanda-license` in the specified namespace, containing the license file. + +. Reference the Secret: ++ +[tabs] +====== +Helm + Operator:: ++ +-- +.`redpanda-cluster.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + chartRef: {} + clusterSpec: + enterprise: + licenseSecretRef: + name: redpanda-license + key: license +---- + +```bash +kubectl apply -f redpanda-cluster.yaml --namespace +``` + +-- +Helm:: ++ +-- + +[tabs] +==== +--values:: ++ +.`redpanda-license.yaml` +[source,yaml] +---- +enterprise: + licenseSecretRef: + name: redpanda-license + key: license +---- ++ +```bash +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --values redpanda-license.yaml --reuse-values +``` + +--set:: ++ +[,bash] +---- +helm upgrade --install redpanda redpanda/redpanda \ + --namespace \ + --create-namespace \ + --set enterprise.licenseSecretRef.name=redpanda-license \ + --set enterprise.licenseSecretRef.key=license +---- + +==== +-- +====== + +[[inline]] +=== Provide the license inline + +If you prefer to provide the license string directly, you can do so as follows: + +[tabs] +====== +Helm + Operator:: ++ +-- +.`redpanda-cluster.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + chartRef: {} + clusterSpec: + enterprise: + license: +---- + +```bash +kubectl apply -f redpanda-cluster.yaml --namespace +``` + +-- +Helm:: ++ +-- + +[tabs] +==== +--values:: ++ +.`redpanda-license.yaml` +[source,yaml] +---- +enterprise: + license: +---- ++ +```bash +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --values redpanda-license.yaml --reuse-values +``` + +--set:: ++ +[,bash] +---- +helm upgrade --install redpanda redpanda/redpanda \ + --namespace \ + --create-namespace \ + --set enterprise.license= +---- + +==== +-- +====== + + +[[console]] +=== Use Redpanda Console + +You can upload a license directly through Redpanda Console. + +See xref:console:ui/add-license.adoc[]. + +== Verify a license + +After adding or updating a license, you can use `rpk` to verify that the license was set. + +include::get-started:partial$licensing/check-license.adoc[] + +== Update an existing license + +The process for updating a license depends on how it was originally applied: + +- <>. +- <>. +- xref:console:ui/add-license.adoc[Use Redpanda Console] + +When a new license is uploaded, enterprise features in Redpanda Self-Managed are unlocked immediately without requiring a cluster restart. However, to unlock enterprise features in Redpanda Console, you must restart the Redpanda Console instance. + +[[secret-update]] +=== Update the Kubernetes Secret + +If the license is provided through a Kubernetes Secret, follow these steps to update it: + +. Download the updated license file and overwrite the existing `redpanda.license` file. + +. Delete the existing Secret: ++ +[,bash] +---- +kubectl delete secret redpanda-license --namespace +---- + +. Create a new Secret with the updated license: ++ +[,bash] +---- +kubectl create secret generic redpanda-license \ + --from-file=license=./redpanda.license \ + --namespace +---- ++ + +. Check the status of new license to make sure it was successfully applied: ++ +[,bash] +---- +rpk cluster license info +---- ++ +The output displays the following details: ++ +---- +Organization: Organization the license was generated for. +Type: Type of license. +Expires: Expiration date of the license. +Version: License schema version. +---- + +. If you use Redpanda Console, delete the Redpanda Console Pods to force Redpanda Console to reload the updated license: ++ +[,bash] +---- +kubectl delete pod $(kubectl get pod --namespace | grep redpanda-console | awk '{print $1}') --namespace +---- + +[[inline-update]] +=== Update the license inline + +If you applied the license inline, follow these steps to update it: + +. Modify the `enterprise.license` value with the new license string: ++ +[tabs] +====== +Helm + Operator:: ++ +-- +.`redpanda-cluster.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + chartRef: {} + clusterSpec: + enterprise: + license: +---- + +```bash +kubectl apply -f redpanda-cluster.yaml --namespace +``` + +-- +Helm:: ++ +-- + +[tabs] +==== +--values:: ++ +.`redpanda-license.yaml` +[source,yaml] +---- +enterprise: + license: +---- ++ +```bash +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --values redpanda-license.yaml --reuse-values +``` + +--set:: ++ +[,bash] +---- +helm upgrade --install redpanda redpanda/redpanda \ + --namespace \ + --create-namespace \ + --set enterprise.license= +---- + +==== +-- +====== + +. Check the status of new license to make sure it was successfully applied: ++ +[,bash] +---- +rpk cluster license info +---- ++ +The output displays the following details: ++ +---- +Organization: Organization the license was generated for. +Type: Type of license:. +Expires: Expiration date of the license. +Version: License schema version. +---- + +. If you use Redpanda Console, delete the Redpanda Console Pods to force a reload of the updated license: ++ +[,bash] +---- +kubectl delete pod $(kubectl get pod --namespace | grep redpanda-console | awk '{print $1}') --namespace +---- + +== Next steps + +xref:get-started:licensing/monitor-license-status.adoc[]. \ No newline at end of file diff --git a/modules/get-started/pages/licensing/add-license-redpanda/linux.adoc b/modules/get-started/pages/licensing/add-license-redpanda/linux.adoc new file mode 100644 index 0000000000..936929208d --- /dev/null +++ b/modules/get-started/pages/licensing/add-license-redpanda/linux.adoc @@ -0,0 +1,73 @@ += Add an Enterprise Edition License to Redpanda in Linux +:description: Learn how to add or update a Redpanda Enterprise Edition license in a Linux environment. + +To enable xref:get-started:licensing/overview.adoc[enterprise features for Redpanda Self-Managed], you must have an Enterprise Edition license. This guide outlines how to apply or update an Enterprise Edition license for Redpanda Self-Managed in a Linux environment. + +== Prerequisites + +- You must have xref:get-started:rpk-install.adoc[`rpk` installed] and configured to connect to your Redpanda cluster. + +- You must have an Enterprise Edition license. To get a trial license key or extend your trial period, https://redpanda.com/try-enterprise[generate a new trial license key^]. To purchase a license, contact https://redpanda.com/upgrade[Redpanda Sales^]. + +== Add a new license + +Redpanda supports the following ways to apply a new license: + +- <>. +- <>. +- <>. + +[[file]] +=== Apply the license using a license file + +If you have the license key stored in a file, you can apply it by specifying the file path: + +```bash +rpk cluster license set --path -X admin.hosts= +``` + +Replace the following placeholders: + +- `` with the path to your license file. +- `` the Redpanda admin host and port + +[[inline]] +=== Apply the license using an inline license string + +If you want to provide the license string directly, use the following command: + +```bash +rpk cluster license set +``` + +If neither the path nor the license string are provided, `rpk` looks for the license in `/etc/redpanda/redpanda.license`. + +[[console]] +=== Use Redpanda Console + +You can upload a license directly through Redpanda Console. + +See xref:console:ui/add-license.adoc[]. + +== Verify a license + +After adding or updating a license, you can use `rpk` to verify that the license was set. + +include::get-started:partial$licensing/check-license.adoc[] + +== Update an existing license + +To update an existing license, you can use one the following methods: + +- xref:reference:rpk/rpk-cluster/rpk-cluster-license-set.adoc[] +- xref:console:ui/add-license.adoc[Redpanda Console] + +When a new license is uploaded, enterprise features in Redpanda Self-Managed are unlocked immediately without requiring a cluster restart. However, to unlock enterprise features in Redpanda Console, you must restart the Redpanda Console instance. + +== Next steps + +xref:get-started:licensing/monitor-license-status.adoc[]. + +== Suggested reading + +xref:reference:rpk/rpk-cluster/rpk-cluster-license-set.adoc[] \ No newline at end of file diff --git a/modules/get-started/pages/licensing/disable-enterprise-features.adoc b/modules/get-started/pages/licensing/disable-enterprise-features.adoc new file mode 100644 index 0000000000..863e76d7c6 --- /dev/null +++ b/modules/get-started/pages/licensing/disable-enterprise-features.adoc @@ -0,0 +1,163 @@ += Disable Enterprise Features in Redpanda +:description: Disable specific enterprise features in Redpanda to ensure your cluster operates within the scope of the Community Edition without enterprise features. + +Enterprise features in Redpanda are available only in the Enterprise Edition and require a valid license. If your cluster has enterprise features enabled without a valid license, it is essential to either upload a valid license or disable these features to maintain compliance with Redpanda licensing terms. + +== Prerequisites + +Before you begin, consider the following: + +- Take a backup of your current configuration to allow rollback if needed. + +- Disabling enterprise features may affect cluster performance, security, or functionality. Test these changes in a staging environment before applying them to production. + +- If you need enterprise features, consider purchasing a valid license to continue using them. To get a trial license key or extend your trial period, https://redpanda.com/try-enterprise[generate a new trial license key^]. To purchase a license, contact https://redpanda.com/upgrade[Redpanda Sales^]. + +== Check for enterprise features in violation + +To check the status of your license, use the `rpk cluster license info` command. This command provides a detailed overview of your Redpanda license, including whether your cluster has enterprise features enabled without a valid license. + +[NOTE] +==== +This command reports license violations only if enterprise features in Redpanda are enabled without a valid license. It does not report license violations for enterprise features in xref:get-started:licensing/overview.adoc#connect[Redpanda Connect] or xref:get-started:licensing/overview.adoc#console[Redpanda Console]. +==== + +. Ensure that xref:get-started:rpk-install.adoc[`rpk` is installed] and configured to connect to your cluster's Admin API endpoint. + +. Get the details about your cluster's license: ++ +[,bash] +---- +rpk cluster license info +---- ++ +If the `license violation` status is `true`, you must either obtain a valid license or disable the enterprise features in use to ensure compliance. + +== Disable enterprise features + +To disable specific enterprise features, refer to the following table: + +[NOTE] +==== +These instructions apply to bare-metal deployments on Linux. If you are running Redpanda in a different environment, such as Kubernetes or Docker, the way you disable features may vary. +==== + +[cols="1a,1a"] +|=== +| Feature | Action to Disable + +| xref:manage:audit-logging.adoc[Audit Logging] +| +Set the cluster config `audit_enabled` to `false`: + +[,bash] +---- +rpk cluster config set audit_enabled false +---- + +| xref:manage:cluster-maintenance/continuous-data-balancing.adoc[Continuous Data Balancing] +| +Set the cluster config `partition_autobalancing_mode` to `node_add`: + +[,bash] +---- +rpk cluster config set partition_autobalancing_mode node_add +---- + +| xref:manage:cluster-maintenance/cluster-balancing.adoc#intra-broker-partition-balancing[Continuous Intra-Broker Partition Balancing] +(`core_balancing_continuous`) +| +Set the cluster config `core_balancing_continuous` to `false`: + +[,bash] +---- +rpk cluster config set core_balancing_continuous false +---- + +| xref:manage:security/fips-compliance.adoc[FIPS Compliance] +| +Set the node config `fips_mode` to `disabled`: + +[,bash] +---- +rpk node config set fips_mode disabled +---- + +| xref:manage:security/authentication.adoc#kerberos[Kerberos authentication] +| +Remove `GSSAPI` from the cluster config `sasl_mechanisms`: + +[,bash] +---- +rpk cluster config set sasl_mechanisms +---- + +| xref:manage:security/authentication.adoc#kerberos[Leader Pinning] +| +Set `default_leaders_preference` to `none`: + +[,bash] +---- +rpk cluster config set default_leaders_preference none +---- + +| xref:manage:security/authentication.adoc#oidc[OAUTHBEARER/OIDC authentication] +| +Remove `OIDC` from the cluster config `sasl_mechanisms` and `http_authentication`: +[,bash] +---- +rpk cluster config set sasl_mechanisms +rpk cluster config set http_authentication +---- + +| xref:manage:remote-read-replicas.adoc[Remote Read Replicas] +| +Set the cluster config `cloud_storage_enable_remote_read` to `false`: + +[,bash] +---- +rpk cluster config set cloud_storage_enable_remote_read false +---- + +| xref:manage:security/authorization/rbac.adoc[Role-Based Access Control (RBAC)] +| +Use `rpk security role delete` to delete all configured roles: + +[,bash] +---- +rpk security role list +rpk security role delete +---- + +| xref:manage:schema-reg/schema-id-validation.adoc[Server-Side Schema ID Validation] +| +Set the cluster config `enable_schema_id_validation` to `false`: + +[,bash] +---- +rpk cluster config set enable_schema_id_validation false +---- + +| xref:manage:tiered-storage.adoc[Tiered Storage] +| +Set the cluster config `cloud_storage_enabled` to `false`: + +[,bash] +---- +rpk cluster config set cloud_storage_enabled false +---- + +|=== + +== Verify the license status + +When all required changes are made, confirm that the `license violation` status is now `false`. + +[,bash] +---- +rpk cluster license info +---- + +== Suggested reading + +For more information about licensing, see xref:get-started:licensing/overview.adoc[]. diff --git a/modules/get-started/pages/licensing/index.adoc b/modules/get-started/pages/licensing/index.adoc new file mode 100644 index 0000000000..945a073380 --- /dev/null +++ b/modules/get-started/pages/licensing/index.adoc @@ -0,0 +1,2 @@ += Redpanda Licensing +:page-layout: index \ No newline at end of file diff --git a/modules/get-started/pages/licensing/monitor-license-status.adoc b/modules/get-started/pages/licensing/monitor-license-status.adoc new file mode 100644 index 0000000000..f94ec8cd99 --- /dev/null +++ b/modules/get-started/pages/licensing/monitor-license-status.adoc @@ -0,0 +1,198 @@ += Check the Status of an Enterprise Edition License +:description: pass:q[Learn how to monitor the status of your Redpanda Enterprise Edition license using `rpk`, Redpanda Console, and observability tools such as logs and metrics.] + +To track license expiration and ensure compliance with xref:get-started:licensing/overview.adoc[Redpanda licensing], you can monitor the status of your Enterprise Edition license using <>, <>, and observability tools such as <> and <>. + +[[rpk]] +== Use `rpk` to check license status and feature usage + +To check the status of your license, use the `rpk cluster license info` command. This command provides a detailed overview of your Redpanda license, including its status, expiration, and enterprise features currently in use. + +NOTE: Enterprise features in use are specific to Redpanda. They do not include enterprise features for xref:get-started:licensing/overview.adoc#connect[Redpanda Connect] or xref:get-started:licensing/overview.adoc#console[Redpanda Console]. + +. Make sure that xref:get-started:rpk-install.adoc[`rpk` is installed] and configured to connect to your cluster's Admin API endpoint. + +. Get the details about your cluster's license: ++ +```bash +rpk cluster license info +``` ++ +The command displays the license information in a user-friendly format. The output depends on the license status. It can include the following: ++ +[cols="1,2a"] +|=== +|Field |Description + +|License status +| +Indicates the current state of the license. Possible values include: + +* `valid`: The license is active and valid. +* `expired`: The license has expired, and restrictions may apply to enterprise features. +* `not_present`: No license is currently applied. + +|License violation +| +A boolean value that indicates whether enterprise features are being used in violation of the licensing terms: + +* `true`: Enterprise features are enabled without a valid license. +* `false`: No violations are detected. + +[NOTE] +==== +If the `license violation` status is `true`, you must either obtain a valid license or disable the enterprise features in use to ensure compliance. +For instructions, see xref:get-started:licensing/disable-enterprise-features.adoc[]. +==== + +|Organization +| +The organization for which the license was issued. Typically corresponds to the organization that purchased or activated the license. + +|Type +| +The type of license applied to the cluster: + +* `enterprise`: A full-featured enterprise license. +* `trial`: A time-limited trial license. + +|Expires +| +The expiration date of the current license. After this date, the license status changes to `expired`. + +|Enterprise features in use +| +A list of enabled enterprise features in the cluster, such as `tiered_storage`, `audit_logging`, and `partition_auto_balancing_continuous`. +|=== + +If the license is within 30 days of expiration, a warning is logged. `rpk` displays warnings when you execute `rpk` commands that use the Admin API in the following scenarios: + +- *License violation*: When enterprise features are enabled without a valid license. + +- *Trial expiration*: When enterprise features are enabled and a trial license expires in less than 15 days. + +- *Enterprise expiration*: When enterprise features are enabled and an enterprise license is expired. + +=== Examples + +This section provides examples of what `rpk` reports depending on the license status. + +Valid license: + +[.no-copy] +---- +LICENSE INFORMATION +=================== +License status: valid +License violation: false +Enterprise features in use: [partition_auto_balancing_continuous] +Organization: Devex +Type: enterprise +Expires: Oct 11 2025 +---- + +Without a valid license: + +[.no-copy] +---- +LICENSE INFORMATION +=================== +License status: not_present +License violation: false +Enterprise features in use: [partition_auto_balancing_continuous] +---- + +Expired license: + +[.no-copy] +---- +WARNING: The following enterprise features are being used in your Redpanda cluster: [partition_auto_balancing_continuous]. These features require a license. +LICENSE INFORMATION +=================== +License status: expired +License violation: true +Enterprise features in use: [partition_auto_balancing_continuous] +---- + +=== Change the output format + +Different output formats can be useful depending on your scenario. For example, if you are writing scripts or automating license monitoring, you may prefer the JSON format, as it's easily parsed by tools like `jq` or integrated into monitoring systems. + +To get the license information in another format, use the `--format` flag: + +```bash +rpk cluster license info --format +``` + +Replace `` with one of the available formats. See xref:reference:rpk/rpk-cluster/rpk-cluster-license-info.adoc[]. + +[[console]] +== Use Redpanda Console to check license status + +You can also check the status of your license in the Redpanda Console UI. See xref:console:ui/add-license.adoc[]. + +Redpanda Console tries to load a valid license at startup in the following order: + +. From the local configuration file or environment variables. +. From the connected Redpanda cluster (if available). + +Redpanda Console displays warnings in the following scenarios: + +- *Upcoming license expiration*: Redpanda Console checks the license status at startup. If the license expires within 30 days, a warning is displayed in the UI. Redpanda Console also logs the license details at startup, including the expiration date. For example: ++ +[,json] +---- +{"level":"info","msg":"successfully loaded Redpanda Enterprise license","license_org":"redpanda","license_type":"enterprise","expires_at":"Oct 12 2024"} +---- + +- *Redpanda Self-Managed enterprise features are in use without a valid license*: If the Redpanda cluster is using enterprise features without a valid license, Redpanda Console displays a warning specifying the features in use without proper licensing. + +== Use the Redpanda Operator to check license status + +If you deployed a Redpanda cluster in Kubernetes using the Redpanda Operator, you can monitor the Enterprise Edition license status from the Redpanda custom resource. The operator integrates license monitoring into the resource's `status.conditions` field, allowing you to get the license state with `kubectl`. + +Identify your Redpanda resources: + +[,bash] +---- +kubectl get redpanda -A +---- + +Example output: + +[.no-copy] +---- +NAMESPACE NAME LICENSE READY STATUS +redpanda cluster Valid True Redpanda reconciliation succeeded +---- + +The `License` field indicates whether the license is valid. Possible values include: + +- `Valid`: The license is valid. +- `Expired`: The license has expired. +- `Not Present`: No license is applied. + +The Redpanda Operator performs a license check during reconciliation, which is triggered in the following cases: + +- If changes are made to the Redpanda resource or its status. + +- If changes are made to resources managed by the operator, such as the StatefulSet resources for brokers or Deployment resources for Redpanda Console. + +- If no changes occur, the operator reconciles every 10 hours (default cache resync interval). + +- If you force reconciliation by making a no-op change, such as adding an annotation, to the Redpanda resource. + +[[logs]] +== Monitor a license in the logs + +Logs are emitted on Redpanda brokers to help you understand how the license is being applied and when enforcement actions are triggered. + +[[metrics]] +== Monitor a license using metrics + +The xref:reference:public-metrics-reference.adoc#redpanda_cluster_features_enterprise_license_expiry_sec[`redpanda_cluster_features_enterprise_license_expiry_sec`] metric tells you how many seconds remain until the license expires. Use this metric to set up alerts or dashboards to track the license expiration status and ensure timely renewal. + +== Next steps + +- xref:get-started:licensing/disable-enterprise-features.adoc[] + diff --git a/modules/get-started/pages/licensing/overview.adoc b/modules/get-started/pages/licensing/overview.adoc new file mode 100644 index 0000000000..54eeab09dd --- /dev/null +++ b/modules/get-started/pages/licensing/overview.adoc @@ -0,0 +1,165 @@ += Redpanda Licenses and Enterprise Features +:description: Learn about Redpanda licensing for Redpanda, Redpanda Console, and Redpanda Connect, available in both Community and Enterprise editions. Understand licensing requirements and how to access enterprise features with a valid license key. +:page-aliases: introduction:licenses.adoc, get-started:licenses.adoc + +Redpanda, Redpanda Console, and Redpanda Connect are available in community and enterprise editions. Each product has a single binary that supports both editions. + +* Redpanda Community Edition is free and source-available on GitHub: +** https://github.com/redpanda-data/redpanda[Redpanda^] +** https://github.com/redpanda-data/console[Redpanda Console^] +** https://github.com/redpanda-data/connect[Redpanda Connect^] +* Redpanda Enterprise Edition requires a license key and includes additional features. + +[NOTE] +==== +Redpanda Cloud is a managed deployment of Redpanda Enterprise Edition. To learn more about Redpanda Cloud, see the xref:redpanda-cloud:get-started:cloud-overview.adoc[]. +==== + +== Community Edition + +The Community Edition is licensed with the Redpanda https://github.com/redpanda-data/redpanda/blob/dev/licenses/bsl.md[Business Source License^] (BSL). These core features are free and source-available. + +=== Restrictions and licensing terms + +* Users cannot provide Redpanda as a commercial streaming or queuing service to others. +* The BSL code converts to Apache 2.0 licensing four years after each code merge. + +== Enterprise Edition + +The Enterprise Edition is licensed with the https://github.com/redpanda-data/redpanda/blob/dev/licenses/rcl.md[Redpanda Community License^] (RCL). It includes the free features licensed under the Redpanda BSL, as well as enterprise features. + +=== License keys + +Enterprise features require an Enterprise Edition license key, sometimes referred to as enterprise license, license key, or license. + +Starting with version 24.3, new Redpanda clusters automatically receive a trial license that's valid for 30 days, allowing unrestricted use of enterprise features. This evaluation period begins when the cluster is created for the first time. After this period expires, inactive enterprise features are disabled, and active features enter a restricted state. + +To get a trial license key or extend your trial period, https://redpanda.com/try-enterprise[generate a new trial license key]. Or, https://redpanda.com/upgrade[upgrade to Redpanda Enterprise^]. + +[IMPORTANT] +==== +Redpanda Connect does not accept trial license keys. To use enterprise features of Redpanda Connect, https://redpanda.com/upgrade[upgrade to Redpanda Enterprise^] +==== + +[NOTE] +==== +include::get-started:partial$licensing/block-upgrades-note.adoc[] +==== + +[[self-managed]] +=== Redpanda enterprise features + +The following table lists the Enterprise features for Redpanda and how Redpanda behaves upon license expiration when each enterprise feature is enabled. + +- *Upon license expiration*: + +** The cluster continues to operate without data loss, but the further use of enterprise features is restricted (see the <>). +** Configuration of enterprise features remains unchanged, allowing you to add a new license and continue using enterprise features as before expiration. For instructions, see xref:get-started:licensing/add-license-redpanda/index.adoc[] + +- *After license expiration*: You cannot enable enterprise features without a valid license. + +[[redpanda-enterprise-features]] +.Enterprise features in Redpanda +[cols="1a,2a,2a"] +|=== +| Feature | Description | Behavior Upon Expiration + +| xref:manage:audit-logging.adoc[Audit Logging] +| Records detailed logs of cluster activities for compliance and monitoring. +| Read access to the audit log topic is denied, but logging continues. + +| xref:manage:cluster-maintenance/continuous-data-balancing.adoc[Continuous Data Balancing] +| Automatically balances partitions across a cluster to optimize resource use and performance. +| Continuous balancing is disabled, reverting to the default `node_add` setting that balances partitions only after a broker is added to the cluster. + +| xref:manage:cluster-maintenance/cluster-balancing.adoc#intra-broker-partition-balancing[Continuous Intra-Broker Partition Balancing] +(`core_balancing_continuous`) +| Balances partition replicas across CPU cores in an individual broker to optimize disk space usage. +| Continuous Intra-Broker Partition Balancing is disabled. + +| xref:manage:security/fips-compliance.adoc[FIPS Compliance] +| Enables compliance with FIPS security standards for cryptography. +| No change. + +| xref:manage:security/authentication.adoc#kerberos[Kerberos Authentication] +| Provides secure Kerberos-based authentication. +| No change. + +| xref:manage:security/authentication.adoc#kerberos[Leader Pinning] +| Specifies the set of availability zones where the leaders of partitions of a given topic should be located. +| Leader Pinning is disabled on all topics. + +| xref:manage:security/authentication.adoc#oidc[OAUTHBEARER/OIDC Authentication] +| Allows for OAUTHBEARER and OpenID Connect (OIDC) authentication. +| No change. + +| xref:manage:remote-read-replicas.adoc[Remote Read Replicas] +| Enables remote clusters to read data stored in object storage for disaster recovery. +| Remote Read Replica topics cannot be created or modified. + +| xref:manage:security/authorization/rbac.adoc[Role-Based Access Control (RBAC)] +| Manages user roles and permissions within the cluster. +| Roles and ACLs associated with roles cannot be created or modified. Role deletion is allowed. + +| xref:manage:schema-reg/schema-id-validation.adoc[Server-Side Schema ID Validation] +| Validates schema IDs server-side to ensure schema compatibility. With schema ID validation, records associated with unregistered schemas are detected and dropped by a broker rather than a consumer. +| Topics with schema validation settings cannot be created or modified. + +| xref:manage:tiered-storage.adoc[Tiered Storage] +| Enables data storage in cloud object storage for long-term retention and retrieval. +| Topics cannot be created or modified to enable Tiered Storage features. Additional partitions cannot be added to topics with Tiered Storage properties enabled. + +|=== + +[[console]] +=== Redpanda Console enterprise features + +The following enterprise features for Redpanda Console are activated with a valid Enterprise Edition license key: + +.Enterprise features in Redpanda Console +[cols="1a,2a,2a"] +|=== +| Feature | Description | Restrictions Without Valid License + +|xref:console:config/security/authentication.adoc[Redpanda Console Authentication] +|Enables authentication for Redpanda Console, allowing secure access. +|All endpoints are redirected to the license expiration landing page, and all other access is restricted. + +|xref:console:config/security/authorization.adoc[Redpanda Console Authorization (RBAC)] +|Manages user roles and permissions for accessing features within Redpanda Console. +|All endpoints are redirected to the license expiration landing page, and all other access is restricted. +|=== + +[[connect]] +=== Redpanda Connect enterprise features + +The Enterprise Edition of Redpanda Connect includes additional connectors. An Enterprise Edition license is required to enable them. + +For all available connectors, see xref:redpanda-connect:components:catalog.adoc[]. + +- *License expires during runtime*: All enterprise features for Redpanda Connect become unavailable. +- *License expired at startup*: Redpanda Connect logs an error message and blocks the use of all enterprise features. + +[IMPORTANT] +==== +Redpanda Connect does not accept trial license keys. To use enterprise features of Redpanda Connect, https://redpanda.com/upgrade[upgrade to Redpanda Enterprise^] +==== + +== Manage licenses for Redpanda + +Redpanda offers multiple ways to manage your license depending on your deployment method. Proper license management ensures access to advanced enterprise features and avoids restrictions when licenses are invalid or expired. + +See xref:get-started:licensing/add-license-redpanda/index.adoc[]. + +== Manage licenses for Redpanda Console + +Redpanda Console offers two methods for applying or updating a license, depending on your setup: + +- If you use Redpanda Console without connecting it to a Redpanda cluster, you can xref:console:config/enterprise-license.adoc[configure the license through the local configuration file or environment variables]. This method allows you to add a license for Redpanda Console independently of the Redpanda cluster. + +- If Redpanda Console is connected to a Redpanda cluster, you can xref:console:ui/add-license.adoc[upload a license through the Redpanda Console UI]. This method allows you to manage and update licenses for both Redpanda Console and the connected Redpanda cluster. + +== Next steps + +- xref:get-started:licensing/add-license-redpanda/index.adoc[] +- xref:get-started:licensing/monitor-license-status.adoc[] diff --git a/modules/get-started/pages/quick-start.adoc b/modules/get-started/pages/quick-start.adoc index 0317142a73..e2e1c8684a 100644 --- a/modules/get-started/pages/quick-start.adoc +++ b/modules/get-started/pages/quick-start.adoc @@ -1,8 +1,12 @@ = Redpanda Self-Managed Quickstart :description: Learn how to quickly start working with a local Redpanda cluster. -:page-aliases: install-upgrade:index.adoc, install-upgrade:index/index.adoc, install-upgrade:start-streaming.adoc, quickstart:console-installation, quickstart:quick-start-docker.adoc, quickstart:quick-start-linux.adoc, quickstart:quick-start-macos.adoc, quickstart:quick-start-windows.adoc, getting-started:quick-start-docker.adoc, getting-started:quick-start-linux.adoc, getting-started:quick-start-windows.adoc, getting-started:quick-start-macos.adoc, console:installation.adoc, get-started:quick-start/quick-start-console.adoc, get-started:quick-start/quick-start-macos.adoc, get-started:quick-start/quick-start-linux.adoc, get-started:quick-start/quick-start-docker.adoc, get-started:quickstarts/index.adoc +:page-aliases: install-upgrade:index.adoc, install-upgrade:index/index.adoc, install-upgrade:start-streaming.adoc, quickstart:console-installation, quickstart:quick-start-docker.adoc, quickstart:quick-start-linux.adoc, quickstart:quick-start-macos.adoc, quickstart:quick-start-windows.adoc, getting-started:quick-start-docker.adoc, getting-started:quick-start-linux.adoc, getting-started:quick-start-windows.adoc, getting-started:quick-start-macos.adoc, console:installation.adoc, get-started:quick-start/quick-start-console.adoc, get-started:quick-start/quick-start-macos.adoc, get-started:quick-start/quick-start-linux.adoc, get-started:quick-start/quick-start-docker.adoc :page-categories: Deployment, Development, rpk :env-docker: true +// ========================AUTOMATED TESTS=================================== +// The comments in this file are used to run automated tests of all the documented steps. Tests are run on each pull request to the upstream repository using GitHub Actions. For more details about the testing tool we use, see https://doc-detective.com/. + +// (test start {"id": "quickstart", "description": "Redpanda Self-Managed quickstart"}) {description} @@ -46,7 +50,7 @@ For production environments where you need more resilience, a three-broker setup Single Broker:: + -- -. xref:redpanda-labs:docker-compose:attachment$single-broker/docker-compose.yml[Download] the following `docker-compose.yml` file on your local file system. +. xref:get-started:attachment$single-broker/docker-compose.yml[Download] the following `docker-compose.yml` file on your local file system. + .Reveal the YAML content [%collapsible] @@ -54,10 +58,14 @@ Single Broker:: .`docker-compose.yml` [,yaml,subs="attributes+"] ---- -include::redpanda-labs:docker-compose:attachment$single-broker/docker-compose.yml[] +include::get-started:attachment$single-broker/docker-compose.yml[] ---- ==== +// (step {"action":"runShell", "command": "docker pull docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest} && docker pull docker.redpanda.com/redpandadata/${CONSOLE_DOCKER_REPO:-console}:${REDPANDA_CONSOLE_VERSION:-latest}", "workingDirectory": "../test-resources/single-broker", "timeout": 100000, "exitCodes": [0,1]}) +// (step {"action":"runShell", "command": "docker compose up -d --wait", "workingDirectory": "../test-resources/single-broker", "timeout": 50000, "exitCodes": [0,1]}) +// (step {"action":"wait", "duration": 10000}) + . Run the following in the directory where you saved the `docker-compose.yml` file: + ```bash @@ -77,7 +85,7 @@ Three Brokers:: + -- -. xref:redpanda-labs:docker-compose:attachment$three-brokers/docker-compose.yml[Download] the following `docker-compose.yml` file on your local file system. +. xref:get-started:attachment$three-brokers/docker-compose.yml[Download] the following `docker-compose.yml` file on your local file system. + .Reveal the YAML content [%collapsible] @@ -85,7 +93,7 @@ Three Brokers:: .`docker-compose.yml` [,yaml,subs="attributes+"] ---- -include::redpanda-labs:docker-compose:attachment$three-brokers/docker-compose.yml[] +include::get-started:attachment$three-brokers/docker-compose.yml[] ---- ==== @@ -125,6 +133,9 @@ To use `rpk` inside the Redpanda broker's Docker container: ```bash docker exec -it redpanda-0 rpk cluster info ``` +// (step {"action":"runShell", "command": "rpk profile create quickstart --from-profile rpk-profile.yaml", "workingDirectory": "../test-resources"}) +// (step {"action":"runShell", "command": "rpk cluster info", "workingDirectory": "../test-resources/single-broker"}) + . Create a topic called **chat-room**: + @@ -147,6 +158,7 @@ If you deployed three brokers, you can configure your topics with a replication docker exec -it redpanda-0 rpk topic create chat-room --replicas 3 ``` ==== +// (step {"action":"runShell", "command": "rpk topic create chat-room", "workingDirectory": "../test-resources/single-broker"}) . Produce a message to the topic: + @@ -166,6 +178,8 @@ Example output: ---- Produced to partition 0 at offset 0 with timestamp 1663282629789. ---- +// (step {"action":"runShell", "command": "echo 'Pandas are fabulous!' | rpk topic produce chat-room", "workingDirectory": "../test-resources/single-broker"}) + . Press kbd:[Ctrl + C] to finish producing messages to the topic. @@ -186,6 +200,8 @@ Your message is displayed along with its metadata: "offset": 0 } ``` +// (step {"action":"runShell", "command": "rpk topic consume chat-room --num 1", "output": "/Pandas are fabulous!/", "timeout": 10000}) + To test external connectivity using your local machine: @@ -295,6 +311,10 @@ To delete the volumes along with all your cluster data: docker compose down -v ``` +// (step {"action":"runShell", "command": "rpk profile delete quickstart"}) +// (step {"action":"runShell", "command": "docker compose down -v", "workingDirectory": "../test-resources/single-broker"}) +// (test end) + == Next steps - xref:redpanda-labs:ROOT:index.adoc[Try more examples in Redpanda Labs] diff --git a/modules/get-started/pages/quickstarts.adoc b/modules/get-started/pages/quickstarts.adoc new file mode 100644 index 0000000000..c4f0e22045 --- /dev/null +++ b/modules/get-started/pages/quickstarts.adoc @@ -0,0 +1,3 @@ += Redpanda Quickstarts +:description: Get started with Redpanda using these hands-on tutorials. Explore features that demonstrate how Redpanda can power your streaming applications. +:page-layout: index \ No newline at end of file diff --git a/modules/get-started/pages/whats-new.adoc b/modules/get-started/pages/whats-new.adoc index 7c538c77b6..e567a7f23b 100644 --- a/modules/get-started/pages/whats-new.adoc +++ b/modules/get-started/pages/whats-new.adoc @@ -4,90 +4,130 @@ This topic includes new content added in version {page-component-version}. For a complete list of all product updates, see the https://github.com/redpanda-data/redpanda/releases/[Redpanda release notes^]. See also: -* xref:redpanda-cloud:get-started:whats-new-cloud.adoc[] +* xref:redpanda-cloud:get-started:whats-new-cloud.adoc[] * xref:redpanda-cloud:get-started:cloud-overview.adoc#redpanda-cloud-vs-self-managed-feature-compatibility[Redpanda Cloud vs Self-Managed feature compatibility] +== Leader pinning -== Declarative user and ACL management in Kubernetes +For a Redpanda cluster deployed across multiple availability zones (AZs), xref:develop:produce-data/leader-pinning.adoc[leader pinning] ensures that a topic's partition leaders are geographically closer to clients. Leader pinning can lower networking costs and help guarantee lower latency by routing produce and consume requests to brokers located in certain AZs. -Starting with Redpanda Operator version v2.2.2-24.2.4, you can now manage users and access control lists (ACLs) with the new User resource. This feature allows you to: +== Iceberg topics -- Create and manage Redpanda users and their authentication settings. -- Define and manage ACLs to control access to Redpanda resources. -- Automatically reconcile changes to users and ACLs using the Redpanda Operator. +The xref:manage:topic-iceberg-integration.adoc[Iceberg integration for Redpanda] allows you to store topic data in the cloud in the Iceberg open table format. This makes your streaming data immediately available in downstream analytical systems without setting up and maintaining additional ETL pipelines. You can also integrate your data directly into commonly-used big data processing frameworks, standardizing and simplifying the consumption of streams as tables in a wide variety of data analytics pipelines. -To learn more, see the xref:manage:kubernetes/security/authentication/k-user-controller.adoc[User resource documentation]. +== Mountable topics -== FIPS compliance +For topics with Tiered Storage enabled, you can unmount a topic to safely detach it from a cluster and keep the topic data in the cluster's object storage bucket or container. You can mount the detached topic to either the same origin cluster, or a different one. This allows you to hibernate a topic and free up system resources taken up by the topic, or migrate a topic to a different cluster. See xref:manage:mountable-topics.adoc[Mountable topics] for details. -Redpanda now provides a FIPS 140-compliant binary to support running Redpanda in environments requiring FIPS-compliant software and to meet certain requirements for organizations undergoing certification through government compliance programs, such as FedRAMP. When using the FIPS binary, both Redpanda and `rpk` leverage FIPS-compliant OpenSSL libraries and ciphers for all encryption activity inside Redpanda, including encryption of data in transit. Install the `redpanda-fips` package to xref:manage:security/fips-compliance.adoc[run Redpanda in FIPS-compliance mode]. +== Intra-broker partition balancing -== Schema Registry support for JSON +xref:manage:cluster-maintenance/cluster-balancing.adoc#intra-broker-partition-balancing[Intra-broker partition balancing] has moved out of beta and is supported for production clusters. This dynamically balances partitions across cores within a Redpanda broker. You also can decrease the number of CPUs assigned to a broker. Existing partition replicas are reassigned to the remaining CPU cores when the broker restarts with fewer available cores. You must explicitly enable this feature. -Redpanda's xref:manage:schema-reg/schema-reg-overview.adoc[Schema Registry] now supports JSON serialization format, in addition to Avro and Protobuf. You can use a https://json-schema.org/[JSON Schema^] format to validate messages and ensure compatibility between schema versions. +== Tombstone removal -== Schema Registry read-only mode +Redpanda now supports the Kafka `delete.retention.ms` topic configuration. You can specify how long Redpanda keeps xref:manage:cluster-maintenance/compaction-settings.adoc#tombstone-record-removal[tombstone records] for compacted topics by setting `delete.retention.ms` at the topic level, or `tombstone_retention_ms` at the cluster level. -You can now put individual subjects or the entire Schema Registry into read-only or read-write mode with the xref:manage:schema-reg/schema-reg-api.adoc#use-readonly-mode-for-disaster-recovery[Schema Registry API] `/mode` endpoint or with xref:reference:rpk/rpk-registry/rpk-registry-mode-set.adoc[`rpk registry mode set`]. You can use the `READONLY` mode in an active/passive disaster recovery configuration. +== Debug bundles in Redpanda Console -== TLS enhancements +You can now xref:troubleshoot:debug-bundle/index.adoc[generate a debug bundle] in Redpanda Console for comprehensive diagnostics. A debug bundle can help debug and diagnose issues with a Redpanda cluster, a broker, or the machines on which the brokers are running. You can use this file to debug issues yourself, or you can send it to the Redpanda support team to help resolve your issue. -* *Minimum TLS version*: For enhanced security, Redpanda now has a cluster configuration property to xref:manage:security/encryption.adoc#manage-the-minimum-tls-version[control the minimum TLS version] to use for encryption. By default, xref:reference:properties/cluster-properties.adoc#tls_min_version[`tls_min_version`] is set to `v1.2`, which prevents client applications from using older TLS versions to connect to your clusters. +== Backfill partitions -* *Certificate revocation lists*: When enabling TLS encryption for the Kafka, Admin, HTTP Proxy or Schema Registry APIs, you can now xref:manage:security/encryption.adoc[add a certificate revocation list (CRL)] to your configuration. Redpanda uses the CRL to check and reject connections from entities using certificates already revoked by a certificate authority. +When running xref:manage:cluster-maintenance/nodewise-partition-recovery.adoc[node-wise partition recovery], it's possible that there may be more recent data (a higher offset) available in Tiered Storage. Redpanda attempts to recover partition data from object storage, recovering the latest offset available for a partition in either storage tier (local or object storage). This allows for the maximum amount of data to be recovered in all cases, even for topics with a replication factor of 1, where no replicas remain in local storage. -== Data transforms enhancements +== Configure access to object storage with a KMS key -Redpanda has a new xref:reference:data-transforms/js/index.adoc[JavaScript SDK] that you can use to build and deploy data transforms in Redpanda. To get started, see xref:develop:data-transforms/run-transforms-index.adoc[]. +Users on AWS or GCP with strict data compliance requirements can manage and store encryption keys separately from their cloud provider with a xref:manage:tiered-storage.adoc#configure-object-storage[customer-managed Key Management Service (KMS) key]. -You can now deploy data transform functions that xref:develop:data-transforms/deploy.adoc#reprocess[reprocess existing records] from an input topic. Processing existing records can be useful, for example, to process historical data into a different format for a new consumer, to re-create lost data from an accidentally-deleted topic, or to resolve issues with a previous version of a transform that processed data incorrectly. +== Licensing updates -The docs now also include an xref:develop:data-transforms/index.adoc[expanded guide] designed to help you master the creation, deployment, and management of data transforms in Redpanda. +This release includes several updates to xref:get-started:licensing/overview.adoc[Redpanda's licensing system] to both improve transparency and make it easier to manage licenses across Redpanda clusters and Redpanda Console. -== Enhanced cache trimming +- *Upgrade restriction for enterprise features*: You cannot upgrade your cluster to Redpanda 24.3 or later if any enterprise feature of Redpanda (excluding Console) is in use without a valid license. The upgrade is aborted immediately and cleanly before any broker is upgraded, minimizing disruption. -Redpanda has two new properties that provide finer control over cache management. These settings allow you to define specific thresholds for triggering xref:manage:tiered-storage.adoc#cache-trimming[cache trimming] based on cache size and the number of objects, helping to optimize performance and prevent slow reads. +- *License reminder in `rpk`*: Every `rpk` command now includes a reminder if the cluster is using licensed features without a valid license. -- config_ref:cloud_storage_cache_trim_threshold_percent_size,true,properties/object-storage-properties[] -- config_ref:cloud_storage_cache_trim_threshold_percent_objects,true,properties/object-storage-properties[] +- *License warnings in Redpanda Console*: If you are using Redpanda Console and one or more enterprise features are active in the connected Redpanda cluster without a valid license, a banner specifies which enterprise features are in use without a license. -== Client throughput management +- *Unified license management in Redpanda Console*: You can now upload and apply a single license key for both Redpanda Console and the connected Redpanda cluster through the Redpanda Console UI. Any existing license key is overridden by the new one. -Redpanda now includes `rpk` and Redpanda Console support for managing xref:manage:cluster-maintenance/manage-throughput.adoc#client-throughput-limits[client throughput limits]. You can set client throughput quotas using `rpk` for an individual client based on client ID, as well as for a group of clients based on ID prefix matching. New xref:manage:cluster-maintenance/manage-throughput.adoc#monitor-client-throughput[metrics] are available for insights on client throughput utilization. +- *30-day trial enterprise license*: Starting with version 24.3, new Redpanda clusters automatically receive a trial license that's valid for 30 days, allowing unrestricted use of Enterprise features. This evaluation period begins when the cluster is created for the first time. After this period expires, inactive Enterprise features are disabled, and active features enter a restricted state. -Client throughput quotas, previously applied on a per-shard basis, now apply on a per-broker basis. Cluster configuration properties for managing client quotas are xref:upgrade:deprecated/index.adoc[deprecated], including `target_quota_byte_rate` which is disabled by default with the value `0`. +== PKCS#12 support for TLS listeners -== Self-test enhancements +Redpanda now supports configuring TLS using PKCS#12 files, which allows you to manage both the certificate and private key in a single, password-protected file. -New tests are added to the xref:manage:cluster-maintenance/cluster-diagnostics.adoc[Redpanda self-test] suite: +For more details, see xref:manage:security/encryption.adoc[]. -* Cloud storage tests to validate xref:manage:tiered-storage.adoc[Tiered Storage] configuration. -* 16K block size disk tests to better assess block storage performance, particularly in response to I/O depth changes. -* 4K block size disk test with dsync off to assess the impact of fdatasync on the storage layer. +This feature is not yet available in Kubernetes deployments that use the Redpanda Helm chart or the Redpanda Operator. -See the xref:reference:rpk/rpk-cluster/rpk-cluster-self-test-status.adoc[`rpk self test`] reference for usage and output examples. +== New features in the Redpanda Operator -== Intra-broker partition balancing: beta +=== Declarative user and ACL management -xref:manage:cluster-maintenance/cluster-balancing.adoc#intra-broker-partition-balancing[Intra-broker partition balancing] is a new topic-aware partition balancer that balances partitions across cores within a Redpanda broker. This balancer considers core count changes as well as partition movement to rebalance partition replicas and improve performance. +The Redpanda Operator now supports declarative management of users and access control lists (ACLs) using the new User custom resource, starting in version v2.2.2-24.2.4. This feature allows you to: -This is a beta feature for v24.2 and is not recommended for use for production clusters. +- Create and manage Redpanda users and their authentication settings. +- Define and manage ACLs to control access to Redpanda resources. +- Automatically reconcile changes to users and ACLs using the Redpanda Operator. -== New commands +To learn more, see the xref:manage:kubernetes/security/authentication/k-user-controller.adoc[User custom resource documentation]. -The following `rpk` commands are new in this version: +=== Declarative schema management + +The Redpanda Operator now supports declarative schema management using the Schema custom resource, starting in version v2.3.0-24.3.1. This feature allows you to: -* xref:reference:rpk/rpk-cluster/rpk-cluster-quotas-alter.adoc[`rpk cluster quotas alter`] -* xref:reference:rpk/rpk-cluster/rpk-cluster-quotas-describe.adoc[`rpk cluster quotas describe`] -* xref:reference:rpk/rpk-cluster/rpk-cluster-quotas-import.adoc[`rpk cluster quotas import`] -* xref:reference:rpk/rpk-cluster/rpk-cluster-quotas.adoc[`rpk cluster quotas`] -* xref:reference:rpk/rpk-registry/rpk-registry-mode.adoc[`rpk registry mode`] -* xref:reference:rpk/rpk-registry/rpk-registry-mode-get.adoc[`rpk registry mode get`] -* xref:reference:rpk/rpk-registry/rpk-registry-mode-reset.adoc[`rpk registry mode reset`] -* xref:reference:rpk/rpk-registry/rpk-registry-mode-set.adoc[`rpk registry mode set`] -* xref:reference:rpk/rpk-transform/rpk-transform-pause.adoc[`rpk transform pause`] -* xref:reference:rpk/rpk-transform/rpk-transform-resume.adoc[`rpk transform resume`] +- Define, create, and manage Avro, Protobuf, and JSON schemas declaratively. +- Enforce schema compatibility and evolution policies directly in Kubernetes. +- Automate schema lifecycle management alongside Redpanda deployments. -== Doc enhancements +To learn more, see the xref:manage:kubernetes/k-schema-controller.adoc[Schema custom resource documentation]. + +=== Use Redpanda Operator without Flux + +The Redpanda Operator now supports the `useFlux` flag, giving you control over resource management, starting in version v2.3.0-24.3.1: + +- `useFlux: true` (default): Delegates Redpanda resource management to Flux controllers through HelmRelease resources. +- `useFlux: false`: Directly manages resources within the Redpanda Operator, bypassing Flux. + +Example: + +[,yaml] +---- +spec: + clusterSpec: + useFlux: false +---- + +== New commands + +The following `rpk` commands are new in this version: -The https://docs.redpanda.com/home/[Redpanda Docs home page] has been redesigned, so you can stay within the context of Redpanda Self-Managed, Redpanda Cloud, or Redpanda Connect docs. We hope that our docs help and inspire our users. Please share your feedback with the links at the bottom of any doc page. +* xref:reference:rpk/rpk-cluster/rpk-cluster-storage-cancel-mount.adoc[`rpk cluster storage cancel mount`] +* xref:reference:rpk/rpk-cluster/rpk-cluster-storage-list-mount.adoc[`rpk cluster storage list mount`] +* xref:reference:rpk/rpk-cluster/rpk-cluster-storage-list-mountable.adoc[`rpk cluster storage list-mountable`] +* xref:reference:rpk/rpk-cluster/rpk-cluster-storage-mount.adoc[`rpk cluster storage mount`] +* xref:reference:rpk/rpk-cluster/rpk-cluster-storage-status-mount.adoc[`rpk cluster storage status mount`] +* xref:reference:rpk/rpk-cluster/rpk-cluster-storage-unmount.adoc[`rpk cluster storage unmount`] + +== New properties + +The following cluster properties are new in this version: + +* xref:reference:properties/cluster-properties.adoc#debug_bundle_auto_removal_seconds[`debug_bundle_auto_removal_seconds`] +* xref:reference:properties/cluster-properties.adoc#debug_bundle_storage_dir[`debug_bundle_storage_dir`] +* xref:reference:properties/cluster-properties.adoc#default_leaders_preference[`default_leaders_preference`] +* xref:reference:properties/cluster-properties.adoc#rpk_path[`rpk_path`] +* xref:reference:properties/cluster-properties.adoc#tombstone_retention_ms[`tombstone_retention_ms`] +* xref:reference:properties/cluster-properties.adoc#iceberg_catalog_base_location[`iceberg_catalog_base_location`] +* xref:reference:properties/cluster-properties.adoc#iceberg_catalog_commit_interval_ms[`iceberg_catalog_commit_interval_ms`] +* xref:reference:properties/cluster-properties.adoc#iceberg_catalog_type[`iceberg_catalog_type`] +* xref:reference:properties/cluster-properties.adoc#iceberg_delete[`iceberg_delete`] +* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_client_id[`iceberg_rest_catalog_client_id`] +* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_client_secret[`iceberg_rest_catalog_client_secret`] +* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_crl_file[`iceberg_rest_catalog_crl_file`] +* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_endpoint[`iceberg_rest_catalog_endpoint`] +* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_prefix[`iceberg_rest_catalog_prefix`] +* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_request_timeout_ms[`iceberg_rest_catalog_request_timeout_ms`] +* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_token[`iceberg_rest_catalog_token`] +* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_trust_file[`iceberg_rest_catalog_trust_file`] diff --git a/modules/get-started/partials/licensing/block-upgrades-note.adoc b/modules/get-started/partials/licensing/block-upgrades-note.adoc new file mode 100644 index 0000000000..6e49f06c52 --- /dev/null +++ b/modules/get-started/partials/licensing/block-upgrades-note.adoc @@ -0,0 +1 @@ +Redpanda blocks upgrades to new feature releases if enterprise features are active without a valid license. Ensure compliance by obtaining a license to maintain access to the latest features and updates. \ No newline at end of file diff --git a/modules/get-started/partials/licensing/check-license.adoc b/modules/get-started/partials/licensing/check-license.adoc new file mode 100644 index 0000000000..fae76197a0 --- /dev/null +++ b/modules/get-started/partials/licensing/check-license.adoc @@ -0,0 +1,28 @@ +ifdef::env-kubernetes[] +[,bash] +---- +kubectl exec --namespace -c redpanda -- \ + rpk cluster license info +---- +endif::[] +ifndef::env-kubernetes[] +[,bash] +---- +rpk cluster license info +---- +endif::[] + +This command will display the current license details, including the expiration date and whether any enterprise features are active. For example: + +---- +LICENSE INFORMATION +=================== +Organization: redpanda +Type: enterprise +Expires: Oct 24 2027 +---- + +[NOTE] +==== +include::get-started:partial$licensing/block-upgrades-note.adoc[] +==== diff --git a/modules/get-started/test-resources/rpk-profile.yaml b/modules/get-started/test-resources/rpk-profile.yaml new file mode 100644 index 0000000000..47f52a08a5 --- /dev/null +++ b/modules/get-started/test-resources/rpk-profile.yaml @@ -0,0 +1,7 @@ +description: For use with the docker compose file +kafka_api: + brokers: + - 127.0.0.1:19092 +admin_api: + addresses: + - 127.0.0.1:19644 \ No newline at end of file diff --git a/modules/get-started/test-resources/single-broker/docker-compose.yml b/modules/get-started/test-resources/single-broker/docker-compose.yml new file mode 100644 index 0000000000..61cae308b8 --- /dev/null +++ b/modules/get-started/test-resources/single-broker/docker-compose.yml @@ -0,0 +1,74 @@ +name: redpanda-quickstart +networks: + redpanda_network: + driver: bridge +volumes: + redpanda-0: null + redpanda-1: null + redpanda-2: null +services: + ################## + # Redpanda Brokers # + ################## + redpanda-0: + command: + - redpanda + - start + - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092 + # Address the broker advertises to clients that connect to the Kafka API. + # Use the internal addresses to connect to the Redpanda brokers' + # from inside the same Docker network. + # Use the external addresses to connect to the Redpanda brokers' + # from outside the Docker network. + - --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092 + - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082 + # Address the broker advertises to clients that connect to the HTTP Proxy. + - --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082 + - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081 + # Redpanda brokers use the RPC API to communicate with each other internally. + - --rpc-addr redpanda-0:33145 + - --advertise-rpc-addr redpanda-0:33145 + # Mode dev-container uses well-known configuration properties for development in containers. + - --mode dev-container + # Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system. + - --smp 1 + - --default-log-level=info + image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest} + container_name: redpanda-0 + volumes: + - redpanda-0:/var/lib/redpanda/data + networks: + - redpanda_network + ports: + - 18081:18081 + - 18082:18082 + - 19092:19092 + - 19644:9644 + #################### + # Redpanda Console # + #################### + console: + container_name: redpanda-console + image: docker.redpanda.com/redpandadata/${CONSOLE_DOCKER_REPO:-console}:${REDPANDA_CONSOLE_VERSION:-latest} + networks: + - redpanda_network + entrypoint: /bin/sh + command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console -config.filepath=${CONFIG_FILEPATH:-/tmp/config.yml}' + volumes: + - ./config:/tmp/config/ + environment: + CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml} + CONSOLE_CONFIG_FILE: | + kafka: + brokers: ["redpanda-0:9092"] + schemaRegistry: + enabled: true + urls: ["http://redpanda-0:8081"] + redpanda: + adminApi: + enabled: true + urls: ["http://redpanda-0:9644"] + ports: + - 8080:8080 + depends_on: + - redpanda-0 diff --git a/modules/get-started/test-resources/three-brokers/docker-compose.yml b/modules/get-started/test-resources/three-brokers/docker-compose.yml new file mode 100644 index 0000000000..3847ff27ae --- /dev/null +++ b/modules/get-started/test-resources/three-brokers/docker-compose.yml @@ -0,0 +1,132 @@ +name: redpanda-quickstart +networks: + redpanda_network: + driver: bridge +volumes: + redpanda-0: null + redpanda-1: null + redpanda-2: null +services: + ################## + # Redpanda Brokers # + ################## + redpanda-0: + command: + - redpanda + - start + - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092 + # Address the broker advertises to clients that connect to the Kafka API. + # Use the internal addresses to connect to the Redpanda brokers' + # from inside the same Docker network. + # Use the external addresses to connect to the Redpanda brokers' + # from outside the Docker network. + - --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092 + - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082 + # Address the broker advertises to clients that connect to the HTTP Proxy. + - --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082 + - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081 + # Redpanda brokers use the RPC API to communicate with each other internally. + - --rpc-addr redpanda-0:33145 + - --advertise-rpc-addr redpanda-0:33145 + # Mode dev-container uses well-known configuration properties for development in containers. + - --mode dev-container + # Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system. + - --smp 1 + - --default-log-level=info + image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest} + container_name: redpanda-0 + volumes: + - redpanda-0:/var/lib/redpanda/data + networks: + - redpanda_network + ports: + - 18081:18081 + - 18082:18082 + - 19092:19092 + - 19644:9644 + redpanda-1: + command: + - redpanda + - start + - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092 + - --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092 + - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082 + - --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082 + - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081 + - --rpc-addr redpanda-1:33145 + - --advertise-rpc-addr redpanda-1:33145 + - --mode dev-container + - --smp 1 + - --default-log-level=info + - --seeds redpanda-0:33145 + image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest} + container_name: redpanda-1 + volumes: + - redpanda-1:/var/lib/redpanda/data + networks: + - redpanda_network + ports: + - 28081:28081 + - 28082:28082 + - 29092:29092 + - 29644:9644 + depends_on: + - redpanda-0 + redpanda-2: + command: + - redpanda + - start + - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092 + - --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092 + - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082 + - --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082 + - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081 + - --rpc-addr redpanda-2:33145 + - --advertise-rpc-addr redpanda-2:33145 + - --mode dev-container + - --smp 1 + - --default-log-level=info + - --seeds redpanda-0:33145 + image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:?Set a Redpanda version} + container_name: redpanda-2 + volumes: + - redpanda-2:/var/lib/redpanda/data + networks: + - redpanda_network + ports: + - 38081:38081 + - 38082:38082 + - 39092:39092 + - 39644:9644 + depends_on: + - redpanda-0 + #################### + # Redpanda Console # + #################### + console: + container_name: redpanda-console + image: docker.redpanda.com/redpandadata/${CONSOLE_DOCKER_REPO:-console}:${REDPANDA_CONSOLE_VERSION:-latest} + networks: + - redpanda_network + entrypoint: /bin/sh + command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console -config.filepath=${CONFIG_FILEPATH:-/tmp/config.yml}' + volumes: + - ./config:/tmp/config/ + environment: + CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml} + CONSOLE_CONFIG_FILE: | + kafka: + brokers: ["redpanda-0:9092", "redpanda-1:9092", "redpanda-2:9092"] + schemaRegistry: + enabled: true + urls: ["http://redpanda-0:8081", "http://redpanda-1:8081", "http://redpanda-2:8081"] + redpanda: + adminApi: + enabled: true + urls: ["http://redpanda-0:9644", "http://redpanda-1:9644", "http://redpanda-2:9644"] + ports: + - 8080:8080 + depends_on: + - redpanda-0 + - redpanda-1 + - redpanda-2 diff --git a/modules/manage/examples/kubernetes/schema-crds.feature b/modules/manage/examples/kubernetes/schema-crds.feature new file mode 100644 index 0000000000..0ffe46a1e8 --- /dev/null +++ b/modules/manage/examples/kubernetes/schema-crds.feature @@ -0,0 +1,106 @@ +@cluster:basic +Feature: Schema CRDs + Background: Cluster available + Given cluster "basic" is available + + @skip:gke @skip:aks @skip:eks + Scenario: Manage customer profile schema (Avro) + Given there is no schema "customer-profile" in cluster "basic" + When I apply Kubernetes manifest: + """ +# tag::customer-profile-avro-schema-manifest[] + # This manifest creates an Avro schema named "customer-profile" in the "basic" cluster. + # The schema defines a record with fields for customer ID, name, and age. + --- + apiVersion: cluster.redpanda.com/v1alpha2 + kind: Schema + metadata: + name: customer-profile + spec: + cluster: + clusterRef: + name: basic + schemaType: avro + compatibilityLevel: Backward + text: | + { + "type": "record", + "name": "CustomerProfile", + "fields": [ + { "type": "string", "name": "customer_id" }, + { "type": "string", "name": "name" }, + { "type": "int", "name": "age" } + ] + } +# end::customer-profile-avro-schema-manifest[] + """ + And schema "customer-profile" is successfully synced + Then I should be able to check compatibility against "customer-profile" in cluster "basic" + + @skip:gke @skip:aks @skip:eks + Scenario: Manage product catalog schema (Protobuf) + Given there is no schema "product-catalog" in cluster "basic" + When I apply Kubernetes manifest: + """ +# tag::product-catalog-protobuf-schema-manifest[] + # This manifest creates a Protobuf schema named "product-catalog" in the "basic" cluster. + # The schema defines a message "Product" with fields for product ID, name, price, and category. + --- + apiVersion: cluster.redpanda.com/v1alpha2 + kind: Schema + metadata: + name: product-catalog + spec: + cluster: + clusterRef: + name: basic + schemaType: protobuf + compatibilityLevel: Backward + text: | + syntax = "proto3"; + + message Product { + int32 product_id = 1; + string product_name = 2; + double price = 3; + string category = 4; + } +# end::product-catalog-protobuf-schema-manifest[] + """ + And schema "product-catalog" is successfully synced + Then I should be able to check compatibility against "product-catalog" in cluster "basic" + + @skip:gke @skip:aks @skip:eks + Scenario: Manage order event schema (JSON) + Given there is no schema "order-event" in cluster "basic" + When I apply Kubernetes manifest: + """ +# tag::order-event-json-schema-manifest[] + # This manifest creates a JSON schema named "order-event" in the "basic" cluster. + # The schema requires an "order_id" (string) and a "total" (number) field, with no additional properties allowed. + --- + apiVersion: cluster.redpanda.com/v1alpha2 + kind: Schema + metadata: + name: order-event + spec: + cluster: + clusterRef: + name: basic + schemaType: json + compatibilityLevel: None + text: | + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "order_id": { "type": "string" }, + "total": { "type": "number" } + }, + "required": ["order_id", "total"], + "additionalProperties": false + } +# end::order-event-json-schema-manifest[] + """ + And schema "order-event" is successfully synced + Then I should be able to check compatibility against "order-event" in cluster "basic" diff --git a/modules/manage/examples/kubernetes/topic-crds.feature b/modules/manage/examples/kubernetes/topic-crds.feature new file mode 100644 index 0000000000..4b8218fb58 --- /dev/null +++ b/modules/manage/examples/kubernetes/topic-crds.feature @@ -0,0 +1,27 @@ +@cluster:basic +Feature: Topic CRDs + Background: Cluster available + Given cluster "basic" is available + + @skip:gke @skip:aks @skip:eks + Scenario: Manage topics + Given there is no topic "topic1" in cluster "basic" + When I apply Kubernetes manifest: + """ +# tag::basic-topic-example[] + # In this example manifest, a topic called "topic1" is created in a cluster called "basic". It has a replication factor of 1 and is distributed across a single partition. + --- + apiVersion: cluster.redpanda.com/v1alpha2 + kind: Topic + metadata: + name: topic1 + spec: + cluster: + clusterRef: + name: basic + partitions: 1 + replicationFactor: 1 +# end::basic-topic-example[] + """ + And topic "topic1" is successfully synced + Then I should be able to produce and consume from "topic1" in cluster "basic" \ No newline at end of file diff --git a/modules/manage/examples/kubernetes/user-crds.feature b/modules/manage/examples/kubernetes/user-crds.feature index 3712f6cb94..872b92845e 100644 --- a/modules/manage/examples/kubernetes/user-crds.feature +++ b/modules/manage/examples/kubernetes/user-crds.feature @@ -25,7 +25,7 @@ Feature: User CRDs | jason | [{"type":"allow","resource":{"type":"cluster"},"operations":["Read"]}] | When I apply Kubernetes manifest: """ - # tag::manage-authn-only-manifest[] +# tag::manage-authn-only-manifest[] # In this example manifest, a user called "jason" is created in a cluster called "sasl". # The user's password is defined in a Secret called "jason-password". # This example assumes that you will create ACLs for this user separately. @@ -45,7 +45,7 @@ Feature: User CRDs secretKeyRef: name: jason-password key: password - # end::manage-authn-only-manifest[] +# end::manage-authn-only-manifest[] """ And user "jason" is successfully synced And I delete the CRD user "jason" @@ -58,7 +58,7 @@ Feature: User CRDs | travis | password | SCRAM-SHA-256 | When I apply Kubernetes manifest: """ - # tag::manage-authz-only-manifest[] +# tag::manage-authz-only-manifest[] # In this example manifest, an ACL called "travis" is created in a cluster called "sasl". # The ACL give an existing user called "travis" permissions to read from all topics whose names start with some-topic. # This example assumes that you already have a user called "travis" in your cluster. @@ -79,8 +79,8 @@ Feature: User CRDs name: some-topic patternType: prefixed operations: [Read] - # end::manage-authz-only-manifest[] +# end::manage-authz-only-manifest[] """ And user "travis" is successfully synced And I delete the CRD user "travis" - Then "travis" should be able to authenticate to the "sasl" cluster with password "password" and mechanism "SCRAM-SHA-256" + Then "travis" should be able to authenticate to the "sasl" cluster with password "password" and mechanism "SCRAM-SHA-256" \ No newline at end of file diff --git a/modules/manage/pages/cluster-maintenance/cluster-balancing.adoc b/modules/manage/pages/cluster-maintenance/cluster-balancing.adoc index 1fdc4aad46..37157f004d 100644 --- a/modules/manage/pages/cluster-maintenance/cluster-balancing.adoc +++ b/modules/manage/pages/cluster-maintenance/cluster-balancing.adoc @@ -10,31 +10,32 @@ Cluster balancing is crucial for optimal performance. Unbalanced clusters can sa | Topic-aware data balancer |Description | <> -| This balancer transfers the leadership of a broker's partitions to other replicas to avoid topic leadership hotspots on one or a few specific brokers in your cluster. - + - + +| +This balancer transfers the leadership of a broker's partitions to other replicas to avoid topic leadership hotspots on one or a few specific brokers in your cluster. ++ The partition leader regularly sends heartbeats to its followers. If a follower does not receive a heartbeat within a timeout, it triggers a new leader election. Redpanda also provides leadership balancing when brokers are added or decommissioned. | <> -| This balancer moves partition replicas to avoid topic replica hotspots on one or a few specific brokers in your cluster. - + - + +| +This balancer moves partition replicas to avoid topic replica hotspots on one or a few specific brokers in your cluster. ++ Redpanda prioritizes balancing a topic's partition replica count evenly across all brokers while it's balancing the cluster's overall partition count. Because different topics in a cluster can have vastly different load profiles, this better distributes the workload evenly across brokers. - + - + - ++ Redpanda provides partition replica balancing when brokers are added or decommissioned. | <> -| This balancer moves partition replicas across CPU cores in an individual broker. Redpanda maintains balanced partition replica assignments between cores to avoid topic hotspots on one or a few specific cores within a broker. +| +This balancer moves partition replicas across CPU cores in an individual broker. Redpanda maintains balanced partition replica assignments between cores to avoid topic hotspots on one or a few specific cores within a broker. ++ +Continuous Intra-Broker Partition Balancing (`core_balancing_continuous`) requires an xref:get-started:licenses.adoc#redpanda-enterprise-edition[enterprise license]. | xref:manage:cluster-maintenance/continuous-data-balancing.adoc[Continuous Data Balancing] -| This balancer monitors broker and rack availability, as well as disk usage, to avoid topic hotspots when moving data off brokers with fuller disks. Continuous Data Balancing enables self-healing clusters that dynamically balance partitions. It also ensures adherence to rack-aware replica placement policy and self-heals after rack (or availability zone) failure or replacement. This balancer does not keep the relative fullness of each broker within a defined range, it just prevents hitting the fullness threshold of each individual broker. - + - + -Continuous Data Balancing requires an xref:get-started:licenses.adoc#redpanda-enterprise-edition[Enterprise license]. +| +This balancer monitors broker and rack availability, as well as disk usage, to avoid topic hotspots when moving data off brokers with fuller disks. Continuous Data Balancing enables self-healing clusters that dynamically balance partitions. It also ensures adherence to rack-aware replica placement policy and self-heals after rack (or availability zone) failure or replacement. This balancer does not keep the relative fullness of each broker within a defined range, it just prevents hitting the fullness threshold of each individual broker. ++ +Continuous Data Balancing requires an xref:get-started:licenses.adoc#redpanda-enterprise-edition[enterprise license]. |=== @@ -108,7 +109,7 @@ This is the default setting. | `continuous` | Redpanda continuously monitors the cluster for broker failures and high disk usage and automatically redistributes partitions to maintain optimal performance and availability. It also monitors rack availability after failures, and for a given partition, it tries to move excess replicas from racks that have more than one replica to racks where there are none. See xref:./continuous-data-balancing.adoc[Configure Continuous Data Balancing]. + + -This requires an Enterprise license. +This requires an enterprise license. | `off` | All partition balancing from Redpanda is turned off. + @@ -118,22 +119,9 @@ This mode is not recommended for production clusters. Only set to `off` if you n == Intra-broker partition balancing -[NOTE] -==== -This is a beta feature for v24.2 and is not recommended for use for production clusters. The feature is disabled by default. To unlock this feature, run this command with a superuser: - -```bash -curl -X PUT -d '{"state": "active"}' http://127.0.0.1:9644/v1/features/node_local_core_assignment -``` - -xref:24.3@ROOT:upgrade:index.adoc[Upgrade to 24.3 or later] to use this feature for production clusters. -==== - In Redpanda, every partition replica is assigned to a CPU core on a broker. While Redpanda's default <> monitors cluster-level events, such as the addition of new brokers or broker failure to balance partition assignments, it does not account for the distribution of partitions _within_ an individual broker. -Starting in this version of Redpanda, topic-aware intra-broker partition balancing allows for dynamically reassigning partitions within a broker. Redpanda prioritizes an even distribution of a topic's partition replicas across all cores in a broker. If a broker's core count changes, when the broker starts back up, Redpanda can check partition assignments across the broker's cores and reassign partitions, so that a balanced assignment is maintained across all cores. Redpanda can also check partition assignments when partitions are added to or removed from a broker, and rebalance the remaining partitions between cores. - -NOTE: Decreasing the number of CPU cores in a running cluster requires that you enable the `node_local_core_assignment` feature flag. +Topic-aware intra-broker partition balancing allows for dynamically reassigning partitions within a broker. Redpanda prioritizes an even distribution of a topic's partition replicas across all cores in a broker. If a broker's core count changes, when the broker starts back up, Redpanda can check partition assignments across the broker's cores and reassign partitions, so that a balanced assignment is maintained across all cores. Redpanda can also check partition assignments when partitions are added to or removed from a broker, and rebalance the remaining partitions between cores. To determine when to use intra-broker partition balancing, use the public metrics for CPU usage described in the xref:manage:monitoring.adoc#cpu-usage[Monitoring] guide. @@ -148,7 +136,7 @@ Configure the following properties to trigger intra-broker partition balancing: | xref:reference:cluster-properties.adoc#core_balancing_continuous[`core_balancing_continuous`] a| Set to `true` to rebalance partition assignments across cores in runtime, for example when partitions are moved to or away from brokers. Default value: `false`. -This requires an Enterprise license. +This requires an enterprise license. |=== You can also manually trigger intra-broker partition balancing with the Admin API: diff --git a/modules/manage/pages/cluster-maintenance/cluster-diagnostics.adoc b/modules/manage/pages/cluster-maintenance/cluster-diagnostics.adoc deleted file mode 100644 index 45b76d81c9..0000000000 --- a/modules/manage/pages/cluster-maintenance/cluster-diagnostics.adoc +++ /dev/null @@ -1,106 +0,0 @@ -= Cluster Diagnostics -:description: Use tools and tests to help diagnose and debug a Redpanda cluster. -:page-categories: Management, Troubleshooting - -This topic provides guides for using tools and tests to help diagnose and debug a Redpanda cluster. - -[[self-test]] -== Disk and network self-test benchmarks - -When anomalous behavior arises in a cluster and you're trying to figure out whether it's caused by faulty hardware (disks, NICs) of a cluster's machines, run xref:reference:rpk/rpk-cluster/rpk-cluster-self-test.adoc[rpk cluster self-test] (self-test) to characterize their performance and compare it with their expected, vendor-specified performance. - -Self-test runs a set of benchmarks to determine the maximum performance of a machine's disks and network connections. For disks, it runs throughput and latency tests by performing concurrent sequential operations. For networks, it selects unique pairs of Redpanda nodes as client/server pairs, then it runs throughput tests between them. Self-test runs each benchmark for a configurable duration, and it returns IOPS, throughput, and latency metrics. - -== Cloud storage tests - -If you use xref:manage:tiered-storage.adoc[Tiered Storage], run self-test to verify that you have configured your cloud storage accounts correctly. - -Self-test performs the following tests to validate cloud storage configuration: - -include::reference:partial$rpk-self-test-cloud-tests.adoc[] - -See the xref:reference:rpk/rpk-cluster/rpk-cluster-self-test-start.adoc[`rpk cluster self-test start`] reference for cloud storage test details. - -== Self-test command examples - -=== Start self-test - -To begin using self-test, run the `self-test start` command. - ----- -rpk cluster self-test start ----- - -For command help, run `rpk cluster self-test start -h`. For additional command flags, see the xref:reference:rpk/rpk-cluster/rpk-cluster-self-test-start.adoc[rpk cluster self-test start] reference. - -Before it starts, `self-test start` asks for your confirmation to run its potentially large workload. - -Example start output: - -[.no-copy] ----- -? Redpanda self-test will run benchmarks of disk and network hardware that will consume significant system resources. Do not start self-test if large workloads are already running on the system. (Y/n) -Redpanda self-test has started, test identifier: "031be460-246b-46af-98f2-5fc16f03aed3", To check the status run: -rpk cluster self-test status ----- - -The `self-test start` command returns immediately, and self-test runs its benchmarks asynchronously. - -=== Check self-test status - -To check on the status of self-test, run the `self-test status` command. - -[,bash] ----- -rpk cluster self-test status ----- - -For command help, run `rpk cluster self-test status -h`. For additional command flags, see the xref:reference:rpk/rpk-cluster/rpk-cluster-self-test-status.adoc[rpk cluster self-test status] reference. - -If benchmarks are currently running, `self-test status` returns a test-in-progress message. - -Example status output: - -[.no-copy] ----- -$ rpk cluster self-test status -Nodes [0 1 2] are still running jobs ----- - -[TIP] -==== -To automate checking the status of self-test, the `status` command can output its results in JSON format by using the `--format=json` option: - -[,bash] ----- -rpk cluster self-test status --format=json ----- - -==== - -If benchmarks have completed, `self-test status` returns their results. - -include::reference:partial$rpk-self-test-descriptions.adoc[] - -.Example status output: test results -include::reference:partial$rpk-self-test-status-output.adoc[] - -=== Stop self-test - -To stop a running self-test, run the `self-test stop` command. - ----- -rpk cluster self-test stop ----- - -Example stop output: - -[.no-copy] ----- -$ rpk cluster self-test stop -All self-test jobs have been stopped ----- - -For command help, run `rpk cluster self-test stop -h`. For additional command flags, see the xref:reference:rpk/rpk-cluster/rpk-cluster-self-test-stop.adoc[rpk cluster self-test stop] reference. - -For more details about self-test, including command flags, see xref:reference:rpk/rpk-cluster/rpk-cluster-self-test.adoc[rpk cluster self-test]. diff --git a/modules/manage/pages/cluster-maintenance/compaction-settings.adoc b/modules/manage/pages/cluster-maintenance/compaction-settings.adoc index f537c806a4..0dfb0bc8ff 100644 --- a/modules/manage/pages/cluster-maintenance/compaction-settings.adoc +++ b/modules/manage/pages/cluster-maintenance/compaction-settings.adoc @@ -6,15 +6,15 @@ Configure compaction for your cluster to optimize storage utilization. == Redpanda compaction overview -Compaction is an optional mechanism intended to reduce the storage needs of Redpanda topics. You can enable compaction through configuration of a cluster or topic's cleanup policy. When compaction is enabled as part of the cleanup policy, a background process executes on a pre-set interval to perform compaction operations. When triggered for a partition, the process purges older versions of messages for a given key and only retains the most recent message in that partition. This is done by analyzing closed segments in the partition, copying the most recent messages for each key into a new segment, then deleting the source segments. +Compaction is an optional mechanism intended to reduce the storage needs of Redpanda topics. You can enable compaction through configuration of a cluster or topic's cleanup policy. When compaction is enabled as part of the cleanup policy, a background process executes on a pre-set interval to perform compaction operations. When triggered for a partition, the process purges older versions of records for a given key and only retains the most recent record in that partition. This is done by analyzing closed segments in the partition, copying the most recent records for each key into a new segment, then deleting the source segments. image::shared:compaction-example.png[Example of topic compaction] -This diagram provides an illustration of a compacted topic. Imagine a remote sensor network that uses image recognition to track appearances of red pandas in a geographic area. The sensor network employs special devices that send messages to a topic when they detect one. You might enable compaction to reduce topic storage while still maintaining a record in the topic of the last time each device saw a red panda, perhaps to see if they stop frequenting a given area. The left side of the diagram shows all messages sent across the topic. The right side illustrates the results of compaction; older messages for certain keys are deleted from the message log. +This diagram illustrates a compacted topic. Imagine a remote sensor network that uses image recognition to track appearances of red pandas in a geographic area. The sensor network employs special devices that send records to a topic when they detect one. You might enable compaction to reduce topic storage while still maintaining a record in the topic of the last time each device saw a red panda, perhaps to see if they stop frequenting a given area. The left side of the diagram shows all records sent across the topic. The right side illustrates the results of compaction; older records for certain keys are deleted from the log. -NOTE: If your application requires consuming every message for a given key, consider using the `delete` xref:develop:config-topics#change-the-cleanup-policy.adoc[cleanup policy] instead. +NOTE: If your application requires consuming every record for a given key, consider using the `delete` xref:develop:config-topics#change-the-cleanup-policy.adoc[cleanup policy] instead. -IMPORTANT: When using Tiered Storage, compaction functions at the local storage level. As long as a segment remains in local storage, its messages are eligible for compaction. Once a segment is uploaded to tiered storage and removed from local storage it is not retrieved for further compaction operations. A key may therefore appear in multiple segments between Tiered Storage and local storage. +IMPORTANT: When using xref:manage:tiered-storage.adoc[Tiered Storage], compaction functions at the local storage level. As long as a segment remains in local storage, its records are eligible for compaction. Once a segment is uploaded to object storage and removed from local storage it is not retrieved for further compaction operations. A key may therefore appear in multiple segments between Tiered Storage and local storage. While compaction reduces storage needs, Redpanda's compaction (just like Kafka's) does not guarantee perfect de-duplication of a topic. It represents a best effort mechanism to reduce storage needs but duplicates of a key may still exist within a topic. Compaction is not a complete topic operation, either, since it operates on subsets of each partition within the topic. @@ -22,9 +22,61 @@ While compaction reduces storage needs, Redpanda's compaction (just like Kafka's Compaction policy may be applied to a cluster or to an individual topic. If both are set, the topic-level policy overrides the cluster-level policy. The cluster-level xref:reference:cluster-properties.adoc#log_cleanup_policy[`log_cleanup_policy`] and the topic-level xref:reference:topic-properties.adoc#cleanuppolicy[`cleanup.policy`] support the following three options: -* `delete`: Messages are deleted from the topic once the specified retention period (time and/or size allocations) is exceeded. This is the default mechanism and is analogous to disabling compaction. -* `compact`: This triggers only cleanup of messages with multiple versions. A message that represents the only version for a given key is not deleted. -* `compact,delete`: This combines both policies, deleting messages exceeding the retention period while compacting multiple versions of messages. +* `delete`: Records are deleted from the topic once the specified retention period (time and/or size allocations) is exceeded. This is the default mechanism and is analogous to disabling compaction. +* `compact`: This triggers only cleanup of records with multiple versions. A record that represents the only version for a given key is not deleted. +* `compact,delete`: This combines both policies, deleting records exceeding the retention period while compacting multiple versions of records. + +== Tombstone record removal + +Compaction also enables deletion of existing records through tombstones. For example, as data is deleted from a source system, clients produce a tombstone record to the log. A tombstone contains a key and the value `null`. Tombstones signal to brokers and consumers that records with the same key prior to it in the log should be deleted. + +You can specify how long Redpanda keeps these tombstones for compacted topics using both a cluster configuration property config_ref:tombstone_retention_ms,true,properties/cluster-properties[] and a topic configuration property xref:reference:properties/topic-properties.adoc#deleteretentionms[`delete.retention.ms`]. If both are set, the topic-level tombstone retention policy overrides the cluster-level policy. + +[NOTE] +==== +Redpanda does not currently remove tombstone records for compacted topics with Tiered Storage enabled. + +You cannot enable `tombstone_retention_ms` if you have enabled any of the Tiered Storage cluster properties `cloud_storage_enabled`, `cloud_storage_enable_remote_read`, and `cloud_storage_enable_remote_write`. + +On the topic level, you cannot enable `delete.retention.ms` at the same time as the Tiered Storage topic configuration properties `redpanda.remote.read` and `redpanda.remote.write`. +==== + +To set the cluster-level tombstone retention policy, run the command: + +[,bash] +---- +rpk cluster config set tombstone_retention_ms=100 +---- + +You can unset the tombstone retention policy for a topic so it inherits the cluster-wide default policy: + +[,bash] +---- +rpk topic alter-config --delete delete.retention.ms +---- + +To override the cluster-wide default for a specific topic: + +[,bash] +---- +rpk topic alter-config --set delete.retention.ms=5 +---- + +To disable tombstone removal for a specific topic: + +[,bash] +---- +rpk topic alter-config --set delete.retention.ms=-1 +---- + +Redpanda removes tombstones as follows: + +* For topics with a `compact` only cleanup policy: Tombstones are removed when the topic exceeds the tombstone retention limit. The `delete.retention.ms` or `tombstone_retention_ms` values therefore also set the time bound that a consumer has in order to see a complete view of the log with tombstones present before they are removed. +* For topics with a `compact,delete` cleanup policy: Both the tombstone retention policy and standard garbage collection can remove tombstone records. + +If obtaining a complete snapshot of the log, including tombstone records, is important to your consumers, set the tombstone retention value such that consumers have enough time for their reads to complete before tombstones are removed. Consumers may not see tombstones if their reads take longer than `delete.retention.ms` and `tombstone_retention_ms`. The trade-offs to ensuring tombstone visibility to consumers are increased disk usage and potentially slower compaction. + +On the other hand, if more frequent cleanup of tombstones is important for optimizing workloads and space management, consider setting a shorter tombstone retention, for example the typical default of 24 hours (86400000 ms). == Compaction policy settings diff --git a/modules/manage/pages/cluster-maintenance/nodewise-partition-recovery.adoc b/modules/manage/pages/cluster-maintenance/nodewise-partition-recovery.adoc index b41ef526c8..45aaa593d2 100644 --- a/modules/manage/pages/cluster-maintenance/nodewise-partition-recovery.adoc +++ b/modules/manage/pages/cluster-maintenance/nodewise-partition-recovery.adoc @@ -1,28 +1,39 @@ = Node-wise Partition Recovery :description: Feature to recover partitions that have lost a majority of replicas. -Multi-node or entire-AZ failures (especially in cloud environments), and some forms of human error may result in ‘stuck’ partitions with fewer replicas than required to make a quorum. In such failure scenarios, some data loss may be unavoidable. This description of node-wise partition recovery helps admins understand what they can or cannot recover. +Multi-broker or entire glossterm:availability zones[,AZ] failures (especially in cloud environments), along with some forms of human error, can result in ‘stuck’ partitions where there are fewer replicas than required to make a quorum. In such failure scenarios, some data loss may be unavoidable. Node-wise partition recovery provides a way to unsafely recover at least a portion of your data using remaining replicas, which are moved off of target brokers and allocated to healthy ones. In one step, this process repairs partitions while draining the target brokers of all partition replicas. This topic helps admins understand what they can or cannot recover using node-wise partition recovery. -IMPORTANT: This is a last-ditch measure when all other recovery options have failed. In some cases, no remaining replicas may exist for the partitions on the dead nodes. This recovery method is intended for scenarios where you are already experiencing data loss - the goal is to stop the loss of additional data. - -Node-wise partition recovery allows you to unsafely recover at least a portion of your data using whatever replicas are remaining. All replicas are moved off the target nodes and allocated to healthy nodes. In one step, this process repairs partitions while draining the target nodes. +IMPORTANT: Only use this operation as a last-resort measure when all other recovery options have failed. In some cases, there may be no remaining replicas for the partitions on the dead brokers. This recovery method is intended for scenarios where you have already experienced data loss, with the goal being to stop the loss of additional data. == Perform the recovery operation -You perform node-wise partition recovery using the `rpk` command `rpk cluster partitions unsafe-recover`. This command includes an interactive prompt to confirm execution of the generated recovery plan as it is a destructive operation. When you trigger node-wise partition recovery, the partitions on the node are rebuilt in a best-effort basis. As a result of executing this operation, you may lose some data that has not yet been replicated to the surviving partition replicas. +To start node-wise partition recovery, run `rpk cluster partitions unsafe-recover`. For example: + +`rpk cluster partitions unsafe-recover --from-nodes 1,3,5` + +This command includes a prompt to confirm the generated recovery plan, as it is a destructive operation. When you run node-wise partition recovery, the partitions on the broker are rebuilt on a best-effort basis. When there are zero surviving partition replicas, such as a topic with a replication factor of 1 (`RF=1`), partition recovery rebuilds empty partitions with no data (although you may be able to recover the partition from Tiered Storage), allowing producers to continue writing to the partition even though no data can be recovered in such situations. + + +The `--from-nodes` flag accepts a comma-separated list of the brokers' node IDs you wish to recover the data from. This example performs recovery operations on nodes 1, 3, and 5. Redpanda assesses these brokers to identify which partitions lack a majority. It then creates a plan to recover the impacted partitions and prompts you for confirmation. You must respond `yes` to continue with recovery. + +The `--dry` flag performs a dry run and allows you to view the recovery plan with no risk to your cluster. -The syntax for this command is as follows: +[NOTE] +==== +When running node-wise partition recovery, it's possible that there may be more recent data (a higher offset) available in Tiered Storage if: - rpk cluster partitions unsafe-recover --from-nodes 1,3,5 +* Raft replication was stuck or slow before the node failure +* Zero live replicas remain in the cluster (because the partition had a replication factor of one, `RF=1`) -The `--from-nodes` parameter accepts a comma-delineated list of dead node IDs you wish to recover the data from. The above example would perform recovery operations on nodes 1, 3, and 5. Redpanda will assess these nodes to identify which partitions lack majority. It will then create a plan to recover the impacted partitions and prompt you to confirm it. You must respond `yes` to continue with recovery. +For topics configured to use Tiered Storage, Redpanda also attempts to recover partition data from object storage, recovering the latest offset available for a partition in either storage tier (local or object storage). This allows for the maximum amount of data to be recovered in all cases, even for topics with a replication factor of 1, where no replicas remain in local storage. +==== -You may also optionally add the `--dry` parameter to this command. This will perform a dry run and allow viewing the recovery plan with no risk to your cluster. +The recovery operation can take some time to complete, especially for a large amount of data. To monitor the status of the recovery operation in real-time, run: -Once the recovery operation is started, you may monitor the status of its execution using the `rpk cluster partitions balancer-status` command. The recovery operation can take some time to complete, especially when a lot of data is involved. This command allows you to monitor progress in real-time. +`rpk cluster partitions balancer-status` -== Example recovery operation -Here's an example of the recovery process in action. +== Example recovery operations +The following example shows the node-wise partition recovery process in action: ---- $ rpk cluster partitions unsafe-recover --from-nodes 1 @@ -37,4 +48,21 @@ Status: ready Seconds Since Last Tick: 26 Current Reassignment Count: 0 Partitions Pending Recovery (1): [kafka/bar/0] ----- \ No newline at end of file +---- + +The following example shows the status of moved partitions: + +---- +$ rpk cluster partitions move-status +PARTITION MOVEMENTS +=================== +NAMESPACE-TOPIC PARTITION MOVING-FROM MOVING-TO COMPLETION-% PARTITION-SIZE BYTES-MOVED BYTES-REMAINING +kafka/prod_tests 4 [045] [045] 0 56204032205 0 56204032205 +kafka/prod_tests 7 [045] [045] 0 64607340009 0 64607340009 +kafka/prod_tests 12 [014] [014] 0 29074311639 0 29074311639 +kafka/prod_tests 20 [014] [014] 0 29673620476 0 29673620476 +kafka/prod_tests 22 [045] [045] 0 28471089141 0 28471089141 +kafka/prod_tests 23 [045] [045] 0 29692435312 0 29692435312 +kafka/prod_tests 31 [014] [014] 0 66982232299 0 66982232299 +kafka/prod_tests 33 [014] [014] 0 46329276747 0 46329276747 +---- diff --git a/modules/manage/pages/kubernetes/k-manage-resources.adoc b/modules/manage/pages/kubernetes/k-manage-resources.adoc index c1531cf74c..cf1505a173 100644 --- a/modules/manage/pages/kubernetes/k-manage-resources.adoc +++ b/modules/manage/pages/kubernetes/k-manage-resources.adoc @@ -24,12 +24,6 @@ kubectl describe nodes - <>. This configuration prevents the operating system from paging out Redpanda's memory to disk, which can significantly impact performance. -== Limitations - -Decreasing the number of CPU cores in a production cluster is not supported in this version of Redpanda. However, you can enable xref:manage:cluster-maintenance/cluster-balancing.adoc#intra-broker-partition-balancing[intra-broker partition balancing] to reduce CPU cores in a development cluster. - -Starting from version 24.3, decreasing the number of CPU cores in a production cluster is supported. xref:24.3@ROOT:upgrade:index.adoc[Upgrade to version 24.3] or later to access this feature. - [[memory]] == Configure memory resources diff --git a/modules/manage/pages/kubernetes/k-manage-topics.adoc b/modules/manage/pages/kubernetes/k-manage-topics.adoc index 9313bfb4ca..4804ebea2f 100644 --- a/modules/manage/pages/kubernetes/k-manage-topics.adoc +++ b/modules/manage/pages/kubernetes/k-manage-topics.adoc @@ -126,6 +126,53 @@ The cleanup policy determines how to manage the partition log files when they re If the cleanup policy is `delete` or `compact,delete` you can <>. +=== Configure Iceberg topics + +Iceberg topics allow you to create a seamless integration with Apache Iceberg by writing records in Redpanda topics to object storage in Iceberg format. + +[NOTE] +==== +include::shared:partial$enterprise-license.adoc[] +==== + +In addition to the general prerequisites for managing topics, you must have the following: + +- xref:manage:topic-iceberg-integration.adoc[Iceberg support] must be enabled on your Redpanda cluster. +- xref:manage:kubernetes/tiered-storage/k-tiered-storage.adoc[Tiered Storage] must be enabled on your Redpanda cluster. + +To create an Iceberg topic, set the `redpanda.iceberg.mode` configuration in the `additionalConfig` property of the `Topic` resource. + +[source,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Topic +metadata: + name: iceberg-key-value +spec: + additionalConfig: + redpanda.iceberg.mode: "" <.> + kafkaApiSpec: + brokers: + - "redpanda-0.redpanda..svc.cluster.local:9093" + tls: + caCertSecretRef: + name: "redpanda-default-cert" + key: "ca.crt" +---- + +<.> Supported modes: + +- `key_value`: Data is written with explicit keys and values. +- `value_schema_id_prefix`: Data is serialized using schemas managed by the Schema Registry, ensuring compatibility with schema-based systems. +- `disabled` (default) + +Apply the resource to your Kubernetes cluster: + +[source,bash] +---- +kubectl apply -f iceberg-topic.yaml --namespace +---- + === Configure write caching Write caching is a relaxed mode of xref:develop:produce-data/configure-producers.adoc#acksall[`acks=all`] that provides better performance at the expense of durability. It acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to be written to disk. This provides lower latency while still ensuring that a majority of brokers acknowledge the write. diff --git a/modules/manage/pages/kubernetes/k-schema-controller.adoc b/modules/manage/pages/kubernetes/k-schema-controller.adoc new file mode 100644 index 0000000000..aaaa8606af --- /dev/null +++ b/modules/manage/pages/kubernetes/k-schema-controller.adoc @@ -0,0 +1,272 @@ += Manage Schemas with the Redpanda Operator +:description: Use the Schema resource to declaratively create and manage schemas as part of a Redpanda deployment in Kubernetes. +:page-categories: Management, Development +:env-kubernetes: true + +Use the Schema resource to declaratively create and manage schemas as part of a Redpanda deployment in Kubernetes. Each Schema resource maps to a schema in your Redpanda cluster, allowing you to define data structures, compatibility, and schema evolution in a declarative way. + +== Prerequisites + +Ensure you have the following: + +* *Kubectl*: Ensure the https://kubernetes.io/docs/tasks/tools/#kubectl[kubectl^] command-line tool is installed and configured to communicate with your cluster. +* *Redpanda cluster*: Ensure you have at least version v2.3.0-24.3.1 of the xref:deploy:deployment-option/self-hosted/kubernetes/k-production-deployment.adoc[Redpanda Operator] and a Redpanda resource deployed and accessible. + +== Create a schema + +. Define a schema using the Schema resource. Here's a basic example configuration that defines an Avro schema: ++ +.`schema.yaml` +[source,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Schema +metadata: + name: example-schema + namespace: +spec: + cluster: + clusterRef: + name: + schemaType: avro + compatibilityLevel: Backward + text: | + { + "type": "record", + "name": "ExampleRecord", + "fields": [ + { "type": "string", "name": "field1" }, + { "type": "int", "name": "field2" } + ] + } +---- ++ +Replace the following placeholders: ++ +- ``: The namespace in which to deploy the Schema resource. The Schema resource must be deployed in the same namespace as the Redpanda resource defined in `clusterRef.name`. +- ``: The name of the Redpanda resource that defines the Redpanda cluster to which you want to upload the schema. + +. Apply the manifest: ++ +[source,bash] +---- +kubectl apply -f schema.yaml --namespace +---- ++ +When the manifest is applied, the schema will be created in your Redpanda cluster. + +. Check the status of the Schema resource using the following command: ++ +[,bash] +---- +kubect get schema example-schema --namespace +---- + +. Verify that the schema was created in Redpanda: ++ +[source,bash] +---- +kubectl exec -it --namespace -- curl https://...svc.cluster.local:8081/subjects -sS --cacert /etc/tls/certs/default/ca.crt -w '\n' +---- ++ +Replace `` with the name of a Pod that's running Redpanda. + +== Schema examples + +These examples demonstrate how to define schemas in Avro, Protobuf, and JSON Schema formats. + +=== Create an Avro schema + +.`avro-schema.yaml` +[,yaml,indent=0] +---- +include::manage:example$kubernetes/schema-crds.feature[tags=customer-profile-avro-schema-manifest,indent=0] +---- + +=== Create a Protobuf schema + +.`proto-schema.yaml` +[,yaml,indent=0] +---- +include::manage:example$kubernetes/schema-crds.feature[tags=product-catalog-protobuf-schema-manifest,indent=0] +---- + +=== Create a JSON schema + +.`json-schema.yaml` +[,yaml,indent=0] +---- +include::manage:example$kubernetes/schema-crds.feature[tags=order-event-json-schema-manifest,indent=0] +---- + +== Configuration + +The Schema resource in Redpanda offers various options to customize and control schema behavior. This section covers schema compatibility, schema references, and schema types, providing a detailed guide on using each of these features to maintain data integrity, manage dependencies, and facilitate schema evolution. + +You can find all configuration options for the Schema resource in the xref:reference:k-crd.adoc#k8s-api-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-schema[CRD reference]. + +.`schema.yaml` +[source,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Schema +metadata: + name: <.> + namespace: <.> +spec: + cluster: + clusterRef: + name: <.> + schemaType: avro <.> + compatibilityLevel: Backward <.> + references: [] <.> + text: | <.> + { + "type": "record", + "name": "test", + "fields": [ + { "type": "string", "name": "field1" }, + { "type": "int", "name": "field2" } + ] + } +---- + +<.> *Subject name*: The name of the subject for the schema. When data formats are updated, a new version of the schema can be registered under the same subject, enabling backward and forward compatibility. +<.> *Namespace*: The namespace in which to deploy the Schema resource. The Schema resource must be deployed in the same namespace as the Redpanda resource defined in `clusterRef.name`. +<.> *Cluster name*: The name of the Redpanda resource that defines the Redpanda cluster to which you want to upload the schema. +<.> *Compatibility level*: Defines the compatibility level for the schema. Options are `Backward` (default), `BackwardTransitive`, `Forward`, `ForwardTransitive` `Full`, `FullTransitive`, or `None`. See <>. +<.> *Schema type*: Specifies the type of the schema. Options are `avro` (default) or `protobuf`. For JSON Schema, include `"$schema":` in the `text` to indicate the JSON Schema draft version. See <>. +<.> *References*: Any references you want to add to other schemas. If no references are needed, this can be an empty list (default). See <>. +<.> *Schema body*: The body of the schema, which defines the data structure. + +=== Choose a schema type + +Redpanda's Schema Registry supports the following schema types: + +* *Avro*: A widely used serialization format in event-driven architectures. +* *Protobuf*: Popular for defining data structures in gRPC APIs and efficient data serialization. +* *JSON Schema*: Dynamic, schema-based validation for JSON documents. + +If no type is specified, Redpanda defaults to Avro. + +=== Choose a compatibility mode + +Compatibility modes determine how schema versions within a subject can evolve without breaking existing data consumers. Redpanda supports the following compatibility levels: + +* `None`: Disables compatibility checks, allowing any schema change. +* `Backward`: Consumers using the new schema (for example, version 10) can read data from producers using the previous schema (for example, version 9). +* `BackwardTransitive`: Enforces backward compatibility across all versions, not just the latest. +* `Forward`: Consumers using the previous schema (for example, version 9) can read data from producers using the new schema (for example, version 10). +* `ForwardTransitive`: Ensures forward compatibility across all schema versions. +* `Full`: Combines backward and forward compatibility, requiring that changes maintain compatibility in both directions. A new schema and the previous schema (for example, versions 10 and 9) are both backward and forward-compatible with each other. +* `FullTransitive`: Enforces full compatibility across all schema versions. + +For example, to set full compatibility, configure the Schema resource with: + +[source,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Schema +metadata: + name: fully-compatible-schema + namespace: redpanda +spec: + cluster: + clusterRef: + name: basic + schemaType: avro + compatibilityLevel: Full + text: | + { + "type": "record", + "name": "ExampleRecord", + "fields": [ + { "type": "string", "name": "field1" }, + { "type": "int", "name": "field2" } + ] + } +---- + +Compatibility settings are essential for maintaining data consistency, especially when updating schemas over time. + +=== Use schema references + +For complex data structures, you can define schema references that allow one schema to reference another, enabling modular and reusable schema components. Schema references are helpful for shared data structures across topics like product information or user profiles, reducing redundancy. + +NOTE: This feature is supported for Avro and Protobuf schemas. + +Define a schema reference using the `references` field. The reference includes the name, subject, and version of the referenced schema: + +[source,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Schema +metadata: + name: order-schema + namespace: redpanda +spec: + cluster: + clusterRef: + name: basic + references: + - name: product-schema + subject: product + version: 1 + text: | + { + "type": "record", + "name": "Order", + "fields": [ + { "name": "product", "type": "Product" } + ] + } +---- + +== Update a schema + +To update a schema, modify the Schema resource and apply the changes: + +[source,bash] +---- +kubectl apply -f .yaml --namespace +---- + +== Check schema version + +Ensure the schema has been versioned by running: + +[source,bash] +---- +kubectl get schema --namespace +---- + +You can also check specific versions of the schema: + +[source,bash] +---- +kubectl exec -it --namespace -- curl https://...svc.cluster.local:8081/schemas/ids/1 -sS --cacert /etc/tls/certs/default/ca.crt -w '\n' +kubectl exec -it --namespace -- curl https://...svc.cluster.local:8081/schemas/ids/2 -sS --cacert /etc/tls/certs/default/ca.crt -w '\n' +---- + +== Delete a schema + +To delete a schema, use the following command: + +[source,bash] +---- +kubectl delete schema --namespace redpanda +---- + +Verify that the schema was deleted by checking the Redpanda Schema Registry: + +[source,bash] +---- +kubectl exec -it --namespace -- curl https://...svc.cluster.local:8081/subjects -sS --cacert /etc/tls/certs/default/ca.crt -w '\n' +---- + +== Suggested reading + +For more details on using schemas in Redpanda, see: + +* xref:manage:schema-reg/index.adoc[] + diff --git a/modules/manage/pages/kubernetes/security/authentication/k-authentication.adoc b/modules/manage/pages/kubernetes/security/authentication/k-authentication.adoc index 56df29cc28..1891906b50 100644 --- a/modules/manage/pages/kubernetes/security/authentication/k-authentication.adoc +++ b/modules/manage/pages/kubernetes/security/authentication/k-authentication.adoc @@ -12,7 +12,7 @@ include::manage:partial$authentication.adoc[] This section lists error messages and provides ways to diagnose and solve issues. For more troubleshooting steps, see xref:manage:kubernetes/troubleshooting/k-troubleshoot.adoc[Troubleshoot Redpanda in Kubernetes]. -include::manage:kubernetes/troubleshooting/k-troubleshoot.adoc[tags=sasl] +include::troubleshoot:partial$errors-and-solutions.adoc[tags=sasl] == Next steps diff --git a/modules/manage/pages/kubernetes/security/authentication/k-user-controller.adoc b/modules/manage/pages/kubernetes/security/authentication/k-user-controller.adoc index 092a6d7009..5c46594f18 100644 --- a/modules/manage/pages/kubernetes/security/authentication/k-user-controller.adoc +++ b/modules/manage/pages/kubernetes/security/authentication/k-user-controller.adoc @@ -91,9 +91,9 @@ spec: operations: [Read,Write] ---- -== Configuration best practices +== Configuration -The following sections provide guidance on setting up user authentication, managing secrets, and defining ACLs within your Kubernetes environment. These recommendations ensure proper user management while minimizing manual interventions and preventing potential security issues. By following these best practices, you can ensure that user access and permissions are correctly configured and maintained across your Redpanda cluster. +The following sections provide guidance on setting up user authentication, managing secrets, and defining ACLs within your Kubernetes environment. These recommendations ensure proper user management while minimizing manual interventions and preventing potential security issues. You can find all configuration options for the User resource in the xref:reference:k-crd.adoc#k8s-api-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-user[CRD reference]. diff --git a/modules/manage/pages/kubernetes/security/tls/k-cert-manager.adoc b/modules/manage/pages/kubernetes/security/tls/k-cert-manager.adoc index d76e1601b8..9bdfaf44bd 100644 --- a/modules/manage/pages/kubernetes/security/tls/k-cert-manager.adoc +++ b/modules/manage/pages/kubernetes/security/tls/k-cert-manager.adoc @@ -426,7 +426,7 @@ include::manage:partial$kubernetes/disable-tls.adoc[leveloffset=+1] Here are some common troubleshooting scenarios and their solutions. For more troubleshooting steps, see xref:manage:kubernetes/troubleshooting/k-troubleshoot.adoc[Troubleshoot Redpanda in Kubernetes]. -include::manage:kubernetes/troubleshooting/k-troubleshoot.adoc[tags=tls;networking] +include::troubleshoot:partial$errors-and-solutions.adoc[tags=tls;networking] == Next steps diff --git a/modules/manage/pages/kubernetes/security/tls/k-secrets.adoc b/modules/manage/pages/kubernetes/security/tls/k-secrets.adoc index ca16c3a999..d99ab3fc31 100644 --- a/modules/manage/pages/kubernetes/security/tls/k-secrets.adoc +++ b/modules/manage/pages/kubernetes/security/tls/k-secrets.adoc @@ -311,7 +311,7 @@ include::manage:partial$kubernetes/disable-tls.adoc[leveloffset=+1] Here are some common troubleshooting scenarios and their solutions. For more troubleshooting steps, see xref:manage:kubernetes/troubleshooting/k-troubleshoot.adoc[Troubleshoot Redpanda in Kubernetes]. -include::manage:kubernetes/troubleshooting/k-troubleshoot.adoc[tags=tls;networking] +include::troubleshoot:partial$errors-and-solutions.adoc[tags=tls;networking] == Next steps diff --git a/modules/manage/pages/kubernetes/tiered-storage/k-tiered-storage.adoc b/modules/manage/pages/kubernetes/tiered-storage/k-tiered-storage.adoc index f4f6eba1b1..d21e4a8d6e 100644 --- a/modules/manage/pages/kubernetes/tiered-storage/k-tiered-storage.adoc +++ b/modules/manage/pages/kubernetes/tiered-storage/k-tiered-storage.adoc @@ -1,10 +1,9 @@ = Use Tiered Storage in Kubernetes :description: Configure your Redpanda cluster to offload log segments to object storage and save storage costs. -:page-context-links: [{"name": "Linux", "to": "manage:tiered-storage.adoc" },{"name": "Kubernetes", "to": "manage:kubernetes/storage/tiered-storage/k-tiered-storage.adoc" } ] :page-categories: Management, High Availability, Data Replication :env-kubernetes: true -:tags: ["Kubernetes", "Helm configuration"] +:page-toclevels: 4 :page-aliases: manage:kubernetes/tiered-storage.adoc, manage:kubernetes/storage/tiered-storage.adoc, manage:kubernetes/data-archiving.adoc, manage:kubernetes/storage/data-archiving.adoc, manage:kubernetes/storage/tiered-storage/k-data-archiving.adoc, manage:kubernetes/storage/tiered-storage/k-tiered-storage.adoc -include::manage:partial$tiered-storage.adoc[] \ No newline at end of file +include::manage:partial$tiered-storage.adoc[] diff --git a/modules/manage/pages/kubernetes/troubleshooting/index.adoc b/modules/manage/pages/kubernetes/troubleshooting/index.adoc deleted file mode 100644 index ff6d3f9c0c..0000000000 --- a/modules/manage/pages/kubernetes/troubleshooting/index.adoc +++ /dev/null @@ -1,5 +0,0 @@ -= Troubleshooting Redpanda in Kubernetes -:description: Learn how to diagnose and troubleshoot problems with Redpanda in Kubernetes. -:page-layout: index -:page-categories: Management, Troubleshooting -:env-kubernetes: true diff --git a/modules/manage/pages/mountable-topics.adoc b/modules/manage/pages/mountable-topics.adoc new file mode 100644 index 0000000000..1d50b35b0b --- /dev/null +++ b/modules/manage/pages/mountable-topics.adoc @@ -0,0 +1,7 @@ += Mountable Topics +:description: Safely attach and detach Tiered Storage topics to and from a Redpanda cluster. +:page-context-links: [{"name": "Linux", "to": "manage:mountable-topics.adoc" } ] +:page-categories: Management +:env-linux: true + +include::manage:partial$mountable-topics.adoc[] \ No newline at end of file diff --git a/modules/manage/pages/schema-reg/manage-schema-reg.adoc b/modules/manage/pages/schema-reg/manage-schema-reg.adoc new file mode 100644 index 0000000000..b0c456ae0e --- /dev/null +++ b/modules/manage/pages/schema-reg/manage-schema-reg.adoc @@ -0,0 +1,5 @@ += Use Schema Registry +:page-layout: index +:description: Learn how to add, delete, and update schemas in Redpanda Self-Managed. + +{description} \ No newline at end of file diff --git a/modules/manage/pages/security/encryption.adoc b/modules/manage/pages/security/encryption.adoc index 7459dc6f6f..d9b26378dd 100644 --- a/modules/manage/pages/security/encryption.adoc +++ b/modules/manage/pages/security/encryption.adoc @@ -10,39 +10,57 @@ By default, Redpanda data is sent unencrypted. A security best practice is to en * Transport Layer Security (TLS), previously SSL, provides encryption for client-server communication. A server certificate prevents third parties from accessing data transferred between the client and server. By default, Redpanda clusters accept connections from clients using TLS version 1.2 or later, but <>. * mTLS, or 2-way TLS, is a protocol that authenticates both the server and the client. In addition to the server certificate required in TLS, mTLS also requires the client to give a certificate. This involves more overhead to implement, but it can be useful for environments that require additional security and only have a small number of verified clients. -For each Redpanda broker, you must specify the: +== Prerequisites -* key file (`broker.key`) -* certificate file (`broker.crt`) -* truststore file (`ca.crt`). +TLS certificates are required for encryption. You can use your own certificates, either self-signed or issued by a trusted CA. -Each broker has its own `broker.key` and `broker.crt`, but all brokers can have the same `ca.crt`. +You'll need the following files: +-- +- A private key file (`broker.key`) for each broker. +- A certificate file (`broker.crt`) for each broker. +- A truststore file (`ca.crt`). All brokers can have the same `ca.crt` file. +-- -If you enable TLS encryption, you can also specify a certificate revocation list (`ca.crl`) so that Redpanda can check and reject connections from entities using certificates already revoked by a certificate authority (CA). All brokers can have the same `ca.crl`. The file must contain a single, concatenated list of certificate revocation lists (CRLs) for all issuing certificates in the truststore file. +Ensure these files are readable by Redpanda and protected against unauthorized access: -== Prerequisites +[,bash] +---- +chmod 400 broker.key broker.crt ca.crt +chown redpanda:redpanda broker.key broker.crt ca.crt +---- -TLS certificates are necessary for encryption. You can use your own certificates, either self-signed or issued by a trusted CA. +For mTLS, client certificates signed by the same CA are also required. -=== Create a local CA for self-signed certificates +If you don't already have these files, you can learn how to generate them in <>. -NOTE: This step is required if you want to generate multiple certificates all signed by -the same root; for example, you want to use mTLS but issue different certificates -to multiple Redpanda brokers and clients. +If you enable TLS encryption, you can also specify a certificate revocation list (`ca.crl`) so that Redpanda can check and reject connections from entities using certificates already revoked by a certificate authority (CA). All brokers can have the same `ca.crl`. The file must contain a single, concatenated list of certificate revocation lists (CRLs) for all issuing certificates in the truststore file. -To generate a self-signed certificate in a single command: +[[gen-certs]] +== Generate certificate files -[,bash] ----- -openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -keyout broker.key -out broker.crt -subj "/CN=redpanda" -addext "subjectAltName = DNS:localhost, IP: 127.0.0.1" ----- +This section shows you how to generate self-signed certificate files for your Redpanda brokers. If you already have your own, you can skip this step. -==== 1. Create CA configuration file (ca.cnf) +NOTE: Self-signed certificates are useful if you want to generate multiple certificates all signed by the same root. For example, you want to use mTLS but issue different certificates to multiple Redpanda brokers and clients. -Edit the `distinguished_name` section with your own organization details. For `default_md`, `sha256` is the minimum message digest level. The `subjectAltName` must be accurate for the broker's certificate. +=== Create a local CA for self-signed certificates -.ca.cnf -[,ini] +. Create a self-signed certificate and private key for the CA: ++ +[source,bash] +---- +openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -keyout broker.key -out broker.crt -subj "/CN=redpanda" -addext "subjectAltName = DNS:localhost, IP:127.0.0.1" +---- ++ +Ensure that `subjectAltName` is accurate for your setup. ++ +This command generates: ++ +- `broker.key`: Private key +- `broker.crt`: Self-signed certificate + +. Create a configuration file for the CA, `ca.cnf`, and include your organization's details. Ensure the `subjectAltName` extension is set correctly for broker certificates. ++ +[source,ini] ---- [ ca ] default_ca = CA_default @@ -55,47 +73,42 @@ default_md = sha256 copy_extensions = copy unique_subject = no policy = signing_policy -[ signing_policy ] -organizationName = supplied -commonName = optional -# Used to create the CA certificate. [ req ] prompt = no distinguished_name = distinguished_name x509_extensions = extensions [ distinguished_name ] -organizationName = Redpanda -commonName = Redpanda CA +organizationName = +commonName = [ extensions ] keyUsage = critical,digitalSignature,nonRepudiation,keyEncipherment,keyCertSign basicConstraints = critical,CA:true,pathlen:1 -# Common policy for nodes and users. [ signing_policy ] organizationName = supplied commonName = optional ---- -Create a CA key to self-sign certificates: - -[,bash] +. Create a private key for the CA and set its permissions: ++ +[source,bash] ---- openssl genrsa -out ca.key 2048 chmod 400 ca.key ---- -Create a public certificate for the CA: - -[,bash] +. Use the CA configuration to generate a public certificate: ++ +[source,bash] ---- openssl req -new -x509 -config ca.cnf -key ca.key -days 365 -batch -out ca.crt ---- - ++ where: - ++ [cols=",",options="header",] |=== |Inputs |Description @@ -113,23 +126,21 @@ certificates (created above). |`-batch` |Batch mode, where certificates are certified automatically. |=== ++ +The output `ca.crt` is the CA's public certificate, which you'll use in the truststore. -[cols=",",options="header",] -|=== -|Output |Description -|`-out ca.crt` |The public key certificate of the CA. -|=== - -=== Create certificate signing requests +=== Create broker certificates and certificate signing requests (CSRs) -This step creates the certificate signing request for the CA to extend trust over the broker's certificates. - -==== 1. Create broker configuration file (broker.cnf) +To issue certificates for brokers, create a certificate signing request (CSR) and sign it with the CA. +. Define the broker's Subject Alternative Name (SAN) entries in `broker.cnf` under `alt_names`. ++ A subject alternative name (SAN) indicates all domain names and IP addresses secured by the certificate. Depending on the address the client uses to connect to Redpanda, you might need to create a CNF file for each broker to modify the `alt_names` section with organizational details. For production usage, edit `alt_names` with DNS resolutions and/or the IP addresses. - -.broker.cnf -[,ini] ++ +For example: ++ +.`broker.cnf` +[source,ini] ---- [ req ] prompt = no @@ -137,7 +148,7 @@ distinguished_name = distinguished_name req_extensions = extensions [ distinguished_name ] -organizationName = Redpanda +organizationName = [ extensions ] subjectAltName = @alt_names @@ -150,23 +161,20 @@ DNS.4 = connect DNS.5 = ec2-3-15-15-272.us-east-2.compute.amazonaws.com IP.1 = 10.0.8.1 ---- - -You could configure alternative names with a single version of `broker.key`/`broker.crt`, -as long as you update the certificate for all brokers in the cluster any time you -edit an entry. For example: - -[,ini] ++ +You could configure alternative names with a single version of `broker.key`/`broker.crt`, as long as you update the certificate for all brokers in the cluster any time you edit an entry. For example: ++ +[source,ini] ---- [ alt_names ] DNS.1 = broker1.example.com DNS.2 = broker2.example.com DNS.3 = broker3.example.com ---- - -Additionally, you can configure alternative names using the public or private -IP addresses of all your brokers. For example: - -[,ini] ++ +Additionally, you can configure alternative names using the public or private IP addresses of all your brokers. For example: ++ +[source,ini] ---- [ alt_names ] IP.1 = 10.0.8.1 @@ -174,36 +182,22 @@ IP.2 = 10.0.8.2 IP.3 = 10.0.8.3 ---- -==== 2. Generate broker private key (broker.key) - -Generate a 2048-bit RSA private key for brokers: - -[,bash] +. Create a 2048-bit private key for the broker: ++ +[source,bash] ---- openssl genrsa -out broker.key 2048 ---- -where: - -[cols=",",options="header",] -|=== -|Output |Description -|`-out broker.key` |The private key certificate for the broker. -|=== - - -==== 3. Generate certificate signing request - -Before signing certificates, you must run the following command to generate the -broker's certificate signing request: - -[,bash] +. Create the broker's certificate signing request (CSR): ++ +[source,bash] ---- openssl req -new -key broker.key -out broker.csr -nodes -config broker.cnf ---- - ++ where: - ++ [cols=",",options="header",] |=== |Inputs |Description @@ -221,38 +215,52 @@ applying extensions. |`-in broker.csr` |Broker certificate signing request (CSR generated above). |=== ++ +The output `broker.crt` is the signed public key certificate for the broker. -[cols=",",options="header",] -|=== -|Output |Description -|`-out broker.crt` |The signed public key certificate for the broker. -|=== - -==== 4. Sign certificates - -Sign the certificate with the CA signature: - -[,bash] +. Sign the CSR with the CA's private key: ++ +[source,bash] ---- touch index.txt echo '01' > serial.txt openssl ca -config ca.cnf -keyfile ca.key -cert ca.crt -extensions extensions -in broker.csr -out broker.crt -outdir . -batch - -chown redpanda:redpanda broker.key broker.crt ca.crt -chmod 400 broker.key broker.crt ca.crt ---- - ++ If generated by a corporate CA, these certificate signing requests must be signed with the following extensions: - -[,bash] ++ +[,ini] ---- keyUsage = critical,digitalSignature,keyEncipherment extendedKeyUsage = serverAuth,clientAuth ---- +. Set ownership and permissions: ++ +[source,bash] +---- +chown redpanda:redpanda broker.key broker.crt ca.crt +chmod 400 broker.key broker.crt ca.crt +---- + == Configure TLS -To configure TLS, in `redpanda.yaml`, enter: +To configure TLS, in `redpanda.yaml`, enter either the standard <> or the <>. + +Choose PEM files when: + +- You are using FIPS mode compliance. +- You prefer file-based configurations with a separate key, certificate, and truststore file. + +Choose PKCS#12 bundles when: + +- FIPS mode is not required in your environment. +- You want a single, password-protected file that contains all certificates and keys. + +[[pem]] +=== Configure TLS with PEM files + +If you have separate files for `key_file`, `cert_file`, and `truststore_file`, use the following configuration in `redpanda.yaml`: .`redpanda.yaml` [,yaml] @@ -298,7 +306,7 @@ systemctl restart redpanda ==== [NOTE] -==== +==== If you replace a working `ca.crl` file with a file that contains an invalid certificate revocation list, such as an unsigned list, Redpanda will reject all connections until you either: * Replace the file. @@ -326,13 +334,44 @@ Schema Registry and HTTP Proxy connect to Redpanda over the Kafka API. If you co * xref:./authentication.adoc#configure-schema-registry-and-http-proxy-to-connect-to-redpanda-with-sasl[Configure Schema Registry and HTTP Proxy to connect to Redpanda with SASL] * xref:./listener-configuration.adoc[Configure Listeners] -include::manage:partial$min-tls-version.adoc[leveloffset=+1] +[[pkcs]] +=== Configure TLS with PKCS#12 bundles + +You can simplify certificate management by generating a password-protected PKCS#12 bundle from your `broker.key` and `broker.crt` files. + +NOTE: PKCS#12 keys are not supported when xref:manage:security/fips-compliance.adoc[FIPS mode] is enabled in Redpanda. The PKCS12KDF algorithm used in PKCS#12 is not FIPS-compliant. To use Redpanda in FIPS mode, configure your certificates and keys in <> instead. + +. Run this command to create a PKCS#12 file from your `broker.key` and `broker.crt` files: ++ +[source,bash] +---- +openssl pkcs12 -export -inkey broker.key -certfile broker.crt -passout pass: -out broker.p12 +---- ++ +Replace `` with your own password. + +. Update `redpanda.yaml` with the path to the PKCS#12 bundle: ++ +[source,yaml] +---- +redpanda: + kafka_api_tls: + - name: tls_listener + p12_path: + p12_password: + enabled: true + require_client_auth: false +---- + +CAUTION: Do not configure both both `key_file`/`cert_file` and `p12_path`/`p12_password` together, as this will cause a startup error. == Configure mTLS -To enable mTLS, add `require_client_auth` set to `true`. +To enable mTLS, add `require_client_auth` set to `true`. Configure either the standard PEM files or the PKCS#12 bundle. + +=== Configure mTLS with PEM files -For example, for the Kafka API, in `redpanda.yaml`, enter: +For the Kafka API, in `redpanda.yaml`, enter: .`redpanda.yaml` [,yaml] @@ -375,9 +414,9 @@ redpanda: - name: internal enabled: true require_client_auth: true - cert_file: - key_file: - truststore_file: + cert_file: + key_file: + truststore_file: ---- [NOTE] @@ -390,6 +429,25 @@ redpanda: See also: xref:./listener-configuration.adoc[Configure Listeners] +=== Configure mTLS with PKCS#12 bundles + +. Update `redpanda.yaml` with the path to the PKCS#12 bundle: ++ +[source,yaml] +---- +redpanda: + kafka_api_tls: + - name: mtls_listener + p12_path: + p12_password: + enabled: true + require_client_auth: true +---- + +CAUTION: Do not configure both the `key_file`/`cert_file` and the `p12_path`/`p12_password` together, as this will cause a startup error. + +include::manage:partial$min-tls-version.adoc[] + == Use rpk with TLS If you're using `rpk` to interact with the Kafka API using mTLS identity (for example, to manage topics or messages), pass the `--tls-key`, `--tls-cert`, and `--tls-truststore` flags to authenticate. diff --git a/modules/manage/pages/security/fips-compliance.adoc b/modules/manage/pages/security/fips-compliance.adoc index bdfbf2734e..d6152bb44b 100644 --- a/modules/manage/pages/security/fips-compliance.adoc +++ b/modules/manage/pages/security/fips-compliance.adoc @@ -9,11 +9,6 @@ Redpanda provides FIPS-compliant cipher enforcement for brokers using OpenSSL 3. include::shared:partial$enterprise-license.adoc[] ==== -[IMPORTANT] -==== -Redpanda is not fully FIPS-compliant when used with the Redpanda Helm chart and Operator in a Kubernetes deployment. -==== - To check if you already have a license key applied to your cluster: [,bash] @@ -25,7 +20,12 @@ rpk cluster license info Before configuring brokers to run in FIPS compliance mode (FIPS mode), check to make sure the `redpanda-rpk-fips` and `redpanda-fips` packages are xref:deploy:deployment-option/self-hosted/manual/production/production-deployment.adoc#install-redpanda-for-fips-compliance[installed]. These packages are required by both the `redpanda` and `redpanda-tuners` install packages. -== Configure FIPS +== Limitations + +- Redpanda is not fully FIPS-compliant when used with the Redpanda Helm chart and Operator in a Kubernetes deployment. +- PKCS#12 keys for xref:manage:security/encryption.adoc[TLS encryption] are not supported when FIPS mode is enabled in Redpanda. The PKCS12KDF algorithm used in PKCS#12 is not FIPS-compliant. To use Redpanda in FIPS mode with TLS enabled, configure your certificates and keys in PEM format instead. + +== Configure FIPS mode When you configure a broker to run in FIPS mode: diff --git a/modules/manage/pages/topic-iceberg-integration.adoc b/modules/manage/pages/topic-iceberg-integration.adoc new file mode 100644 index 0000000000..bd7139990a --- /dev/null +++ b/modules/manage/pages/topic-iceberg-integration.adoc @@ -0,0 +1,438 @@ += Iceberg Topics +:description: Learn how to integrate Redpanda topics with Apache Iceberg. +:page-context-links: [{"name": "Linux", "to": "manage:topic-iceberg-integration.adoc" } ] +:page-categories: Iceberg, Tiered Storage, Management, High Availability, Data Replication, Integration +:page-beta: true + +[NOTE] +==== +include::shared:partial$enterprise-license.adoc[] +==== + +The Apache Iceberg integration for Redpanda allows you to store topic data in the cloud in the Iceberg open table format. This makes your streaming data immediately available in downstream analytical systems, including data warehouses like Snowflake, Databricks, Clickhouse, and Redshift, without setting up and maintaining additional ETL pipelines. You can also integrate your data directly into commonly-used big data processing frameworks, such as Apache Spark and Flink, standardizing and simplifying the consumption of streams as tables in a wide variety of data analytics pipelines. + +The Iceberg integration uses xref:manage:tiered-storage.adoc[Tiered Storage]. When a cluster or topic has Tiered Storage enabled, Redpanda stores the Iceberg files in the configured Tiered Storage bucket or container. + +== Prerequisites + +* Install xref:get-started:rpk-install.adoc[`rpk`]. +* To check if you already have a license key applied to your cluster: ++ +[,bash] +---- +rpk cluster license info +---- +* Enable xref:manage:tiered-storage.adoc#set-up-tiered-storage[Tiered Storage] for the topics for which you want to generate Iceberg tables. + +== Limitations + +* It is not possible to append topic data to an existing Iceberg table that is not created by Redpanda. +* If you enable the Iceberg integration on an existing Redpanda topic, Redpanda does not backfill the generated Iceberg table with topic data. +* JSON schemas are not currently supported. If the topic data is in JSON, use the `key_value` mode to store the JSON in Iceberg, which then can be parsed by most query engines. +* If you are using Avro or Protobuf data, you must use the Schema Registry wire format, where producers include the magic byte and schema ID in the message payload header. See also: xref:manage:schema-reg/schema-id-validation.adoc[] and the https://www.redpanda.com/blog/schema-registry-kafka-streaming#how-does-serialization-work-with-schema-registry-in-kafka[Understanding Apache Kafka Schema Registry^] blog post to learn more about the wire format. +* You can only use one schema per topic. Schema versioning as well as upcasting (where a value is cast into its more generic data type) are not supported. See <> for more details. +* xref:manage:remote-read-replicas.adoc[Remote read replicas] and xref:manage:topic-recovery.adoc[topic recovery] are not supported for Iceberg-enabled topics. + +== Iceberg concepts + +Apache Iceberg is an open source format specification for defining structured tables in a data lake. The table format lets you quickly and easily manage, query, and process huge amounts of structured and unstructured data. This is similar to the way in which you would manage and run SQL queries against relational data in a database or data warehouse. The open format lets you use many different languages, tools, and applications to process the same data in a consistent way, so you can avoid vendor lock-in. This data management system is referred to as a _data lakehouse_. + +In the Iceberg specification, tables consist of the following layers: + +* Data layer +** Data files: Store the data. The Iceberg integration currently supports the Parquet file format. Parquet files are column-based and suitable for analytical workloads at scale. They come with compression capabilities that optimize files for object storage. +* Metadata layer: Stores table metadata separately from data files. The metadata layer allows multiple writers to stage metadata changes and apply updates atomically. It also supports database snapshots, and time travel queries that query the database at a previous point in time. ++ +-- +** Manifest files: Track data files and contain metadata about these files, such as record count, partition membership, and file paths. +** Manifest list: Tracks all the manifest files belonging to a table, including file paths and upper and lower bounds for partition fields. +** Metadata file: Stores metadata about the table, including its schema, partition information, and snapshots. Whenever a change is made to the table, a new metadata file is created and becomes the latest version of the metadata in the catalog. +-- ++ +In the Redpanda Iceberg integration, the manifest files are in JSON format. +* Catalog: Contains the current metadata pointer for the table. Clients reading and writing data to the table see the same version of the current state of the table. The Iceberg integration supports two <> types. You can configure Redpanda to catalog files stored in the same object storage bucket or container where the Iceberg data files are located, or you can configure Redpanda to use an https://iceberg.apache.org/concepts/catalog/#decoupling-using-the-rest-catalog[Iceberg REST catalog^] endpoint to update an externally-managed catalog when there are changes to the Iceberg data and metadata. + +image::shared:iceberg-integration.png[] + +When you enable the Iceberg integration for a Redpanda topic, Redpanda brokers store streaming data in the Iceberg-compatible format in Parquet files in object storage, in addition to the log segments uploaded using Tiered Storage. Storing the streaming data in Iceberg tables in the cloud allows you to derive real-time insights through many compatible data lakehouse, data engineering, and business intelligence https://iceberg.apache.org/vendors/[tools^]. + +== Enable Iceberg integration + +To create an Iceberg table for a Redpanda topic, you must set the cluster configuration property xref:reference:properties/cluster-properties.adoc#iceberg_enabled[`iceberg_delete`] to `true`, and also configure the topic property xref:reference:properties/topic-properties.adoc#redpandaicebergmode[`redpanda.iceberg.mode`]. You can choose to provide a schema if you need the Iceberg table to be structured with defined columns. + +. Set the `iceberg_enabled` configuration option on your cluster to `true`. You must restart your cluster if you change this configuration for a running cluster. ++ +[,bash] +---- +rpk cluster config set iceberg_enabled true +---- ++ +[,bash,role=no-copy] +---- +Successfully updated configuration. New configuration version is 2. +---- + +. Create a new topic. ++ +[,bash,] +---- +rpk topic create +---- ++ +[,bash,role=no-copy] +---- +TOPIC STATUS + OK +---- + +. Enable the integration for the topic by configuring `redpanda.iceberg.mode`. You can choose one of the following modes: ++ +-- +* `key_value`: Creates an Iceberg table using a simple schema, consisting of two columns, one for the record metadata including the key, and another binary column for the record's value. +* `value_schema_id_prefix`: Creates an Iceberg table whose structure matches the Redpanda schema for this topic, with columns corresponding to each field. You must register a schema in the Schema Registry (see next step), and producers must write to the topic using the Schema Registry wire format. Redpanda parses the schema used by the record based on the schema ID encoded in the payload header, and stores the topic values in the corresponding table columns. +* `disabled` (default): Disables writing to an Iceberg table for this topic. +-- ++ +[,bash] +---- +rpk topic alter-config --set redpanda.iceberg.mode= +---- ++ +[,bash,role=no-copy] +---- +TOPIC STATUS + OK +---- + +. Register a schema for the topic. This step is required for the `value_schema_id_prefix` mode, but is optional otherwise. ++ +[,bash] +---- +rpk registry schema create --schema --type +---- ++ +[,bash,role=no-copy] +---- +SUBJECT VERSION ID TYPE + 1 1 PROTOBUF +---- + +The Iceberg table is inside a namespace called `redpanda`, and has the same name as the Redpanda topic name. As you produce records to the topic, the data also becomes available in object storage for consumption by Iceberg-compatible clients. + +== Schema support and mapping + +The `redpanda.iceberg.mode` property determines how Redpanda maps the topic data to the Iceberg table structure. You can either have the generated Iceberg table match the stucture of a Avro or Protobuf schema in the Schema Registry, or use the `key_value` mode where Redpanda stores the record values as-is in the table. + +The JSON Schema format is not supported. If your topic data is in JSON, it is recommended to use the `key_value` mode. + +=== Iceberg modes and table schemas + +For both `key_value` and `value_schema_id_prefix` modes, Redpanda writes to a `redpanda` table column that stores a single Iceberg https://iceberg.apache.org/spec/#nested-types[struct^] per record, containing nested columns of the metadata from each record, including the record key, headers, timestamp, the partition it belongs to, and its offset. + +For example, if you produce to a topic according to the following Avro schema: + +[,avro] +---- +{ + "type": "record", + "name": "ClickEvent", + "fields": [ + { + "name": "user_id", + "type": "int" + }, + { + "name": "event_type", + "type": "string" + }, + { + "name": "ts", + "type": "string" + } + ] +} +---- + +The `key_value` mode writes to the following table format: + +[,sql] +---- +CREATE TABLE ClickEvent ( + redpanda struct< + partition: integer NOT NULL, + timestamp: timestamp NOT NULL, + offset: long NOT NULL, + headers: array>, + key: binary + >, + value binary +) +---- + +Consider this approach if the topic data is in JSON, or if you can use the Iceberg data in its semi-structured format. + +The `value_schema_id_prefix` mode translates to the following table format: + +[,sql] +---- +CREATE TABLE ClickEvent ( + redpanda struct< + partition: integer NOT NULL, + timestamp: timestamp NOT NULL, + offset: long NOT NULL, + headers: array>, + key: binary + >, + user_id integer NOT NULL, + event_type string, + ts string +) +---- + +With schema integration, Redpanda uses the schema ID prefix embedded in each record to find the matching schema in the Schema Registry. Producers to the topic must use the schema ID prefix in the serialization process so Redpanda can determine the schema used for each record, parse the record according to that schema, and use the schema for the Iceberg table as well. + +=== Schema types translation + +Redpanda supports direct translations of the following types to Iceberg value domains: + +[tabs] +====== +Avro:: ++ +-- +|=== +| Avro type | Iceberg type + +| boolean | boolean +| int | int +| long | long +| float | float +| double | double +| bytes | binary +| string | string +| record | struct +| array | list +| maps | list +| fixed | fixed +| decimal | decimal +| uuid | uuid +| date | date +| time | time +| timestamp | timestamp +|=== + +* Different flavors of time (such as `time-millis`) and timestamp (such as `timestamp-millis`) types are translated to the same Iceberg `time` and `timestamp` types respectively. +* Avro unions are flattened to Iceberg structs with optional fields: +** For example, the union `["int", "long", "float"]` is represented as an Iceberg struct `struct<0 INT NULLABLE, 1 LONG NULLABLE, 2 FLOAT NULLABLE>`. +** The union `["int", null, "float"]` is represented as an Iceberg struct `struct<0 INT NULLABLE, 1 FLOAT NULLABLE>`. +* All fields are required by default (Avro always sets a default in binary representation). +* The Avro duration logical type is ignored. +* The Avro null type is ignored and not represented in the Iceberg schema. +* Recursive types are not supported. +-- + +Protobuf:: ++ +-- +|=== +| Protobuf type | Iceberg type + +| bool | boolean +| double | double +| float | float +| int32 | int +| sint32 | int +| int64 | long +| sint64 | long +| sfixed32 | int +| sfixed64 | int +| string | string +| bytes | binary +| map | map +|=== + +* Repeated values are translated into Iceberg `array` types. +* Enums are translated into Iceberg `int` types based on the integer value of the enumerated type. +* `uint32` and `fixed32` are translated into Iceberg `long` types as that is the existing semantic for unsigned 32-bit values in Iceberg. +* `uint64` and `fixed64` values are translated into their Base-10 string representation. +* The `timestamp` type in Protobuf is translated into `timestamp` in Iceberg. +* Messages are converted into Iceberg structs. +* Recursive types are not supported. +-- +====== + +== Set up catalog integration + +You can configure the Iceberg integration to either store the metadata in https://iceberg.apache.org/javadoc/1.5.0/org/apache/iceberg/hadoop/HadoopCatalog.html[HadoopCatalog^] format in the same object storage bucket or container, or connect to a REST-based catalog. + +Set the cluster configuration property `iceberg_catalog_type` with one of the following values: + +* `rest`: Connect to and update an Iceberg catalog using a REST API. See the https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml[Iceberg REST Catalog API specification]. +* `object_storage`: Write catalog files to the same object storage bucket as the data files. Use the object storage URL with an Iceberg client to access the catalog and data files for your Redpanda Iceberg tables. + +Once you have enabled the Iceberg integration for a topic and selected a catalog type, you cannot switch to another catalog type. + +For production use cases, Redpanda recommends the `rest` option with REST-enabled Iceberg catalog services such as https://docs.tabular.io/[Tabular^], https://docs.databricks.com/en/data-governance/unity-catalog/index.html[Databricks Unity^] and https://github.com/apache/polaris[Apache Polaris^]. + +For an Iceberg REST catalog, set the following additional cluster configuration properties: + +* `iceberg_rest_catalog_endpoint`: The endpoint URL for your Iceberg catalog, which you are either managing directly, or is managed by an external catalog service. +* `iceberg_rest_catalog_client_id`: The ID to connect to the REST server. +* `iceberg_rest_catalog_client_secret`: The secret data to connect to the REST server. ++ +-- +For REST catalogs that use self-signed certificates, also configure these properties: + +* `iceberg_rest_catalog_trust_file`: The path to a file containing a certificate chain to trust for the REST catalog. +* `iceberg_rest_catalog_crl_file`: The path to the certificate revocation list for the specified trust file. +-- + +// update xref when PR for extracted properties is ready +See xref:reference:properties/cluster-properties.adoc[Cluster Configuration Properties] for the full list of cluster properties to configure for a catalog integration. + +=== Example catalog integration + +You'll be able to use to the catalog to load, query, or refresh the Iceberg data as you produce to the Redpanda topic. Refer to the official documentation of your query engine or Iceberg-compatible tool for guidance on integrating your Iceberg catalog. + +==== REST catalog + +For example, if you have Redpanda cluster configuration properties set to connect to a REST catalog named `streaming`: + +[,yaml] +---- +iceberg_rest_catalog_type: rest +iceberg_rest_catalog_endpoint: http://catalog-service:8181 +iceberg_rest_catalog_client_id: +iceberg_rest_catalog_client_secret: +---- + +And you use Spark as a processing engine, configured to use the `streaming` catalog: + +``` +spark.sql.catalog.streaming = org.apache.iceberg.spark.SparkCatalog +spark.sql.catalog.streaming.type = rest +spark.sql.catalog.streaming.uri = http://catalog-service:8181 +``` + +Using Spark SQL, you can query the Iceberg table directly by specifying the catalog name: + +[,sql] +---- +SELECT * FROM streaming.redpanda.ClickEvent; +---- + +Spark can use the REST catalog to automatically discover the topic's Iceberg table. + +==== File system-based catalog (`object_storage`) + +If you are using the `object_storage` catalog type, you must set up the catalog integration in your processing engine accordingly. For example, you can configure Spark to use a file system-based catalog with at least the following properties, is using AWS S3 for object storage: + +``` +spark.sql.catalog.streaming.type = hadoop +spark.sql.catalog.streaming.warehouse = s3a:///path/to/redpanda-iceberg-table +``` + +Depending on your processing engine, you may also need to create a new table in your data warehouse or lakehouse for the Iceberg data. + +== Access data in Iceberg tables + +You can use the same analytical tools to access table data in a data lake as you would for a relational database. + +For example, suppose you produce the same stream of events to a topic `ClickEvent`, which uses a schema, and another topic `ClickEvent_key_value`, which uses the key-value mode. The topics have Tiered Storage configured to an AWS S3 bucket. A sample record contains the following data: + +[,bash,role=no-copy] +---- +{"user_id": 2324, "event_type": "BUTTON_CLICK", "ts": "2024-11-25T20:23:59.380Z"} +---- + +When you point your Iceberg-compatible tool or framework to the object storage location of the Iceberg tables, how you consume the data depends on the topic Iceberg mode and whether you've registered a schema on which to derive the table format. Depending on the processing engine, you may need to first create a new table that gets added to your Iceberg catalog implementation. + +In either mode, you do not need to rely on complex ETL jobs or pipelines to access real-time data from Redpanda in your data lakehouse. + +=== Query topic with schema (`value_schema_id_prefix` mode) + +In this example, it is assumed you have created the `ClickEvent` topic, set `redpanda.iceberg.mode` to `value_schema_id_prefix`, and are connecting to a REST-based Iceberg catalog. The following is an Avro schema for `ClickEvent`: + +.`schema.avsc` +[,avro] +---- +{ + "type" : "record", + "namespace" : "com.redpanda.examples.avro", + "name" : "ClickEvent", + "fields" : [ + { "name": "user_id", "type" : "int" }, + { "name": "event_type", "type" : "string" }, + { "name": "ts", "type": "string" } + ] + } +---- + +You can register the schema under the `ClickEvent-value` subject: + +[,bash] +---- +rpk registry schema create ClickEvent-value --schema path/to/schema.avsc --type avro +---- + +You can then produce to the `ClickEvent` topic using the following format: + +[,bash] +---- +echo '"key1" {"user_id":2324,"event_type":"BUTTON_CLICK","ts":"2024-11-25T20:23:59.380Z"}' | rpk topic produce ClickEvent --format='%k %v\n' --schema-id=topic +---- + +The following SQL query returns values from columns in the `ClickEvent` table, with the table structure derived from the schema, and column names matching the schema fields. If you've integrated a catalog, query engines such as Spark SQL provide Iceberg integrations that allow easy discovery and access to existing Iceberg tables in object storage. + +[,sql] +---- +SELECT user_id, + event_type, + ts +FROM .ClickEvent; +---- + +[,bash,role=no-copy] +---- ++---------+--------------+--------------------------+ +| user_id | event_type | ts | ++---------+--------------+--------------------------+ +| 2324 | BUTTON_CLICK | 2024-11-25T20:23:59.380Z | ++---------+--------------+--------------------------+ +---- + +=== Query topic in key-value mode + +In `key_value` mode, you do not associate the topic with a schema in the Schema Registry, which means using semi-structured data in Iceberg. + +In this example, it is assumed you have created the `ClickEvent_key_value` topic, set `redpanda.iceberg.mode` to `key_value`, and are also connecting to a REST-based Iceberg catalog. + +You can produce to the `ClickEvent_key_value` topic using the following format: + +[,bash] +---- +echo 'key1 {"user_id":2324,"event_type":"BUTTON_CLICK","ts":"2024-11-25T20:23:59.380Z"}' | rpk topic produce ClickEvent_key_value --format='%k %v\n' +---- + +This example queries the semi-structured data in the `ClickEvent_key_value` table, which also consists of another column `redpanda` containing the record key and other metadata: + +[,sql] +---- +SELECT + value +FROM .ClickEvent_key_value; +---- + +[,bash,role=no-copy] +---- ++------------------------------------------------------------------------------+ +| value | ++------------------------------------------------------------------------------+ +| {"user_id":2324,"event_type":"BUTTON_CLICK","ts":"2024-11-25T20:23:59.380Z"} | ++------------------------------------------------------------------------------+ +---- + +== Suggested reading + +* xref:manage:schema-reg/schema-id-validation.adoc[] +* https://www.redpanda.com/blog/schema-registry-kafka-streaming#how-does-serialization-work-with-schema-registry-in-kafka[Understanding Apache Kafka Schema Registry^] diff --git a/modules/manage/partials/authentication.adoc b/modules/manage/partials/authentication.adoc index f5171f9231..921903f127 100644 --- a/modules/manage/partials/authentication.adoc +++ b/modules/manage/partials/authentication.adoc @@ -690,7 +690,7 @@ endif::[] [[oidc]] ==== OAUTHBEARER (OIDC) -NOTE: OpenID Connect (OIDC) authentication requires an xref:get-started:licenses.adoc[Enterprise license]. +NOTE: OpenID Connect (OIDC) authentication requires an xref:get-started:licenses.adoc[enterprise license]. To upgrade, contact https://redpanda.com/try-redpanda?section=enterprise-trial[Redpanda sales^]. When you enable https://openid.net/developers/how-connect-works/[OIDC^], Redpanda and @@ -971,7 +971,7 @@ ifndef::env-kubernetes[] [[kerberos]] ==== GSSAPI (Kerberos) -NOTE: Kerberos authentication requires an xref:get-started:licenses.adoc[Enterprise license]. To upgrade, contact https://redpanda.com/try-redpanda?section=enterprise-trial[Redpanda sales^]. +NOTE: Kerberos authentication requires an xref:get-started:licenses.adoc[enterprise license]. To upgrade, contact https://redpanda.com/try-redpanda?section=enterprise-trial[Redpanda sales^]. To configure Kerberos authentication, use a keytab, which contains credentials for the service. diff --git a/modules/manage/partials/mountable-topics.adoc b/modules/manage/partials/mountable-topics.adoc new file mode 100644 index 0000000000..68c31b8369 --- /dev/null +++ b/modules/manage/partials/mountable-topics.adoc @@ -0,0 +1,284 @@ +For topics with Tiered Storage enabled, you can unmount a topic to safely detach it from a cluster and keep the topic data in the cluster's object storage bucket or container. You can mount the detached topic to either the same origin cluster, or a different one. This allows you to hibernate a topic and free up system resources taken up by the topic, or migrate a topic to a different cluster. + +== Prerequisites + +[NOTE] +==== +include::shared:partial$enterprise-license.adoc[] +==== + +. xref:get-started:rpk-install.adoc[Install `rpk`], or ensure that you have access to the Admin API. +. Enable xref:manage:tiered-storage.adoc[Tiered Storage] for specific topics, or for the entire cluster (all topics). + +== Unmount a topic from a cluster to object storage + +When you unmount a topic, all incoming writes to the topic are blocked as Redpanda unmounts the topic from the cluster to object storage. Producers and consumers of the topic receive a message in the protocol replies indicating that the topic is no longer available: + +- Produce requests receive an `invalid_topic_exception` or `resource_is_being_migrated` response from the broker. +- Consume requests receive an `invalid_topic_exception` response from the broker. + +An unmounted topic in object storage is detached from all clusters. The original cluster releases ownership of the topic. + +NOTE: The unmounted topic is deleted in the source cluster, but can be mounted back again from object storage. + +[tabs] +====== +rpk:: ++ +-- +In your cluster, run this command to unmount a topic to object storage: + +``` +rpk cluster storage unmount / +``` +-- +Admin API:: ++ +-- +To unmount topics from a cluster using the Admin API, make a POST request to the `/v1/topics/unmount` endpoint. Specify the names of the desired topics in the request body: + +``` +curl -X POST http://localhost:9644/v1/topics/unmount -d { + "topics": [ + { + "topic": "" + }, + { + "topic": "" + }, + { + "topic": "" + } + ] +} +``` + +You may optionally include the topic namespace (`ns`). Only `kafka` is supported. +-- +====== + +You can use the ID returned by the command to <> of the unmount operation using `rpk` or the Admin API. + +== Mount a topic to a cluster + +[tabs] +====== +rpk:: ++ +-- +. In your target cluster, run this command to list the topics that are available to mount from object storage: ++ +``` +rpk cluster storage list-mountable +``` ++ +The command output returns a `LOCATION` value in the format `//`. Redpanda assigns an `initial-revision` number to a topic upon creation. ++ +The location value uniquely identifies a topic in object storage if multiple topics had the same name when they were unmounted from different origin clusters. For example: ++ +``` +TOPIC NAMESPACE LOCATION +testtopic kafka testtopic/67f5505a-32f3-4677-bcad-3c75a1a702a6/10 +``` ++ +You can use the location as the topic reference instead of just the topic name to uniquely identify a topic to mount in the next step. + +. Mount a topic from object storage: ++ +``` +rpk cluster storage mount +``` ++ +Replace `` with the name of the topic to mount. If there are multiple topics wih the same name in object storage, you are required to use the location value from `rpk cluster storage list-mountable` to uniquely identify a topic. ++ +You can also specify a new name for the topic as you mount it to the target cluster: ++ +``` +rpk cluster storage mount --to +``` ++ +You only use the new name for the topic in the target cluster. This name does not persist if you unmount this topic again. Redpanda keeps the original name in object storage if you remount the topic later. +-- +Admin API:: ++ +-- +. List the topics that are available to mount from object storage by making a GET request to the `v1/topics/mountable` endpoint. ++ +``` +curl http://localhost:9644/v1/topics/mountable +``` ++ +The response object contains an array of topics: ++ +[,bash,role=no-placeholders] +---- +"topics": [ + { + "topic_location": "topic-1-name//", + "topic": "topic-1-name" + }, + { + "topic_location": "topic-2-name//", + "topic": "topic-2-name" + } +] +---- ++ +The `topic_location` is the unique topic location in object storage, in the format `//`. Redpanda assigns the number `initial-revision` to a topic upon creation. You can use `topic-location` as the topic reference instead of just the topic name to identify a unique topic to mount in the next step. + +. To mount topics to a target cluster using the Admin API, make a POST request to the `/v1/topics/mount` endpoint. Specify the names of the topics in the request body: ++ +``` +curl -X POST http://localhost:9644/v1/topics/mount -d { + "topics": [ + { + "source_topic_reference": {"ns": "kafka", "topic": "//"}, + "alias": {"ns": "kafka", "topic": ""} + }, + { + "source_topic_reference": {"ns": "kafka", "topic": ""} + }, + { + "source_topic_reference": {"ns": "kafka", "topic": "//"}, + "alias": {"ns": "kafka", "topic": ""} + } + ] +} +``` ++ +* `ns` is the topic namespace. This field is optional and only `kafka` is supported. +* You may have multiple topics with the same name that are available to mount from object storage. This can happen if you have unmounted topics with this name from different clusters. To uniquely identify a source topic, use `//` as the topic reference. +* To rename a topic in the target cluster, use the optional `alias` object in the request body. In the example, you specify new names for topics 1 and 3 in the target cluster, but topic 2 retains its original name in the target cluster. + +-- + +====== + +You can use the ID returned by the operation to <> using `rpk` or the Admin API. + +When the mount operation is complete, the target cluster handles produce and consume workloads for the topics. + +== Monitor progress + +[tabs] +====== +rpk:: ++ +-- +To list active mount and unmount operations, run the command: + +``` +rpk cluster storage list-mount +``` +-- + +Admin API:: ++ +-- +Issue a GET request to the `/migrations` endpoint to view the status of topic mount and unmount operations: + +``` +curl http://localhost:9644/v1/migrations +``` +-- +====== + +You can also retrieve the status of a specific operation by running the command: + + +[tabs] +====== +rpk:: ++ +-- +``` +rpk cluster storage status-mount +``` +-- +Admin API:: ++ +-- +``` +curl http://localhost:9644/v1/migrations/ +``` +-- +====== + +`` is the unique identifier of the operation. Redpanda returns this ID when you start a mount or unmount. You can also retrieve the ID by listing <>. + +The response returns the IDs and state of existing mount and unmount operations ("migrations"): + +|=== +| State | Unmount operation (outbound) | Mount operation (inbound) + +| `planned` +2+| Redpanda validates the mount or unmount operation definition. + +| `preparing` +| Redpanda flushes topic data, including topic manifests, to the destination bucket or container in object storage. +| Redpanda recreates the topics in a disabled state in the target cluster. The cluster allocates partitions but does not add log segments yet. Topic metadata is populated from the topic manifests found in object storage. + +| `prepared` +| The operation is ready to execute. In this state, the cluster still accepts client reads and writes for the topics. +| Topics exist in the cluster but clients do not yet have access to consume or produce. + +| `executing` +| The cluster rejects client reads and writes for the topics. Redpanda uploads any remaining topic data that has not yet been copied to object storage. Uncommitted transactions involving the topic are aborted. +| The target cluster checks that the topic to be mounted has not already been mounted in any cluster. + +| `executed` +| All unmounted topic data from the cluster is available in object storage. +| The target cluster has verified that the topic has not already been mounted. + +| `cut_over` +| Redpanda deletes topic metadata from the cluster, and marks the data in object storage as available for mount operations. +| The topic data in object storage is no longer available to mount to any clusters. + +| `finished` +| The operation is complete. +| The operation is complete. The target cluster starts to handle produce and consume requests. + +| `canceling` +2+| Redpanda is in the process of canceling the mount or unmount operation. + +| `cancelled` +2+| The mount or unmount operation is cancelled. + +|=== + +== Cancel a mount or unmount operation + +You can cancel a topic mount or unmount by running the command: + +[tabs] +====== +rpk:: ++ +-- +``` +rpk cluster storage cancel-mount +``` +-- + +Admin API:: ++ +-- +``` +curl -X POST http://localhost:9644/v1//?action=cancel +``` +-- +====== + +You cannot cancel mount and unmount operations in the following <>: + +- `planned` (but you may still xref:api:ROOT:admin-api.adoc#delete-/v1/migrations/-id-[delete] a planned mount or unmount) +- `cut_over` +- `finished` +- `canceling` +- `cancelled` + +== Additional considerations + +Redpanda prevents you from mounting the same topic to multiple clusters at once. This ensures that multiple clusters don't write to the same location in object storage and corrupt the topic. + +If you attempt to mount a topic where the name matches a topic already in the target cluster, Redpanda fails the operation and emits a warning message in the logs. diff --git a/modules/manage/partials/tiered-storage.adoc b/modules/manage/partials/tiered-storage.adoc index 80697192b0..614c0513f6 100644 --- a/modules/manage/partials/tiered-storage.adoc +++ b/modules/manage/partials/tiered-storage.adoc @@ -58,9 +58,12 @@ TIP: If deploying Redpanda on an AWS Auto-Scaling group (ASG), keep in mind that You can configure access to Amazon S3 with either an IAM role attached to the instance or with access keys. -===== Use IAM roles +[TIP] +==== +If you need to manage and store encryption keys separately from your cloud provider, you can <> instead of the default AWS S3-managed key (SSE-S3). This option enables you to segregate data from different teams or departments and remove that data at will by removing the encryption keys. +==== -To configure access to an S3 bucket with an IAM role: +===== **Configure access with an IAM role** . Configure an xref:manage:security/iam-roles.adoc#configuring-iam-roles[IAM role]. @@ -138,9 +141,7 @@ Replace the following placeholders: + CAUTION: Do not set an object storage property to an empty string `""` or to `null` as a way to reset it to its default value. -===== Use access keys - -To configure access to an S3 bucket with access keys instead of an IAM role: +===== **Configure access with access keys** . Grant an IAM user the following permissions to read and create objects in your buckets: - `GetObject` @@ -258,13 +259,13 @@ Replace the following placeholders: + CAUTION: Do not set an object storage property to an empty string `""` or to `null` as a way to reset it to its default value. -==== Google Cloud Storage +include::manage:partial$tiered-storage/aws-kms-key.adoc[leveloffset=+4] -You can configure access to Google Cloud Storage with either an IAM role attached to the instance or with access keys. +==== Google Cloud Storage -===== Use IAM roles +Configure access to Google Cloud Storage with either an IAM role attached to the instance, with access keys, or with customer-managed keys. -To configure access to Google Cloud Storage with an IAM role: +===== **Configure access with an IAM role** . Configure an xref:manage:security/iam-roles.adoc#configuring-iam-roles[IAM role]. . Override the following required cluster properties in the Helm chart: @@ -343,7 +344,7 @@ Replace the following placeholders: CAUTION: Do not set an object storage property to an empty string `""` or to `null` as a way to reset it to its default value. -===== Use access keys +===== **Configure access with access keys** To configure access to Google Cloud Storage with access keys instead of an IAM role: @@ -462,15 +463,15 @@ Replace the following placeholders: + CAUTION: Do not set an object storage property to an empty string `""` or to `null` as a way to reset it to its default value. +include::manage:partial$tiered-storage/gcp-kms-key.adoc[leveloffset=+4] + ==== Microsoft ABS/ADLS You can configure access to Azure Blob Storage with either account access keys or Azure's managed identities system to securely interact with Azure Blob Storage. Account access keys, as static credentials, require manual management and vigilant security practices to prevent breaches due to their unchanging nature. In contrast, managed identities provide a more secure and maintenance-free solution by automating credential management and rotation, though they are exclusive to the Azure ecosystem. include::manage:partial$azure-blob-limitations.adoc[] -===== Use managed identities - -To configure access to an Azure container with a managed identity in AKS: +===== **Configure access with a managed identity** . Configure an xref:manage:security/iam-roles.adoc#configuring-iam-roles[Azure managed identity]. . Override the following required cluster properties in the Helm chart: @@ -570,9 +571,7 @@ NOTE: The `serviceAccount` annotations and the `statefulset` Pod labels are esse CAUTION: Do not set an object storage property to an empty string `""` or to `null` as a way to reset it to its default value. -===== Use access keys - -To configure access to ABS/ADLS with account access keys: +===== **Configure access with account access keys** . Get an account access key for the Azure container that Redpanda will run on. For information on how to view your account access keys, see the https://learn.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json&bc=%2Fazure%2Fstorage%2Fblobs%2Fbreadcrumb%2Ftoc.json&tabs=azure-portal#view-account-access-keys[Azure documentation^]. . Create a Secret in which to store the access key. @@ -688,9 +687,14 @@ Amazon S3:: TIP: If deploying Redpanda on an AWS Auto-Scaling group (ASG), keep in mind that the ASG controller terminates nodes and spins up replacements if the nodes saturate and are unable to heartbeat the controller (based on the EC2 health check). For more information, see the https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/monitoring-system-instance-status-check.html#types-of-instance-status-checks[AWS documentation^]. Redpanda recommends deploying on Linux or Kubernetes. For more information, see xref:deploy:deployment-option/self-hosted/index.adoc[Deploy Redpanda]. -Configure access to Amazon S3 with either an IAM role attached to the instance or with access keys. +Configure access to Amazon S3 with either an IAM role attached to the instance, with access keys, or with customer-managed keys. + +[TIP] +==== +If you need to manage and store encryption keys separately from your cloud provider, you can <> instead of the default AWS S3-managed key (SSE-S3). This option enables you to segregate data from different teams or departments and remove that data at will by removing the encryption keys. +==== -To configure access to an S3 bucket with an IAM role: +**Configure access with an IAM role** . Configure an xref:manage:security/iam-roles.adoc#configuring-iam-roles[IAM role]. . Run the `rpk cluster config edit` command, then edit the following required properties: @@ -707,7 +711,7 @@ Replace `` with your own values. CAUTION: Do not set an object storage property to an empty string `""` or to `null` as a way to reset it to its default value. To reset a property to its default value, run `rpk cluster config force-reset ` or remove that line from the cluster configuration with `rpk cluster config edit`. -To configure access to an S3 bucket with access keys instead of an IAM role: +**Configure access with access keys** . Grant a user the following permissions to read and create objects on the bucket to be used with the cluster (or on all buckets): `GetObject`, `DeleteObject`, `PutObject`, `PutObjectTagging`, `ListBucket`. . Copy the access key and secret key for the `cloud_storage_access_key` and `cloud_storage_secret_key` cluster properties. @@ -726,13 +730,15 @@ Replace `` with your own values. + CAUTION: Do not set an object storage property to an empty string `""` or to `null` as a way to reset it to its default value. To reset a property to its default value, run `rpk cluster config force-reset ` or remove that line from the cluster configuration with `rpk cluster config edit`. +include::manage:partial$tiered-storage/aws-kms-key.adoc[leveloffset=+3] + -- Google Cloud Storage:: + -- -Configure access to Google Cloud Storage with either an IAM role attached to the instance or with access keys. +Configure access to Google Cloud Storage with either an IAM role attached to the instance, with access keys, or with customer-managed keys. -To configure access to Google Cloud Storage with an IAM role: +**Configure access with an IAM role** . Configure an xref:manage:security/iam-roles.adoc#configuring-iam-roles[IAM role]. . Run the `rpk cluster config edit` command, then edit the following required properties: @@ -750,7 +756,7 @@ Replace `` with your own values. + CAUTION: Do not set an object storage property to an empty string `""` or to `null` as a way to reset it to its default value. To reset a property to its default value, run `rpk cluster config force-reset ` or remove that line from the cluster configuration with `rpk cluster config edit`. -To configure access to Google Cloud Storage with access keys instead of an IAM role: +**Configure access with access keys** . Choose a uniform access control when you create the bucket. . Use a Google-managed encryption key. @@ -772,6 +778,8 @@ Replace `` with your own values. + CAUTION: Do not set an object storage property to an empty string `""` or to `null` as a way to reset it to its default value. To reset a property to its default value, run `rpk cluster config force-reset ` or remove that line from the cluster configuration with `rpk cluster config edit`. +include::manage:partial$tiered-storage/gcp-kms-key.adoc[leveloffset=+3] + -- Microsoft ABS/ADLS:: + @@ -779,7 +787,7 @@ Microsoft ABS/ADLS:: include::manage:partial$azure-blob-limitations.adoc[] -To configure access to an Azure container with a managed identity: +**Configure access with managed identities** . Configure an xref:manage:security/iam-roles.adoc#azure-prerequisites[Azure managed identity]. + @@ -805,7 +813,7 @@ cloud_storage_azure_container: + Replace `` with your own values. -To configure access to Azure Blob Storage with account access keys: +**Configure access with account access keys** . Copy an account access key for the Azure container you want Redpanda to use and enter it in the `cloud_storage_azure_shared_key` property. For information on how to view your account access keys, see the https://learn.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json&bc=%2Fazure%2Fstorage%2Fblobs%2Fbreadcrumb%2Ftoc.json&tabs=azure-portal#view-account-access-keys[Azure documentation^]. . Run the `rpk cluster config edit` command, then edit the following required properties: diff --git a/modules/manage/partials/tiered-storage/aws-kms-key.adoc b/modules/manage/partials/tiered-storage/aws-kms-key.adoc new file mode 100644 index 0000000000..a7b34787ee --- /dev/null +++ b/modules/manage/partials/tiered-storage/aws-kms-key.adoc @@ -0,0 +1,71 @@ +ifndef::env-kubernetes[] +[discrete] +endif::[] += **Configure access with an AWS KMS key** + +When there are strict data compliance requirements and you must manage and store encryption keys separate from your cloud provider, you can configure an Amazon S3 bucket that Tiered Storage can use to leverage your customer-provided key (SSE-KMS) instead of the default AWS-managed key (SSE-S3). + +To convert an existing S3 bucket and its contents, you must: + +. Create a new KMS key. +. Configure the S3 bucket to use the new KMS key. +. (Optional) Re-encrypt existing objects to use the new KMS key. + +[NOTE] +==== +ifdef::env-cloud[] +You cannot configure a cloud-provider managed encryption key at the topic level or in Redpanda Cloud Dedicated clusters. +endif::[] +ifndef::env-cloud[] +You cannot configure a cloud provider-managed encryption key at the topic level. +endif::[] + +For topic-level control, each CLI Get or Put for a partition must use the correct key as configured on the topic. +==== + +ifndef::env-kubernetes[] +[discrete] +endif::[] +== **Prerequisites** + +- The user configuring S3 bucket encryption must be assigned the Key admin permission. Without this permission, the user is unable to re-encrypt existing bucket objects to use the KMS key. +- The S3 bucket must be assigned the Key user permission. Without this permission, Redpanda is unable to write new objects to Tiered Storage. +- If you intend to retroactively re-encrypt existing data with the new KMS key, store the ARN identifier of the key upon creation. It is required for AWS CLI commands. + +To create a new KMS key in the AWS Console: + +. In AWS Console, search for “Key Management Service”. +. Click **Create a key**. +. On the Configure key page, select the **Symmetric** key type, then select **Encrypt and decrypt**. +. Click the **Advanced options** tab and configure Key material origin and Regionality as needed. For example, if you are using xref:manage:remote-read-replicas.adoc[Remote Read Replicas] or have Redpanda spanning across regions, select **Multi-Region key**. +. Click **Next**. +. On the Add labels page, specify an alias name and description for the key. Do not include sensitive information in these fields. +. Click **Next**. +. On the Define key administrative permissions page, specify a user who can administer this key through the KMS API. +. Click **Next**. +. On the Define key usage permissions page, assign the S3 bucket as a Key user. This is required for the S3 bucket to encrypt and decrypt. +. Click **Next**. +. Review your KMS key configuration and click **Finish**. + +For more information, see the https://docs.aws.amazon.com/kms/latest/developerguide/create-symmetric-cmk.html[AWS documentation^]. + +To configure the S3 bucket to use the new KMS key (and reduce KMS costs through caching): + +. In AWS Console, search for "S3". +. Select the bucket used by Redpanda. +. Click the **Properties** tab. +. In Default encryption, click **Edit**. +. For Encryption type, select “Server-side encryption with AWS Key Management Service keys (SSE-KMS)”. +. Locate and select your AWS KMS key ARN identifier. +. Click **Save changes**. + +(Optional) To re-encrypt existing data using the new KMS key: + +Existing data in your S3 bucket continues to be read using the AWS-managed key, while new objects are encrypted using the new KMS key. If you wish to re-encrypt all S3 bucket data to use the KMS key, run: + +[,bash] +---- +aws s3 cp s3://{BUCKET_NAME}/ s3://{BUCKET_NAME}/ --recursive --sse-kms-key-id {KMS_KEY_ARN} --sse aws:kms +---- + +For more information, see the https://docs.aws.amazon.com/AmazonS3/latest/userguide/configuring-bucket-key.html[AWS documentation^]. \ No newline at end of file diff --git a/modules/manage/partials/tiered-storage/gcp-kms-key.adoc b/modules/manage/partials/tiered-storage/gcp-kms-key.adoc new file mode 100644 index 0000000000..90c8bc7775 --- /dev/null +++ b/modules/manage/partials/tiered-storage/gcp-kms-key.adoc @@ -0,0 +1,78 @@ +ifndef::env-kubernetes[] +[discrete] +endif::[] += **Configure access with a KMS key** + +To configure the Google Cloud bucket used by Redpanda Tiered Storage to leverage a customer-managed key using the Cloud Key Management Service API instead of the default Google-managed key, you must: + +. Create a KMS key. +. Configure the bucket to use the KMS key. +. Optionally, re-encrypt existing data with the new KMS key. + +To manage Google Cloud KMS using the command line, first install or upgrade to the latest version of https://cloud.google.com/sdk/docs/install[Google Cloud CLI^]. + +To create a KMS key: + +. In Google Cloud Console, search for "Cloud Key Managment Service API". Click **Enable** if the service is not already enabled. +. Using the Google Cloud CLI, create a new keyring in the https://cloud.google.com/kms/docs/locations^[region] where the bucket used by Redpanda is located. Note that region is case sensitive. ++ +[,bash, indent] +---- +gcloud kms keyrings create "redpanda-keyring" --location="{REGION}" +---- ++ +. Create a new key for the keyring in the same region as the bucket: ++ +[,bash, indent] +---- +gcloud kms keys create "redpanda-key" \ + --location="{REGION}" \ + --keyring="redpanda-keyring" \ + --purpose="encryption" +---- ++ +. Get the key identifier: ++ +[,bash] +---- +gcloud kms keys list \ + --location="REGION" \ + --keyring="redpanda-keyring" +---- ++ +The result should look like the following. Be sure to store the name, as this is used to assign and manage the key. Use this as the \{KEY_RESOURCE} placeholder in subsequent commands. ++ +[,bash] +---- +NAME PURPOSE ALGORITHM PROTECTION_LEVEL LABELS PRIMARY_ID PRIMARY_STATE +projects/{PROJECT_NAME}/locations/us/keyRings/redpanda-keyring/cryptoKeys/redpanda-key +ENCRYPT_DECRYPT GOOGLE_SYMMETRIC_ENCRYPTION SOFTWARE 1 ENABLED +---- + +To configure the GCP bucket to use the KMS key: + +. Assign the key to a service agent: ++ +[,bash] +---- +gcloud storage service-agent \ + --project={PROJECT_ID_STORING_OBJECTS} \ + --authorize-cmek={KEY_RESOURCE} +---- ++ +. Set the bucket default encryption key to the KMS key: ++ +[,bash] +---- +gcloud storage buckets update gs://{BUCKET_NAME} \ + --default-encryption-key={KEY_RESOURCE} +---- + +(Optional) To re-encrypt existing data using the new KMS key: + +Existing data in the bucket continues to be read using the Google-managed key, while new objects are encrypted using the new KMS key. If you wish to re-encrypt all data in the bucket to use the KMS key, run: +[,bash] +---- +gcloud storage objects update gs://{BUCKET_NAME}/ --recursive \ + --encryption-key={KEY_RESOURCE} +---- \ No newline at end of file diff --git a/modules/reference/pages/internal-metrics-reference.adoc b/modules/reference/pages/internal-metrics-reference.adoc index 3f0d8ea283..84bd8c6921 100644 --- a/modules/reference/pages/internal-metrics-reference.adoc +++ b/modules/reference/pages/internal-metrics-reference.adoc @@ -44,6 +44,12 @@ If this is the last record received by the cluster, then the cluster is up-to-da --- +=== vectorized_cluster_features_enterprise_license_expiry_sec + +Number of seconds remaining until the Enterprise Edition license expires. + +--- + === vectorized_cluster_partition_schema_id_validation_records_failed Number of records that failed schema ID validation. diff --git a/modules/reference/pages/monitor-metrics.adoc b/modules/reference/pages/monitor-metrics.adoc index 8645bc6ab6..45373ff02e 100644 --- a/modules/reference/pages/monitor-metrics.adoc +++ b/modules/reference/pages/monitor-metrics.adoc @@ -2,6 +2,6 @@ :description: Reference of monitoring metrics provided by Redpanda. :page-layout: index -Redpanda exports metrics through Prometheus endpoints, `/public_metrics` and `/metrics`. To learn how to monitor Redpanda, see xref:manage:monitoring.adoc[Monitor Redpanda]. +Redpanda exports metrics through Prometheus endpoints, `/public_metrics` and `/metrics`. To learn how to monitor Redpanda, see xref:manage:monitoring.adoc[Monitor Redpanda] or xref:manage:kubernetes/monitoring/index.adoc[]. include::shared:partial$metrics-usage-tip.adoc[] diff --git a/modules/reference/pages/properties/broker-properties.adoc b/modules/reference/pages/properties/broker-properties.adoc index 94d39e2301..ebcf4fab4f 100644 --- a/modules/reference/pages/properties/broker-properties.adoc +++ b/modules/reference/pages/properties/broker-properties.adoc @@ -14,8 +14,6 @@ NOTE: All broker properties require that you restart Redpanda for any update to Network address for the glossterm:Admin API[] server. -*Optional:* Yes - *Visibility:* `user` *Default:* `127.0.0.1:9644` @@ -26,8 +24,6 @@ Network address for the glossterm:Admin API[] server. Path to the API specifications for the Admin API. -*Optional:* Yes - *Visibility:* `user` *Type:* string @@ -40,8 +36,6 @@ Path to the API specifications for the Admin API. Specifies the TLS configuration for the HTTP Admin API. -*Optional:* Yes - *Visibility:* `user` *Default:* `null` @@ -61,9 +55,7 @@ The crash-tracking logic is reset (to zero consecutive crashes) by any of the fo * The `redpanda.yaml` broker configuration file is updated. * The `startup_log` file in the broker's <> is manually deleted. -*Units*: number of consecutive crashes of a broker - -*Optional:* No +*Unit*: number of consecutive crashes of a broker *Visibility:* `user` @@ -79,8 +71,6 @@ The crash-tracking logic is reset (to zero consecutive crashes) by any of the fo Path to the directory for storing Redpanda's streaming data files. -*Optional:* Yes - *Visibility:* `user` *Type:* string @@ -93,9 +83,7 @@ Path to the directory for storing Redpanda's streaming data files. CAUTION: Enabling `developer_mode` isn't recommended for production use. -Flag to enable developer mode, which skips most of the checks performed at startup. - -*Optional:* Yes +Enable developer mode, which skips most of the checks performed at startup. *Visibility:* `tunable` @@ -109,8 +97,6 @@ Flag to enable developer mode, which skips most of the checks performed at start Override the cluster property xref:reference:properties/cluster-properties.adoc#data_transforms_enabled[`data_transforms_enabled`] and disable Wasm-powered data transforms. This is an emergency shutoff button. -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -127,8 +113,6 @@ Controls how a new cluster is formed. All brokers in a cluster must have the sam TIP: For backward compatibility, `true` is the default. Redpanda recommends using `false` in production environments to prevent accidental cluster formation. -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -147,8 +131,6 @@ Controls whether Redpanda starts in FIPS mode. This property allows for three v * Enabled - Redpanda verifies that the operating system is enabled for FIPS by checking `/proc/sys/crypto/fips_enabled`. If the file does not exist or does not return `1`, Redpanda immediately exits. -*Optional:* No - *Visibility:* `user` *Accepted values:* `0` (disabled), `1` (permissive), `2` (enabled) @@ -161,8 +143,6 @@ Controls whether Redpanda starts in FIPS mode. This property allows for three v IP address and port of the Kafka API endpoint that handles requests. -*Optional:* Yes - *Visibility:* `user` *Default:* `127.0.0.1:9092` @@ -173,8 +153,6 @@ IP address and port of the Kafka API endpoint that handles requests. Transport Layer Security (TLS) configuration for the Kafka API endpoint. -*Optional:* Yes - *Visibility:* `user` *Default:* `null` @@ -186,9 +164,6 @@ Transport Layer Security (TLS) configuration for the Kafka API endpoint. Threshold for log messages that contain a larger memory allocation than specified. *Unit:* bytes - -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -203,13 +178,10 @@ A number that uniquely identifies the broker within the cluster. If `null` (the CAUTION: The `node_id` property must not be changed after a broker joins the cluster. - *Accepted values:* [`0`, `4294967295`] *Type:* integer -*Optional:* No - *Visibility:* `user` *Default:* `null` @@ -220,8 +192,6 @@ CAUTION: The `node_id` property must not be changed after a broker joins the clu Path to the configuration file used by OpenSSL to properly load the FIPS-compliant module. -*Optional:* Yes - *Visibility:* `user` *Type:* string @@ -234,8 +204,6 @@ Path to the configuration file used by OpenSSL to properly load the FIPS-complia Path to the directory that contains the OpenSSL FIPS-compliant module. The filename that Redpanda looks for is `fips.so`. -*Optional:* Yes - *Visibility:* `user` *Type:* string @@ -248,8 +216,6 @@ Path to the directory that contains the OpenSSL FIPS-compliant module. The filen A label that identifies a failure zone. Apply the same label to all brokers in the same failure zone. When xref:./cluster-properties.adoc#enable_rack_awareness[enable_rack_awareness] is set to `true` at the cluster level, the system uses the rack labels to spread partition replicas across different failure zones. -*Optional:* No - *Visibility:* `user` *Default:* `null` @@ -260,8 +226,6 @@ A label that identifies a failure zone. Apply the same label to all brokers in t If `true`, start Redpanda in xref:manage:recovery-mode.adoc[recovery mode], where user partitions are not loaded and only administrative operations are allowed. -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -274,8 +238,6 @@ If `true`, start Redpanda in xref:manage:recovery-mode.adoc[recovery mode], wher IP address and port for the Remote Procedure Call (RPC) server. -*Optional:* Yes - *Visibility:* `user` *Default:* `127.0.0.1:33145` @@ -286,8 +248,6 @@ IP address and port for the Remote Procedure Call (RPC) server. TLS configuration for the RPC server. -*Optional:* Yes - *Visibility:* `user` --- @@ -310,8 +270,6 @@ Only one broker, the designated cluster root, should have an empty `seed_servers The `seed_servers` list must be consistent across all seed brokers to prevent cluster fragmentation and ensure stable cluster formation. ==== -*Optional:* Yes - *Visibility:* `user` *Type:* array @@ -324,8 +282,6 @@ The `seed_servers` list must be consistent across all seed brokers to prevent cl Path to the configuration file used for low level storage failure injection. -*Optional:* No - *Visibility:* `tunable` *Type:* string @@ -338,8 +294,6 @@ Path to the configuration file used for low level storage failure injection. If `true`, inject low level storage failures on the write path. Do _not_ use for production instances. -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -352,8 +306,6 @@ If `true`, inject low level storage failures on the write path. Do _not_ use for Whether to violate safety checks when starting a Redpanda version newer than the cluster's consensus version. -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -366,9 +318,7 @@ Whether to violate safety checks when starting a Redpanda version newer than the Maximum duration in seconds for verbose (`TRACE` or `DEBUG`) logging. Values configured above this will be clamped. If null (the default) there is no limit. Can be overridden in the Admin API on a per-request basis. -*Units:* seconds - -*Optional:* No +*Unit:* seconds *Visibility:* `tunable` @@ -388,15 +338,37 @@ The Schema Registry provides configuration properties to help you enable produce For information on how to edit broker properties for the Schema Registry, see xref:manage:cluster-maintenance/node-property-configuration.adoc[]. -=== schema_registry_api +=== api_doc_dir -Specifies the listener address and port in the Schema Registry API. +API doc directory. -*Optional:* Yes +*Visibility:* `user` -*Visibility:* `None` +*Type:* string -*Default:* `127.0.0.1:8081` +*Default:* `/usr/share/redpanda/proxy-api-doc` + +--- + +=== mode_mutability + +Enable modifications to the read-only `mode` of the Schema Registry. When set to `true`, the entire Schema Registry or its subjects can be switched to `READONLY` or `READWRITE`. This property is useful for preventing unwanted changes to the entire Schema Registry or specific subjects. + +*Visibility:* `user` + +*Type:* boolean + +*Default:* `true` + +--- + +=== schema_registry_api + +Schema Registry API listener address and port. + +*Visibility:* `user` + +*Default:* `0.0.0.0:8081` --- @@ -404,9 +376,7 @@ Specifies the listener address and port in the Schema Registry API. TLS configuration for Schema Registry API. -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Default:* `null` @@ -416,9 +386,7 @@ TLS configuration for Schema Registry API. Replication factor for internal `_schemas` topic. If unset, defaults to `default_topic_replication`. -*Optional:* No - -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -443,39 +411,9 @@ See xref:develop:http-proxy.adoc[] Network address for the HTTP Proxy API server to publish to clients. -*Optional:* Yes - -*Visibility:* `None` - -*Default:* `null` - ---- - -=== api_doc_dir - -Path to the API specifications for the HTTP Proxy API. - -*Optional:* Yes - -*Visibility:* `None` - -*Type:* string - -*Default:* `/usr/share/redpanda/proxy-api-doc` - ---- - -=== mode_mutability - -Enable modifications to the read-only `mode` of the Schema Registry. When set to `true`, the entire Schema Registry or its subjects can be switched to `READONLY` or `READWRITE`. This property is useful for preventing unwanted changes to the entire Schema Registry or specific subjects. - -*Nullable:* No - *Visibility:* `user` -*Type:* boolean - -*Default:* `true` +*Default:* `null` --- @@ -483,9 +421,7 @@ Enable modifications to the read-only `mode` of the Schema Registry. When set to The maximum number of Kafka client connections that Redpanda can cache in the LRU (least recently used) cache. The LRU cache helps optimize resource utilization by keeping the most recently used clients in memory, facilitating quicker reconnections for frequent clients while limiting memory usage. -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -497,47 +433,41 @@ The maximum number of Kafka client connections that Redpanda can cache in the LR Time, in milliseconds, that an idle client connection may remain open to the HTTP Proxy API. -*Units* : milliseconds +*Unit:* milliseconds -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* integer *Accepted values:* [`-17592186044416`, `17592186044415`] -*Default:* `300000` +*Default:* `300000` (5min) --- -=== consumer_instance_timeout +=== consumer_instance_timeout_ms How long to wait for an idle consumer before removing it. A consumer is considered idle when it's not making requests or heartbeats. -*Units*: milliseconds +*Unit:* milliseconds -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* integer *Accepted values:* [`-17592186044416`, `17592186044415`] -*Default:* `300000` (5min) +*Default:* `300000` --- === pandaproxy_api -Specifies the listener address and port for the Rest API. - -*Optional:* Yes +Rest API listener address and port. -*Visibility:* `None` +*Visibility:* `user` -*Default:* `127.0.0.1:8082` +*Default:* `0.0.0.0:8082` --- @@ -545,9 +475,7 @@ Specifies the listener address and port for the Rest API. TLS configuration for Pandaproxy api. -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Default:* `null` @@ -561,11 +489,7 @@ Configuration options for HTTP Proxy Client. TLS configuration for the Kafka API servers to which the HTTP Proxy client should connect. -*Optional:* Yes - -*Visibility:* `None` - -*Default:* `config::tls_config()` +*Visibility:* `user` --- @@ -573,13 +497,11 @@ TLS configuration for the Kafka API servers to which the HTTP Proxy client shoul Network addresses of the Kafka API servers to which the HTTP Proxy client should connect. -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* array -*Default:* `["127.0.0.1:9092"]` +*Default:* `['127.0.0.1:9092']` --- @@ -587,9 +509,7 @@ Network addresses of the Kafka API servers to which the HTTP Proxy client should Custom identifier to include in the Kafka request header for the HTTP Proxy client. This identifier can help debug or monitor client activities. -*Optional:* No - -*Visibility:* `None` +*Visibility:* `user` *Type:* string @@ -597,15 +517,13 @@ Custom identifier to include in the Kafka request header for the HTTP Proxy clie --- -=== consumer_heartbeat_interval +=== consumer_heartbeat_interval_ms Interval (in milliseconds) for consumer heartbeats. -*Units*: milliseconds +*Unit:* milliseconds -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -615,21 +533,19 @@ Interval (in milliseconds) for consumer heartbeats. --- -=== consumer_rebalance_timeout +=== consumer_rebalance_timeout_ms Timeout (in milliseconds) for consumer rebalance. -*Units*: milliseconds +*Unit:* milliseconds -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* integer *Accepted values:* [`-17592186044416`, `17592186044415`] -*Default:* `200` +*Default:* `2000` --- @@ -637,11 +553,9 @@ Timeout (in milliseconds) for consumer rebalance. Maximum bytes to fetch per request. -*Units*: bytes - -*Optional:* Yes +*Unit:* bytes -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -655,11 +569,9 @@ Maximum bytes to fetch per request. Minimum bytes to fetch per request. -*Units*: bytes - -*Optional:* Yes +*Unit:* bytes -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -669,15 +581,13 @@ Minimum bytes to fetch per request. --- -=== consumer_request_timeout +=== consumer_request_timeout_ms Interval (in milliseconds) for consumer request timeout. -*Units*: milliseconds - -*Optional:* Yes +*Unit:* milliseconds -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -687,15 +597,13 @@ Interval (in milliseconds) for consumer request timeout. --- -=== consumer_session_timeout +=== consumer_session_timeout_ms Timeout (in milliseconds) for consumer session. -*Units*: milliseconds - -*Optional:* Yes +*Unit:* milliseconds -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -709,9 +617,7 @@ Timeout (in milliseconds) for consumer session. Number of acknowledgments the producer requires the leader to have received before considering a request complete. -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -721,15 +627,13 @@ Number of acknowledgments the producer requires the leader to have received befo --- -=== produce_batch_delay +=== produce_batch_delay_ms Delay (in milliseconds) to wait before sending batch. -*Units*: milliseconds +*Unit:* milliseconds -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -743,9 +647,7 @@ Delay (in milliseconds) to wait before sending batch. Number of records to batch before sending to broker. -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -759,11 +661,9 @@ Number of records to batch before sending to broker. Number of bytes to batch before sending to broker. -*Units*: bytes - -*Optional:* Yes +*Unit:* bytes -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -777,27 +677,21 @@ Number of bytes to batch before sending to broker. Enable or disable compression by the Kafka client. Specify `none` to disable compression or one of the supported types [gzip, snappy, lz4, zstd]. -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* string -*Accepted values:* `gzip`, `snappy`, `lz4`, `zstd` - *Default:* `none` --- -=== produce_shutdown_delay +=== produce_shutdown_delay_ms Delay (in milliseconds) to allow for final flush of buffers before shutting down. -*Units*: milliseconds - -*Optional:* Yes +*Unit:* milliseconds -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -811,9 +705,7 @@ Delay (in milliseconds) to allow for final flush of buffers before shutting down Number of times to retry a request to a broker. -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -821,15 +713,13 @@ Number of times to retry a request to a broker. --- -=== retry_base_backoff +=== retry_base_backoff_ms Delay (in milliseconds) for initial retry backoff. -*Units*: milliseconds - -*Optional:* Yes +*Unit:* milliseconds -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -843,9 +733,7 @@ Delay (in milliseconds) for initial retry backoff. The SASL mechanism to use when connecting. -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* string @@ -857,9 +745,7 @@ The SASL mechanism to use when connecting. Password to use for SCRAM authentication mechanisms. -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* string @@ -871,12 +757,10 @@ Password to use for SCRAM authentication mechanisms. Username to use for SCRAM authentication mechanisms. -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* string *Default:* `null` ---- +--- \ No newline at end of file diff --git a/modules/reference/pages/properties/cluster-properties.adoc b/modules/reference/pages/properties/cluster-properties.adoc index 48a128b420..25642b3336 100644 --- a/modules/reference/pages/properties/cluster-properties.adoc +++ b/modules/reference/pages/properties/cluster-properties.adoc @@ -14,11 +14,9 @@ NOTE: Some cluster properties require that you restart the cluster for any updat Capacity (in number of txns) of an abort index segment. -Each partition tracks the aborted transaction offset ranges to help service client requests. If the number transactions increase beyond this threshold, they are flushed to disk to easy memory pressure. Then they're loaded on demand. This configuration controls the maximum number of aborted transactions before they are flushed to disk. +Each partition tracks the aborted transaction offset ranges to help service client requests. If the number of transactions increases beyond this threshold, they are flushed to disk to ease memory pressure. Then they're loaded on demand. This configuration controls the maximum number of aborted transactions before they are flushed to disk. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -36,9 +34,7 @@ Interval, in milliseconds, at which Redpanda looks for inactive transactions and *Unit:* milliseconds -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -56,8 +52,6 @@ Whether Admin API clients must provide HTTP basic authentication headers. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -72,9 +66,7 @@ Enable aggregation of metrics returned by the xref:reference:internal-metrics-re *Requires restart:* No -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* boolean @@ -90,8 +82,6 @@ The amount of time since the last broker status heartbeat. After this time, a br *Requires restart:* No -*Nullable:* No - *Visibility:* `tunable` *Type:* integer @@ -110,8 +100,6 @@ The duration, in milliseconds, that Redpanda waits for the replication of entrie *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -126,9 +114,7 @@ The duration, in milliseconds, that Redpanda waits for the replication of entrie Size of direct write operations to disk in bytes. A larger chunk size can improve performance for write-heavy workloads, but increase latency for these writes as more data is collected before each write operation. A smaller chunk size can decrease write latency, but potentially increase the number of disk I/O operations. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -144,8 +130,6 @@ Defines the number of bytes allocated by the internal audit client for audit mes *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -160,8 +144,6 @@ Enables or disables audit logging. When you set this to true, Redpanda checks fo *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -176,8 +158,6 @@ List of strings in JSON style identifying the event types to include in the audi *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* array @@ -192,8 +172,6 @@ List of user principals to exclude from auditing. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* array @@ -208,8 +186,6 @@ List of topics to exclude from auditing. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* array @@ -222,9 +198,9 @@ List of topics to exclude from auditing. Defines the number of partitions used by a newly-created audit topic. This configuration applies only to the audit log topic and may be different from the cluster or other topic configurations. This cannot be altered for existing audit log topics. -*Requires restart:* No +*Unit:* number of partitions per topic -*Optional:* Yes +*Requires restart:* No *Visibility:* `user` @@ -242,8 +218,6 @@ Defines the replication factor for a newly-created audit log topic. This configu *Requires restart:* No -*Optional:* No - *Visibility:* `user` *Type:* integer @@ -262,8 +236,6 @@ Interval, in milliseconds, at which Redpanda flushes the queued audit log messag *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -280,8 +252,6 @@ Defines the maximum amount of memory in bytes used by the audit buffer in each s *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -298,8 +268,6 @@ If you produce to a topic that doesn't exist, the topic will be created with def *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -314,9 +282,7 @@ Cluster identifier. *Requires restart:* No -*Optional:* No - -*Visibility:* `None` +*Visibility:* `user` *Type:* string @@ -330,8 +296,6 @@ Size (in bytes) for each compacted log segment. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -344,11 +308,9 @@ Size (in bytes) for each compacted log segment. === compaction_ctrl_backlog_size -Target backlog size for compaction controller. If not set the max backlog size is configured to 80% of total disk space available. +Target backlog size for compaction controller. If not set the max backlog size is configured to 80% of total disk space available. -*Requires restart:* No - -*Optional:* No +*Requires restart:* Yes *Visibility:* `tunable` @@ -362,9 +324,7 @@ Target backlog size for compaction controller. If not set the max backlog size i Derivative coefficient for compaction PID controller. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -378,9 +338,7 @@ Derivative coefficient for compaction PID controller. Integral coefficient for compaction PID controller. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -394,9 +352,7 @@ Integral coefficient for compaction PID controller. Maximum number of I/O and CPU shares that compaction process can use. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -412,9 +368,7 @@ Maximum number of I/O and CPU shares that compaction process can use. Minimum number of I/O and CPU shares that compaction process can use. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -430,9 +384,7 @@ Minimum number of I/O and CPU shares that compaction process can use. Proportional coefficient for compaction PID controller. This must be negative, because the compaction backlog should decrease when the number of compaction shares increases. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -450,8 +402,6 @@ Interval between iterations of controller backend housekeeping loop. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -468,8 +418,6 @@ Maximum capacity of rate limit accumulation in controller ACLs and users operati *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -484,8 +432,6 @@ Maximum capacity of rate limit accumulation in controller configuration operatio *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -500,8 +446,6 @@ Maximum capacity of rate limit accumulation in controller move operations limit. *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -516,8 +460,6 @@ Maximum capacity of rate limit accumulation in controller node management operat *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -532,8 +474,6 @@ Maximum capacity of rate limit accumulation in controller topic operations limit *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -546,12 +486,10 @@ Maximum capacity of rate limit accumulation in controller topic operations limit Maximum amount of time before Redpanda attempts to create a controller snapshot after a new controller command appears. -*Unit*: seconds +*Unit:* seconds *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -568,8 +506,6 @@ If set to `true`, move partitions between cores in runtime to maintain balanced *Requires restart:* No -*Optional:* No - *Visibility:* `user` *Type:* boolean @@ -586,15 +522,13 @@ Interval, in milliseconds, between trigger and invocation of core balancing. *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer *Accepted values:* [`-17592186044416`, `17592186044415`] -*Default:* `10000` +*Default:* `10000` (10s) --- @@ -604,8 +538,6 @@ If set to `true`, and if after a restart the number of cores changes, Redpanda w *Requires restart:* No -*Optional:* No - *Visibility:* `user` *Type:* boolean @@ -620,8 +552,6 @@ Enables CPU profiling for Redpanda. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -638,8 +568,6 @@ The sample period for the CPU profiler. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -658,8 +586,6 @@ Timeout, in milliseconds, to wait for new topic creation. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -676,8 +602,6 @@ The maximum size for a deployable WebAssembly binary that the broker can store. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -694,8 +618,6 @@ The commit interval at which data transforms progress. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -712,8 +634,6 @@ Enables WebAssembly-powered data transforms directly in the broker. When `data_t *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -726,9 +646,9 @@ Enables WebAssembly-powered data transforms directly in the broker. When `data_t Buffer capacity for transform logs, per shard. Buffer occupancy is calculated as the total size of buffered log messages; that is, logs emitted but not yet produced. -*Requires restart:* Yes +*Unit:* bytes -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -746,8 +666,6 @@ Flush interval for transform logs. When a timer expires, pending logs are collec *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -766,8 +684,6 @@ Transform log lines truncate to this length. Truncation occurs after any charact *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -782,8 +698,6 @@ The amount of memory to reserve per core for data transform (Wasm) virtual machi *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -798,8 +712,6 @@ The amount of memory to give an instance of a data transform (Wasm) virtual mach *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -816,8 +728,6 @@ The percentage of available memory in the transform subsystem to use for read bu *Requires restart:* Yes -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -834,8 +744,6 @@ The maximum amount of runtime to start up a data transform, and the time it take *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -854,8 +762,6 @@ The percentage of available memory in the transform subsystem to use for write b *Requires restart:* Yes -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -864,14 +770,62 @@ The percentage of available memory in the transform subsystem to use for write b --- +=== datalake_coordinator_snapshot_max_delay_secs + +Maximum amount of time the coordinator waits to snapshot after a command appears in the log. + +*Unit*: seconds + +*Requires restart:* No + +*Visibility:* `tunable` + +*Type:* integer + +*Accepted values:* [`-17179869184`, `17179869183`] + +*Default:* `900` + +--- + +=== debug_bundle_auto_removal_seconds + +If set, how long debug bundles are kept in the debug bundle storage directory after they are created. If not set, debug bundles are kept indefinitely. + +*Unit:* seconds + +*Requires restart:* No + +*Visibility:* `user` + +*Type:* integer + +*Accepted values:* [`-17179869184`, `17179869183`] + +*Default:* `null` + +--- + +=== debug_bundle_storage_dir + +Path to the debug bundle storage directory. Note: Changing this path does not clean up existing debug bundles. If not set, the debug bundle is stored in the Redpanda data directory specified in the redpanda.yaml broker configuration file. + +*Requires restart:* No + +*Visibility:* `user` + +*Type:* string + +*Default:* `null` + +--- + === debug_load_slice_warning_depth The recursion depth after which debug logging is enabled automatically for the log reader. *Requires restart:* No -*Nullable:* Yes - *Visibility:* `tunable` *Type:* integer @@ -882,14 +836,32 @@ The recursion depth after which debug logging is enabled automatically for the l --- +=== default_leaders_preference + +Default settings for preferred location of topic partition leaders. It can be either "none" (no preference), or "racks:,,..." (prefer brokers with rack ID from the list). + +The list can contain one or more rack IDs. If you specify multiple IDs, Redpanda tries to distribute the partition leader locations equally across brokers in these racks. + +If config_ref:enable_rack_awareness,true,properties/cluster-properties[] is set to `false`, leader pinning is disabled across the cluster. + +*Requires restart:* No + +*Visibility:* `user` + +*Default:* `none` + +*Related topics*: + +- xref:develop:produce-data/leader-pinning.adoc[Leader pinning] + +--- + === default_num_windows Default number of quota tracking windows. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -904,12 +876,10 @@ Default number of quota tracking windows. Default number of partitions per topic. -*Unit*: number of partitions per topic +*Unit:* number of partitions per topic *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -926,8 +896,6 @@ Default replication factor for new topics. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -948,8 +916,6 @@ Default quota tracking window size in milliseconds. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -960,13 +926,25 @@ Default quota tracking window size in milliseconds. --- -=== disable_batch_cache +=== development_enable_cloud_topics -Disable batch cache in log manager. +Enable cloud topics. *Requires restart:* No -*Optional:* Yes +*Visibility:* `user` + +*Type:* boolean + +*Default:* `false` + +--- + +=== disable_batch_cache + +Disable batch cache in log manager. + +*Requires restart:* Yes *Visibility:* `tunable` @@ -978,12 +956,12 @@ Disable batch cache in log manager. === disable_cluster_recovery_loop_for_tests -Disables the cluster recovery loop. This property is used to simplify testing and should not be set in production. +include::reference:partial$internal-use-property.adoc[] + +Disables the cluster recovery loop. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -996,11 +974,9 @@ Disables the cluster recovery loop. This property is used to simplify testing an Disable registering the metrics exposed on the internal `/metrics` endpoint. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes -*Visibility:* `None` +*Visibility:* `user` *Type:* boolean @@ -1012,11 +988,9 @@ Disable registering the metrics exposed on the internal `/metrics` endpoint. Disable registering the metrics exposed on the `/public_metrics` endpoint. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes -*Visibility:* `None` +*Visibility:* `user` *Type:* boolean @@ -1035,8 +1009,6 @@ It is recommended to not run disks near capacity to avoid blocking I/O due to lo *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* number @@ -1051,8 +1023,6 @@ Enables cluster metadata uploads. Required for xref:manage:whole-cluster-restore *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -1067,8 +1037,6 @@ Limits the write rate for the controller log. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -1081,9 +1049,7 @@ Limits the write rate for the controller log. Enable idempotent producers. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `user` @@ -1095,12 +1061,10 @@ Enable idempotent producers. === enable_leader_balancer -Enable automatic leadership rebalancing. Mode is set by <>. +Enable automatic leadership rebalancing. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -1123,8 +1087,6 @@ The cluster metrics of the metrics reporter are different from xref:manage:monit *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -1139,8 +1101,6 @@ Enable Redpanda extensions for MPX. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -1153,9 +1113,7 @@ Enable Redpanda extensions for MPX. Enable PID file. You should not need to change. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -1171,8 +1129,6 @@ Enable rack-aware replica assignment. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -1187,8 +1143,6 @@ Enable SASL authentication for Kafka connections. Authorization is required to m *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -1207,8 +1161,6 @@ Mode to enable server-side schema ID validation. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Accepted Values*: @@ -1225,9 +1177,7 @@ Mode to enable server-side schema ID validation. Enable transactions (atomic writes). -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `user` @@ -1243,8 +1193,6 @@ Enables the usage tracking mechanism, storing windowed history of kafka/cloud_st *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -1259,8 +1207,6 @@ Whether new feature flags auto-activate after upgrades (true) or must wait for m *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -1273,12 +1219,10 @@ Whether new feature flags auto-activate after upgrades (true) or must wait for m Maximum number of bytes returned in a fetch request. -*Unit*: bytes +*Unit:* bytes *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -1287,6 +1231,82 @@ Maximum number of bytes returned in a fetch request. --- +=== fetch_pid_d_coeff + +Derivative coefficient for fetch PID controller. + +*Requires restart:* No + +*Visibility:* `tunable` + +*Type:* number + +*Default:* `0.0` + +--- + +=== fetch_pid_i_coeff + +Integral coefficient for fetch PID controller. + +*Requires restart:* No + +*Visibility:* `tunable` + +*Type:* number + +*Default:* `0.01` + +--- + +=== fetch_pid_max_debounce_ms + +The maximum debounce time the fetch PID controller will apply, in milliseconds. + +*Unit:* milliseconds + +*Requires restart:* No + +*Visibility:* `tunable` + +*Type:* integer + +*Accepted values:* [`-17592186044416`, `17592186044415`] + +*Default:* `100` + +--- + +=== fetch_pid_p_coeff + +Proportional coefficient for fetch PID controller. + +*Requires restart:* No + +*Visibility:* `tunable` + +*Type:* number + +*Default:* `100.0` + +--- + +=== fetch_pid_target_utilization_fraction + +A fraction, between 0 and 1, for the target reactor utilization of the fetch scheduling group. + +*Unit:* fraction + +*Requires restart:* No + +*Visibility:* `tunable` + +*Type:* number + +*Default:* `0.2` + +--- + === fetch_read_strategy The strategy used to fulfill fetch requests. @@ -1299,8 +1319,6 @@ The strategy used to fulfill fetch requests. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Accepted Values:* `polling`, `non_polling`, `non_polling_with_debounce` @@ -1317,8 +1335,6 @@ Time to wait for the next read in fetch requests when the requested minimum byte *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1335,9 +1351,7 @@ Time duration after which the inactive fetch session is removed from the fetch s *Unit:* milliseconds -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -1357,8 +1371,6 @@ Delay added to the rebalance phase to wait for new members. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1373,13 +1385,11 @@ Delay added to the rebalance phase to wait for new members. The maximum allowed session timeout for registered consumers. Longer timeouts give consumers more time to process messages in between heartbeats at the cost of a longer time to detect failures. -*Unit*: milliseconds +*Unit:* milliseconds *Requires restart:* No -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -1393,13 +1403,11 @@ The maximum allowed session timeout for registered consumers. Longer timeouts gi The minimum allowed session timeout for registered consumers. Shorter timeouts result in quicker failure detection at the cost of more frequent consumer heartbeating, which can overwhelm broker resources. -*Unit*: milliseconds +*Unit:* milliseconds *Requires restart:* No -*Optional:* Yes - -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -1417,8 +1425,6 @@ Timeout for new member joins. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1437,8 +1443,6 @@ Frequency rate at which the system should check for expired group offsets. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1457,8 +1461,6 @@ Consumer group offset retention seconds. To disable offset retention, set this t *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -1473,9 +1475,9 @@ Consumer group offset retention seconds. To disable offset retention, set this t Number of partitions in the internal group membership topic. -*Requires restart:* No +*Unit:* number of partitions per topic -*Optional:* Yes +*Requires restart:* No *Visibility:* `tunable` @@ -1492,10 +1494,7 @@ Number of partitions in the internal group membership topic. How often the health manager runs. *Unit:* milliseconds - -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -1515,8 +1514,6 @@ Maximum age of the metadata cached in the health monitor of a non-controller bro *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1533,8 +1530,6 @@ A list of supported HTTP authentication mechanisms. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* array @@ -1545,13 +1540,201 @@ A list of supported HTTP authentication mechanisms. --- -=== id_allocator_batch_size +=== iceberg_catalog_base_location -The ID allocator allocates messages in batches (each batch is a one log record) and then serves requests from memory without touching the log until the batch is exhausted. +Base path for the object storage backed Iceberg catalog. After Iceberg is enabled, do not change this value. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* string + +*Default:* `redpanda-iceberg-catalog` + +--- + +=== iceberg_catalog_commit_interval_ms + +The frequency at which the Iceberg coordinator commits topic files to the catalog. This is the interval between commit transactions across all topics monitored by the coordinator, not the interval between individual commits. + +*Unit:* milliseconds + +*Requires restart:* No + +*Visibility:* `tunable` + +*Type:* integer + +*Accepted values:* [`-17592186044416`, `17592186044415`] + +*Default:* `60000` + +--- + +=== iceberg_catalog_type + +Iceberg catalog type that Redpanda will use to commit table metadata updates. Supported types: 'rest', 'object_storage'. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Accepted values:* `rest`, `object_storage` + +*Default:* `object_storage` + +--- + +=== iceberg_delete + +Default value for the `redpanda.iceberg.delete` topic property that determines if the corresponding Iceberg table is deleted upon deleting the topic. + +*Requires restart:* No + +*Visibility:* `user` + +*Type:* boolean + +*Default:* `true` + +--- + +=== iceberg_enabled + +Enables the translation of topic data into Iceberg tables. Setting `iceberg_enabled` to `true` activates the feature at the cluster level, but each topic must also set the `redpanda.iceberg.enabled` topic-level property to `true` to use it. If `iceberg_enabled` is set to `false`, then the feature is disabled for all topics in the cluster, overriding any topic-level settings. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* boolean + +*Default:* `false` + +--- + +=== iceberg_rest_catalog_client_id + +Iceberg REST catalog user ID. This ID is used to query the catalog API for the OAuth token. Required if catalog type is set to `rest`. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* string + +*Default:* `null` + +--- + +=== iceberg_rest_catalog_client_secret + +Secret to authenticate against Iceberg REST catalog. Required if catalog type is set to `rest`. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* string + +*Default:* `null` + +--- + +=== iceberg_rest_catalog_crl_file + +Path to certificate revocation list for `iceberg_rest_catalog_trust_file`. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* string + +*Default:* `null` + +--- + +=== iceberg_rest_catalog_endpoint + +URL of Iceberg REST catalog endpoint. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* string + +*Default:* `null` + +--- + +=== iceberg_rest_catalog_prefix + +Prefix part of the Iceberg REST catalog URL. Prefix is appended to the catalog path, for example `/v1/\{prefix}/namespaces`. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* string + +*Default:* `null` + +--- + +=== iceberg_rest_catalog_request_timeout_ms + +Maximum length of time that Redpanda waits for a response from the REST catalog before aborting the request. + +*Unit:* milliseconds *Requires restart:* No -*Optional:* Yes +*Visibility:* `tunable` + +*Type:* integer + +*Accepted values:* [`-17592186044416`, `17592186044415`] + +*Default:* `10000` + +--- + +=== iceberg_rest_catalog_token + +Token used to access the REST Iceberg catalog. If the token is present, Redpanda ignores credentials stored in the properties <> and <>. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* string + +*Default:* `null` + +--- + +=== iceberg_rest_catalog_trust_file + +Path to a file containing a certificate chain to trust for the REST Iceberg catalog. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* string + +*Default:* `null` + +--- + +=== id_allocator_batch_size + +The ID allocator allocates messages in batches (each batch is a one log record) and then serves requests from memory without touching the log until the batch is exhausted. + +*Requires restart:* Yes *Visibility:* `tunable` @@ -1567,9 +1750,7 @@ The ID allocator allocates messages in batches (each batch is a one log record) Capacity of the `id_allocator` log in number of batches. After it reaches `id_allocator_stm`, it truncates the log's prefix. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -1589,8 +1770,6 @@ Initial local retention size target for partitions of topics with xref:manage:ti *Requires restart:* No -*Optional:* No - *Visibility:* `user` *Type:* integer @@ -1607,8 +1786,6 @@ Initial local retention time target for partitions of topics with xref:manage:ti *Requires restart:* No -*Optional:* No - *Visibility:* `user` *Type:* integer @@ -1624,10 +1801,7 @@ Initial local retention time target for partitions of topics with xref:manage:ti Target replication factor for internal topics. *Unit*: number of replicas per topic. - -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `user` @@ -1645,17 +1819,31 @@ Time between cluster join retries in milliseconds. *Unit:* milliseconds +*Requires restart:* Yes + +*Visibility:* `tunable` + +*Type:* integer + +*Accepted values:* [`-17592186044416`, `17592186044415`] + +*Default:* `5000` + +--- + +=== kafka_admin_topic_api_rate + +Target quota rate (partition mutations per default_window_sec). + *Requires restart:* No -*Optional:* Yes - -*Visibility:* `tunable` +*Visibility:* `user` *Type:* integer -*Accepted values:* [`-17592186044416`, `17592186044415`] +*Accepted values:* [`0`, `4294967295`] -*Default:* `5000` +*Default:* `null` --- @@ -1663,9 +1851,9 @@ Time between cluster join retries in milliseconds. Maximum size of a batch processed by the server. If the batch is compressed, the limit applies to the compressed batch size. -*Requires restart:* No +*Unit:* bytes -*Optional:* Yes +*Requires restart:* No *Visibility:* `tunable` @@ -1677,19 +1865,35 @@ Maximum size of a batch processed by the server. If the batch is compressed, the --- -=== kafka_connection_rate_limit +=== kafka_client_group_byte_rate_quota -Maximum connections per second for one core. If `null` (the default), then the number of connections per second is unlimited. +Per-group target produce quota byte rate (bytes per second). Client is considered part of the group if client_id contains clients_prefix. -*Unit*: number of connections per second, per core +*Requires restart:* No -*Related topics*: +*Visibility:* `user` -* xref:manage:cluster-maintenance/configure-availability.adoc#limit-client-connections[Limit client connections] +*Default:* `null` + +--- + +=== kafka_client_group_fetch_byte_rate_quota + +Per-group target fetch quota byte rate (bytes per second). Client is considered part of the group if client_id contains clients_prefix. *Requires restart:* No -*Optional:* No +*Visibility:* `user` + +*Default:* `null` + +--- + +=== kafka_connection_rate_limit + +Maximum connections per second for one core. If `null` (the default), then the number of connections per second is unlimited. + +*Requires restart:* No *Visibility:* `user` @@ -1711,8 +1915,6 @@ Overrides the maximum connections per second for one core for the specified IP a *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* array @@ -1731,8 +1933,6 @@ Maximum number of Kafka client connections per broker. If `null`, the property i *Requires restart:* No -*Optional:* No - *Visibility:* `user` *Type:* integer @@ -1747,12 +1947,10 @@ Maximum number of Kafka client connections per broker. If `null`, the property i === kafka_connections_max_overrides -A list of IP addresses for which Kafka client connection limits are overridden and don't apply. For example, `(['127.0.0.1:90', '50.20.1.1:40']).` +A list of IP addresses for which Kafka client connection limits are overridden and don't apply. For example, `(['127.0.0.1:90', '50.20.1.1:40']).`. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* array @@ -1771,8 +1969,6 @@ Maximum number of Kafka client connections per IP address, per broker. If `null` *Requires restart:* No -*Optional:* No - *Visibility:* `user` *Type:* integer @@ -1793,8 +1989,6 @@ Flag to require authorization for Kafka connections. If `null`, the property is *Requires restart:* No -*Optional:* No - *Visibility:* `user` *Type:* boolean @@ -1820,8 +2014,6 @@ Whether to include Tiered Storage as a special remote:// directory in `DescribeL *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -1836,8 +2028,6 @@ Enable the Kafka partition reassignment API. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -1850,12 +2040,10 @@ Enable the Kafka partition reassignment API. Kafka group recovery timeout. -*Unit*: milliseconds +*Unit:* milliseconds *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -1872,8 +2060,6 @@ Limit fetch responses to this many bytes, even if the total of partition bytes l *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1888,8 +2074,6 @@ The size of the batch used to estimate memory consumption for fetch requests, in *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -1904,8 +2088,6 @@ The share of Kafka subsystem memory that can be used for fetch read buffers, as *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `user` *Type:* number @@ -1920,8 +2102,6 @@ Principal mapping rules for mTLS authentication on the Kafka API. If `null`, the *Requires restart:* No -*Optional:* No - *Visibility:* `user` *Type:* array @@ -1936,8 +2116,6 @@ A list of topics that are protected from deletion and configuration changes by K *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* string array @@ -1957,8 +2135,6 @@ A list of topics that are protected from being produced to by Kafka clients. Set *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* array @@ -1971,9 +2147,7 @@ A list of topics that are protected from being produced to by Kafka clients. Set Smoothing factor for Kafka queue depth control depth tracking. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -1989,9 +2163,7 @@ Update frequency for Kafka queue depth control. *Unit:* milliseconds -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -2009,8 +2181,6 @@ Enable Kafka queue depth control. *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -2023,9 +2193,7 @@ Enable Kafka queue depth control. Queue depth when idleness is detected in Kafka queue depth control. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -2039,9 +2207,7 @@ Queue depth when idleness is detected in Kafka queue depth control. Smoothing parameter for Kafka queue depth control latency tracking. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -2055,9 +2221,7 @@ Smoothing parameter for Kafka queue depth control latency tracking. Maximum queue depth used in Kafka queue depth control. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -2071,11 +2235,9 @@ Maximum queue depth used in Kafka queue depth control. Maximum latency threshold for Kafka queue depth control depth tracking. -*Unit*: milliseconds - -*Requires restart:* No +*Unit:* milliseconds -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `user` @@ -2091,9 +2253,7 @@ Maximum latency threshold for Kafka queue depth control depth tracking. Minimum queue depth used in Kafka queue depth control. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -2107,9 +2267,7 @@ Minimum queue depth used in Kafka queue depth control. Number of windows used in Kafka queue depth control latency tracking. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -2125,9 +2283,7 @@ Window size for Kafka queue depth control latency tracking. *Unit:* milliseconds -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -2158,8 +2314,6 @@ If set to `0`, no minimum is enforced. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -2190,8 +2344,6 @@ If set to `0.0`, the minimum is disabled. If set to `1.0`, the balancer won't be *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* number @@ -2204,7 +2356,7 @@ If set to `0.0`, the minimum is disabled. If set to `1.0`, the balancer won't be --- -=== kafka_quota_balancer_node_period +=== kafka_quota_balancer_node_period_ms Intra-node throughput quota balancer invocation period, in milliseconds. When set to 0, the balancer is disabled and makes all the throughput quotas immutable. @@ -2212,8 +2364,6 @@ Intra-node throughput quota balancer invocation period, in milliseconds. When se *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -2224,7 +2374,7 @@ Intra-node throughput quota balancer invocation period, in milliseconds. When se --- -=== kafka_quota_balancer_window +=== kafka_quota_balancer_window_ms Time window used to average current throughput measurement for quota balancer, in milliseconds. @@ -2232,8 +2382,6 @@ Time window used to average current throughput measurement for quota balancer, i *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -2248,9 +2396,9 @@ Time window used to average current throughput measurement for quota balancer, i Maximum size of a single request processed using the Kafka API. -*Requires restart:* No +*Unit:* bytes -*Optional:* Yes +*Requires restart:* No *Visibility:* `tunable` @@ -2266,9 +2414,9 @@ Maximum size of a single request processed using the Kafka API. Maximum size of the user-space receive buffer. If `null`, this limit is not applied. -*Requires restart:* No +*Unit:* bytes -*Optional:* No +*Requires restart:* Yes *Visibility:* `tunable` @@ -2282,17 +2430,15 @@ Maximum size of the user-space receive buffer. If `null`, this limit is not appl Size of the Kafka server TCP receive buffer. If `null`, the property is disabled. -*Unit*: bytes - -*Requires restart:* No +*Unit:* bytes -*Optional:* No +*Requires restart:* Yes -*Visibility:* `None` +*Visibility:* `user` *Type:* integer -*Accepted values:* [`-2147483648`, `2147483647`] aligned to 4096 bytes +*Accepted values:* [`-2147483648`, `2147483647`] *Default:* `null` @@ -2302,13 +2448,11 @@ Size of the Kafka server TCP receive buffer. If `null`, the property is disabled Size of the Kafka server TCP transmit buffer. If `null`, the property is disabled. -*Unit*: bytes - -*Requires restart:* No +*Unit:* bytes -*Optional:* No +*Requires restart:* Yes -*Visibility:* `None` +*Visibility:* `user` *Type:* integer @@ -2328,8 +2472,6 @@ IMPORTANT: If this property is not set (or set to `null`), session expiry is dis *Requires restart:* No -*Optional:* No - *Visibility:* `user` *Type:* integer @@ -2346,8 +2488,6 @@ Per-shard capacity of the cache for validating schema IDs. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -2356,7 +2496,7 @@ Per-shard capacity of the cache for validating schema IDs. --- -=== kafka_tcp_keepalive_idle_timeout_seconds +=== kafka_tcp_keepalive_timeout TCP keepalive idle timeout in seconds for Kafka connections. This describes the timeout between TCP keepalive probes that the remote site successfully acknowledged. Refers to the TCP_KEEPIDLE socket option. When changed, applies to new connections only. @@ -2364,8 +2504,6 @@ TCP keepalive idle timeout in seconds for Kafka connections. This describes the *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -2384,8 +2522,6 @@ TCP keepalive probe interval in seconds for Kafka connections. This describes th *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -2402,8 +2538,6 @@ TCP keepalive unacknowledged probes until the connection is considered dead for *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -2433,8 +2567,6 @@ A connection is assigned the first matching group and is then excluded from thro *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* string array @@ -2455,8 +2587,6 @@ List of Kafka API keys that are subject to cluster-wide and node-wide throughput *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* list @@ -2467,16 +2597,12 @@ List of Kafka API keys that are subject to cluster-wide and node-wide throughput === kafka_throughput_limit_node_in_bps -The maximum rate of all ingress Kafka API traffic for a node. Includes all Kafka API traffic (requests, responses, headers, fetched data, produced data, etc.). - -If `null`, the property is disabled, and traffic is not limited. +The maximum rate of all ingress Kafka API traffic for a node. Includes all Kafka API traffic (requests, responses, headers, fetched data, produced data, etc.). If `null`, the property is disabled, and traffic is not limited. -*Unit*: bytes per second +*Unit:* bytes per second *Requires restart:* No -*Optional:* No - *Visibility:* `user` *Type:* integer @@ -2493,16 +2619,12 @@ If `null`, the property is disabled, and traffic is not limited. === kafka_throughput_limit_node_out_bps -The maximum rate of all egress Kafka traffic for a node. Includes all Kafka API traffic (requests, responses, headers, fetched data, produced data, etc.). +The maximum rate of all egress Kafka traffic for a node. Includes all Kafka API traffic (requests, responses, headers, fetched data, produced data, etc.). If `null`, the property is disabled, and traffic is not limited. -If `null`, the property is disabled, and traffic is not limited. - -*Unit*: bytes per second +*Unit:* bytes per second *Requires restart:* No -*Optional:* No - *Visibility:* `user` *Type:* integer @@ -2525,8 +2647,6 @@ This threshold is evaluated with each request for data. When the number of token *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -2549,8 +2669,6 @@ WARNING: Disabling this property is not recommended. It causes your Redpanda clu *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -2567,8 +2685,6 @@ Key-value store flush interval (in milliseconds). *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -2583,9 +2699,7 @@ Key-value store flush interval (in milliseconds). Key-value maximum segment size (in bytes). -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -2603,8 +2717,6 @@ Leadership rebalancing idle timeout. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -2615,27 +2727,6 @@ Leadership rebalancing idle timeout. --- -=== leader_balancer_mode - -Mode of the leader balancer for optimizing movements of leadership between shards (logical CPU cores). Enabled by <>. - -*Requires restart:* No - -*Optional:* Yes - -*Visibility:* `user` - -*Type:* `string` - -*Accepted Values*: - -* `random_hill_climbing`: a shard is randomly chosen and leadership is moved to it if the load on the original shard is reduced. -* `greedy_balanced_shards`: leadership movement is based on a greedy heuristic of moving leaders from the most loaded shard to the least loaded shard. - -*Default:* `random_hill_climbing` - ---- - === leader_balancer_mute_timeout Leadership rebalancing mute timeout. @@ -2644,8 +2735,6 @@ Leadership rebalancing mute timeout. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -2656,7 +2745,7 @@ Leadership rebalancing mute timeout. --- -=== leader_balancer_node_mute_timeout +=== leader_balancer_mute_timeout Leadership rebalancing node mute timeout. @@ -2664,8 +2753,6 @@ Leadership rebalancing node mute timeout. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -2682,8 +2769,6 @@ Per shard limit for in-progress leadership transfers. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -2698,8 +2783,6 @@ Group offset retention is enabled by default starting in Redpanda version 23.1. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -2710,14 +2793,10 @@ Group offset retention is enabled by default starting in Redpanda version 23.1. === legacy_permit_unsafe_log_operation -Flag to enable a Redpanda cluster operator to use unsafe control characters within strings, such as consumer group names or user names. - -This flag applies only for Redpanda clusters that were originally on version 23.1 or earlier and have been upgraded to version 23.2 or later. Starting in version 23.2, newly-created Redpanda clusters ignore this property. +Flag to enable a Redpanda cluster operator to use unsafe control characters within strings, such as consumer group names or user names. This flag applies only for Redpanda clusters that were originally on version 23.1 or earlier and have been upgraded to version 23.2 or later. Starting in version 23.2, newly-created Redpanda clusters ignore this property. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* boolean @@ -2730,16 +2809,12 @@ This flag applies only for Redpanda clusters that were originally on version 23. === legacy_unsafe_log_warning_interval_sec -Period at which to log a warning about using unsafe strings containing control characters. - -If unsafe strings are permitted by <>, a warning will be logged at an interval specified by this property. +Period at which to log a warning about using unsafe strings containing control characters. If unsafe strings are permitted by `legacy_permit_unsafe_log_operation`, a warning will be logged at an interval specified by this property. -*Unit*: seconds +*Unit:* seconds *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -2760,8 +2835,6 @@ The topic property xref:./topic-properties.adoc#cleanuppolicy[`cleanup.policy`] *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Accepted Values*: `compact`, `delete`, `compact,delete` @@ -2774,12 +2847,10 @@ The topic property xref:./topic-properties.adoc#cleanuppolicy[`cleanup.policy`] How often to trigger background compaction. -*Unit*: milliseconds +*Unit:* milliseconds *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -2796,8 +2867,6 @@ Use sliding window compaction. *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -2814,8 +2883,6 @@ The topic property xref:./topic-properties.adoc#compressiontype[`compression.typ *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Accepted Values:* `gzip`, `snappy`, `lz4`, `zstd`, `producer`, `none`. @@ -2826,14 +2893,10 @@ The topic property xref:./topic-properties.adoc#compressiontype[`compression.typ === log_disable_housekeeping_for_tests -Disables the housekeeping loop for local storage. - -IMPORTANT: This property is used to simplify testing, and should not be set in production. +Disables the housekeeping loop for local storage. This property is used to simplify testing, and should not be set in production. *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -2850,8 +2913,6 @@ Threshold in milliseconds for alerting on messages with a timestamp after the br *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -2870,8 +2931,6 @@ Threshold in milliseconds for alerting on messages with a timestamp before the b *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -2890,8 +2949,6 @@ The topic property xref:./topic-properties.adoc#messagetimestamptype[`message.ti *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Accepted Values:* `CreateTime`, `LogAppendTime`. @@ -2904,12 +2961,10 @@ The topic property xref:./topic-properties.adoc#messagetimestamptype[`message.ti The amount of time to keep a log file before deleting it (in milliseconds). If set to `-1`, no time limit is applied. This is a cluster-wide default when a topic does not set or disable xref:./topic-properties.adoc#retentionms[`retention.ms`]. -*Unit*: milliseconds +*Unit:* milliseconds *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Accepted values:* [`-17592186044416`, `17592186044415`] @@ -2924,12 +2979,10 @@ Default lifetime of log segments. If `null`, the property is disabled, and no de The topic property xref:./topic-properties.adoc#segmentms[`segment.ms`] overrides the value of `log_segment_ms` at the topic level. -*Unit*: milliseconds +*Unit:* milliseconds *Requires restart:* No -*Optional:* No - *Visibility:* `user` *Type:* integer @@ -2953,8 +3006,6 @@ Upper bound on topic `segment.ms`: higher values will be clamped to this value. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -2973,8 +3024,6 @@ Lower bound on topic `segment.ms`: lower values will be clamped to this value. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -2991,8 +3040,6 @@ Default log segment size in bytes for topics which do not set `segment.bytes`. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -3007,9 +3054,9 @@ Default log segment size in bytes for topics which do not set `segment.bytes`. Random variation to the segment size limit used for each partition. -*Requires restart:* Yes +*Unit:* percent -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -3027,8 +3074,6 @@ Upper bound on topic `segment.bytes`: higher values will be clamped to this limi *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -3045,8 +3090,6 @@ Lower bound on topic `segment.bytes`: lower values will be clamped to this limit *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -3063,8 +3106,6 @@ Disable reusable preallocated buffers for LZ4 decompression. *Requires restart:* Yes -*Nullable:* No - *Visibility:* `tunable` *Type:* boolean @@ -3079,8 +3120,6 @@ Maximum compacted segment size after consolidation. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -3091,12 +3130,10 @@ Maximum compacted segment size after consolidation. === max_concurrent_producer_ids -Maximum number of the active producers sessions. When the threshold is passed, Redpanda terminates old sessions. When an idle producer corresponding to the terminated session wakes up and produces, its message batches are rejected, and an out of order sequence error is emitted. Consumers don't affect this setting. +Maximum number of active producer sessions. When the threshold is passed, Redpanda terminates old sessions. When an idle producer corresponding to the terminated session wakes up and produces, its message batches are rejected, and an out of order sequence error is emitted. Consumers don't affect this setting. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -3113,8 +3150,6 @@ Maximum number of in-flight HTTP requests to HTTP Proxy permitted per shard. An *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -3129,8 +3164,6 @@ Maximum number of in-flight HTTP requests to Schema Registry permitted per shard *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -3147,8 +3180,6 @@ Fail-safe maximum throttle delay on Kafka requests. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -3167,8 +3198,6 @@ For details, see xref:develop:transactions#transaction-usage-tips[Transaction us *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -3185,9 +3214,7 @@ Time between members backend reconciliation loop retries. *Unit:* milliseconds -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -3205,8 +3232,6 @@ If `true`, the Redpanda process will terminate immediately when an allocation ca *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -3219,11 +3244,9 @@ If `true`, the Redpanda process will terminate immediately when an allocation ca Interval for metadata dissemination batching. -*Unit*: milliseconds - -*Requires restart:* No +*Unit:* milliseconds -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -3237,13 +3260,9 @@ Interval for metadata dissemination batching. === metadata_dissemination_retries -Number of attempts to look up a topic's metadata-like shard before a request fails. +Number of attempts to look up a topic's metadata-like shard before a request fails. This configuration controls the number of retries that request handlers perform when internal topic metadata (for topics like tx, consumer offsets, etc) is missing. These topics are usually created on demand when users try to use the cluster for the first time and it may take some time for the creation to happen and the metadata to propagate to all the brokers (particularly the broker handling the request). In the meantime Redpanda waits and retries. This configuration controls the number retries. -This configuration controls the number of retries that request handlers perform when internal topic metadata (for topics like tx, consumer offsets, etc) is missing. These topics are usually created on demand when users try to use the cluster for the first time and it may take some time for the creation to happen and the metadata to propagate to all the brokers (particularly the broker handling the request). In the mean time Redpanda waits and retry. This configuration controls the number retries. - -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -3261,9 +3280,7 @@ Delay before retrying a topic lookup in a shard or other meta tables. *Unit:* milliseconds -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -3281,9 +3298,7 @@ Maximum time to wait in metadata request for cluster health to be refreshed. *Unit:* milliseconds -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -3300,11 +3315,8 @@ Maximum time to wait in metadata request for cluster health to be refreshed. Cluster metrics reporter report interval. *Unit:* milliseconds - *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -3323,8 +3335,6 @@ Cluster metrics reporter tick interval. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -3341,8 +3351,6 @@ URL of the cluster metrics reporter. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* string @@ -3361,8 +3369,6 @@ If you change the `minimum_topic_replications` setting, the replication factor o *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -3379,8 +3385,6 @@ How long after the last heartbeat request a node will wait before considering it *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -3397,9 +3401,7 @@ Timeout for executing node management operations. *Unit:* milliseconds -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -3416,11 +3418,8 @@ Timeout for executing node management operations. Time interval between two node status messages. Node status messages establish liveness status outside of the Raft protocol. *Unit:* milliseconds - *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -3439,8 +3438,6 @@ Maximum backoff (in milliseconds) to reconnect to an unresponsive peer during no *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -3459,8 +3456,6 @@ The amount of time (in seconds) to allow for when validating the expiry claim in *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -3477,8 +3472,6 @@ The URL pointing to the well-known discovery endpoint for the OIDC provider. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* string @@ -3495,8 +3488,6 @@ The frequency of refreshing the JSON Web Keys (JWKS) used to validate access tok *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -3513,8 +3504,6 @@ Rule for mapping JWT payload claim to a Redpanda user principal. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* string @@ -3529,8 +3518,6 @@ A string representing the intended recipient of the token. *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* string @@ -3545,8 +3532,6 @@ Number of partitions that can be reassigned at once. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -3565,8 +3550,6 @@ When the disk usage of a node exceeds this threshold, it triggers Redpanda to mo *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -3587,8 +3570,6 @@ Minimum size of partition that is going to be prioritized when rebalancing a clu *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -3603,13 +3584,11 @@ Mode of xref:manage:cluster-maintenance/cluster-balancing.adoc[partition balanci *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Accepted values:* -* `continuous`: partition balancing happens automatically to maintain optimal performance and availability, based on continuous monitoring for node changes (same as `node_add`) and also high disk usage. This option requires an xref:get-started:licenses.adoc[Enterprise license], and it is customized by <> and <> properties. +* `continuous`: partition balancing happens automatically to maintain optimal performance and availability, based on continuous monitoring for node changes (same as `node_add`) and also high disk usage. This option requires an xref:get-started:licenses.adoc[enterprise license], and it is customized by <> and <> properties. * `node_add`: partition balancing happens when a node is added. * `off`: partition balancing is disabled. This option is not recommended for production clusters. @@ -3627,12 +3606,10 @@ NOTE: This property applies only when <= v23.3. + +*Requires restart:* No + +*Visibility:* `user` + +*Type:* boolean + +*Default:* `false` + +--- + === space_management_max_log_concurrency Maximum parallel logs inspected during space management process. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -4855,8 +4744,6 @@ Maximum parallel segments inspected during space management process. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -4873,8 +4760,6 @@ Maximum number of bytes that may be used on each shard by compaction index write *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -4891,8 +4776,6 @@ Maximum number of bytes that may be used on each shard by compaction key-offset *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -4911,8 +4794,6 @@ NOTE: Memory per shard is computed after <>. -*Unit*: milliseconds +*Unit:* milliseconds *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -5291,12 +5192,8 @@ Delete segments older than this age. To ensure transaction state is retained as The size (in bytes) each log segment should be. -*Unit:* bytes - *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -5311,9 +5208,9 @@ The size (in bytes) each log segment should be. Number of partitions for transactions coordinator. -*Requires restart:* No +*Unit:* number of partitions per topic -*Optional:* Yes +*Requires restart:* No *Visibility:* `tunable` @@ -5333,8 +5230,6 @@ The maximum allowed timeout for transactions. If a client-requested transaction *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -5349,12 +5244,10 @@ The maximum allowed timeout for transactions. If a client-requested transaction Expiration time of producer IDs. Measured starting from the time of the last write until now for a given ID. -*Unit*: milliseconds +*Unit:* milliseconds *Requires restart:* No -*Optional:* Yes - *Visibility:* `user` *Type:* integer @@ -5369,11 +5262,9 @@ Expiration time of producer IDs. Measured starting from the time of the last wri Delay before scheduling the next check for timed out transactions. -*Unit*: milliseconds - -*Requires restart:* No +*Unit:* milliseconds -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `user` @@ -5391,8 +5282,6 @@ Enables delete retention of consumer offsets topic. This is an internal-only con *Requires restart:* Yes -*Nullable:* No - *Visibility:* `user` *Type:* boolean @@ -5409,8 +5298,6 @@ The interval in which all usage stats are written to disk. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -5427,8 +5314,6 @@ The number of windows to persist in memory and disk. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -5445,8 +5330,6 @@ The width of a usage window, tracking cloud and kafka ingress/egress traffic eac *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -5463,8 +5346,6 @@ Use a separate scheduler group for fetch processing. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -5479,8 +5360,6 @@ Minimum number of active producers per virtual cluster. *Requires restart:* No -*Nullable:* No - *Visibility:* `tunable` *Type:* integer @@ -5499,8 +5378,6 @@ Timeout to wait for leadership in metadata cache. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -5519,8 +5396,6 @@ The `write_caching_default` cluster property can be overridden with the xref:top *Requires restart:* no -*Optional:* No - *Type*: string *Accepted values:* @@ -5541,9 +5416,9 @@ The `write_caching_default` cluster property can be overridden with the xref:top Size of the zstd decompression workspace. -*Requires restart:* No +*Unit:* bytes -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -5552,3 +5427,4 @@ Size of the zstd decompression workspace. *Default:* `8388608` --- + diff --git a/modules/reference/pages/properties/object-storage-properties.adoc b/modules/reference/pages/properties/object-storage-properties.adoc index 43c0090ac6..0f8f0bfeb1 100644 --- a/modules/reference/pages/properties/object-storage-properties.adoc +++ b/modules/reference/pages/properties/object-storage-properties.adoc @@ -11,14 +11,12 @@ Object storage properties should only be set if you enable xref:manage:tiered-st === cloud_storage_access_key -AWS or GCP access key. This access key is part of the credentials that Redpanda requires to authenticate with object storage services for Tiered Storage. This access key is used with the <> to form the complete credentials required for authentication. +AWS or GCP access key. This access key is part of the credentials that Redpanda requires to authenticate with object storage services for Tiered Storage. This access key is used with the <> to form the complete credentials required for authentication. -To authenticate using IAM roles, see <>. +To authenticate using IAM roles, see <>. *Requires restart:* Yes -*Optional:* No - *Visibility:* `user` *Type:* string @@ -51,287 +49,15 @@ Optional API endpoint. The only instance in which you must set this value is whe TLS port override. -*Requires restart:* No - -*Optional:* Yes - -*Visibility:* `user` - -*Type:* integer - -*Accepted values:* [`-32768`, `32767`] - -*Default:* `443` - ---- - -=== cloud_storage_bucket - -AWS or GCP bucket that should be used to store data. - -*Requires restart:* No - -*Optional:* No - -*Visibility:* `user` - -*Type:* string - -*Default:* `null` - ---- - - -=== cloud_storage_cache_size - -Maximum size of object storage cache. - -If both this property and <> are set, Redpanda uses the minimum of the two. - -*Units*: bytes - -*Requires restart:* No - -*Optional:* Yes - -*Visibility:* `user` - -*Type:* integer - -*Accepted values:* [`0`, `18446744073709551615`] - -*Default:* `0` - - ---- - -=== cloud_storage_cluster_metadata_upload_interval_ms - -Time interval to wait between cluster metadata uploads. - -*Units*: milliseconds - -*Requires restart:* No - -*Optional:* Yes - -*Visibility:* `tunable` - -*Type:* integer - -*Accepted values:* [`-17592186044416`, `17592186044415`] - -*Default:* `3600000` (1 hour) - ---- - -=== cloud_storage_credentials_source - -The source of credentials used to authenticate to object storage services. -Required for AWS or GCP authentication with IAM roles. - -To authenticate using access keys, see <>. - -*Accepted values*: `config_file`, `aws_instance_metadata`, `sts, gcp_instance_metadata`, `azure_vm_instance_metadata`, `azure_aks_oidc_federation` - -*Requires restart:* Yes - -*Optional:* Yes - -*Visibility:* `user` - -*Default:* `config_file` - ---- - -=== cloud_storage_crl_file - -Path to certificate revocation list for <>. - -*Requires restart:* No - -*Optional:* Yes - -*Visibility:* `user` - -*Type:* string - -*Default:* `null` - ---- - -=== cloud_storage_disable_archiver_manager - -Use legacy upload mode and do not start archiver_manager. - *Requires restart:* Yes -*Optional:* No - -*Visibility:* `user` - -*Type:* boolean - -*Default:* `true` - ---- - -=== cloud_storage_disable_tls - -Disable TLS for all object storage connections. - -*Type*: boolean - -*Default*: false - -*Restart required*: yes - ---- - -=== cloud_storage_enabled - -Enable object storage. Must be set to `true` to use Tiered Storage or Remote Read Replicas. - -*Requires restart:* No - -*Optional:* Yes - -*Visibility:* `user` - -*Type:* boolean - -*Default:* `false` - ---- - -=== cloud_storage_max_connections - -Maximum simultaneous object storage connections per shard, applicable to upload and download activities. - -*Units*: number of simultaneous connections - -*Requires restart:* No - -*Optional:* Yes - *Visibility:* `user` *Type:* integer *Accepted values:* [`-32768`, `32767`] -*Default:* `20` - ---- - -=== cloud_storage_recovery_topic_validation_mode - -Validation performed before recovering a topic from object storage. In case of failure, the reason for the failure appears as `ERROR` lines in the Redpanda application log. For each topic, this reports errors for all partitions, but for each partition, only the first error is reported. - -This property accepts the following parameters: - -- `no_check`: Skips the checks for topic recovery. -- `check_manifest_existence`: Runs an existence check on each `partition_manifest`. Fails if there are connection issues to the object storage. -- `check_manifest_and_segment_metadata`: Downloads the manifest and runs a consistency check, comparing the metadata with the cloud storage objects. The process fails if metadata references any missing cloud storage objects. - -Example: Redpanda validates the topic `kafka/panda-topic-recovery-NOT-OK` and stops due to a fatal error on partition 0: - -```bash -ERROR 2024-04-24 21:29:08,166 [shard 1:main] cluster - [fiber11|0|299996ms recovery validation of {kafka/panda-topic-recovery-NOT-OK/0}/24] - manifest metadata check: missing segment, validation not ok -ERROR 2024-04-24 21:29:08,166 [shard 1:main] cluster - topics_frontend.cc:519 - Stopping recovery of {kafka/panda-topic-recovery-NOT-OK} due to validation error -``` - -Each failing partition error message has the following format: - -```bash -ERROR .... [... recovery validation of {}...] - , validation not ok -``` - -At the end of the process, Redpanda outputs a final ERROR message: - -```bash -ERROR ... ... - Stopping recovery of {} due to validation error -``` - -*Requires restart:* No - -*Required:* No - -*Visibility:* `tunable` - -*Type:* string - -*Default:* `check_manifest_existence` - -*Accepted values:* [`no_check`, `check_manifest_existence`, `check_manifest_and_segment_metadata`] - -*Related topics:* xref:manage:whole-cluster-restore.adoc[] - ---- - -=== cloud_storage_recovery_topic_validation_depth - -Number of metadata segments to validate, from newest to oldest, when <> is set to `check_manifest_and_segment_metadata`. - -*Requires restart:* No - -*Required:* No - -*Visibility:* `tunable` - -*Type:* integer - -*Accepted values:* [`0`, `4294967295`] - -*Default:* `10` - ---- - -=== cloud_storage_region - -AWS or GCP region that houses the bucket or container used for storage. - -*Requires restart:* No - -*Optional:* No - -*Visibility:* `user` - -*Type:* string - -*Default:* `null` - ---- - -=== cloud_storage_secret_key - -AWS or GCP secret key. - -*Requires restart:* Yes - -*Optional:* No - -*Visibility:* `user` - -*Type:* string - -*Default:* `null` - ---- - -=== cloud_storage_trust_file - -Path to certificate that should be used to validate server certificate during TLS handshake. - -*Requires restart:* No - -*Optional:* No - -*Visibility:* `user` - -*Type:* string - -*Default:* `null` +*Default:* `443` --- @@ -341,8 +67,6 @@ When set to `true`, Redpanda automatically retrieves cluster metadata from a spe *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -359,8 +83,6 @@ If not set, this is automatically generated using `dfs.core.windows.net` and <>, and whichever limit is hit first will trigger trimming of the cache. +Maximum number of objects that may be held in the Tiered Storage cache. This applies simultaneously with <>, and whichever limit is hit first will trigger trimming of the cache. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -588,8 +305,6 @@ Divide the object storage cache across the specified number of buckets. This onl *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -600,17 +315,29 @@ Divide the object storage cache across the specified number of buckets. This onl --- -=== cloud_storage_cache_size_percent +=== cloud_storage_cache_size -Maximum size of the cloud cache as a percentage of unreserved disk space (see config_ref:disk_reservation_percent,true,cluster-properties[]). The default value for this option is tuned for a shared disk configuration. Consider increasing the value if using a dedicated cache disk. +Maximum size of object storage cache. If both this property and <> are set, Redpanda uses the minimum of the two. -The property <> controls the same limit expressed as a fixed number of bytes. If both `cloud_storage_cache_size` and `cloud_storage_cache_size_percent` are set, Redpanda uses the minimum of the two. +*Requires restart:* No -*Units*: percentage of total disk size. +*Visibility:* `user` -*Requires restart:* No +*Type:* integer + +*Accepted values:* [`0`, `18446744073709551615`] + +*Default:* `0` + +--- -*Optional:* No +=== cloud_storage_cache_size_percent + +Maximum size of the cloud cache as a percentage of unreserved disk space disk_reservation_percent. The default value for this option is tuned for a shared disk configuration. Consider increasing the value if using a dedicated cache disk. The property <> controls the same limit expressed as a fixed number of bytes. If both `cloud_storage_cache_size` and `cloud_storage_cache_size_percent` are set, Redpanda uses the minimum of the two. + +*Unit:* percent + +*Requires restart:* No *Visibility:* `user` @@ -628,8 +355,6 @@ Cache trimming is triggered when the number of objects in the cache reaches this *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* number @@ -646,8 +371,6 @@ Cache trimming is triggered when the cache size reaches this percentage relative *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* number @@ -662,8 +385,6 @@ The maximum number of concurrent tasks launched for traversing the directory str *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -680,8 +401,6 @@ Selects a strategy for evicting unused cache chunks. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Accepted values:* [`eager`, `capped`, `predictive`] @@ -696,8 +415,6 @@ Number of chunks to prefetch ahead of every downloaded chunk. Prefetching additi *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -710,12 +427,10 @@ Number of chunks to prefetch ahead of every downloaded chunk. Prefetching additi === cloud_storage_cluster_metadata_num_consumer_groups_per_upload -Number of groups to upload in a single snapshot object during consumer offsets upload. Setting a lower value means a larger number of smaller snapshots are uploaded. +Number of groups to upload in a single snapshot object during consumer offsets upload. Setting a lower value will mean a larger number of smaller snapshots are uploaded. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -730,8 +445,6 @@ Number of attempts metadata operations may be retried. *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -742,6 +455,24 @@ Number of attempts metadata operations may be retried. --- +=== cloud_storage_cluster_metadata_upload_interval_ms + +Time interval to wait between cluster metadata uploads. + +*Unit:* milliseconds + +*Requires restart:* No + +*Visibility:* `tunable` + +*Type:* integer + +*Accepted values:* [`-17592186044416`, `17592186044415`] + +*Default:* `3600000` + +--- + === cloud_storage_cluster_metadata_upload_timeout_ms Timeout for cluster metadata uploads. @@ -750,8 +481,6 @@ Timeout for cluster metadata uploads. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -764,12 +493,10 @@ Timeout for cluster metadata uploads. === cloud_storage_credentials_host -The hostname to connect to for retrieving role based credentials. Derived from <> if not set. Only required when using IAM role-based access. +The hostname to connect to for retrieving role based credentials. Derived from <> if not set. Only required when using IAM role based access. To authenticate using access keys, see <>. *Requires restart:* Yes -*Optional:* No - *Visibility:* `tunable` *Type:* @@ -780,14 +507,43 @@ The hostname to connect to for retrieving role based credentials. Derived from < --- +=== cloud_storage_credentials_source + +The source of credentials used to authenticate to object storage services. +Required for AWS or GCP authentication with IAM roles. + +To authenticate using access keys, see <>. + +*Accepted values*: `config_file`, `aws_instance_metadata`, `sts, gcp_instance_metadata`, `azure_vm_instance_metadata`, `azure_aks_oidc_federation` + +*Requires restart:* Yes + +*Visibility:* `user` + +*Default:* `config_file` + +--- + +=== cloud_storage_crl_file + +Path to certificate revocation list for <>. + +*Requires restart:* No + +*Visibility:* `user` + +*Type:* string + +*Default:* `null` + +--- + === cloud_storage_disable_chunk_reads Disable chunk reads and switch back to legacy mode where full segments are downloaded. When set to `true`, this option disables the more efficient chunk-based reads, causing Redpanda to download entire segments. This legacy behavior might be useful in specific scenarios where chunk-based fetching is not optimal. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -802,8 +558,6 @@ Disable all metadata consistency checks to allow Redpanda to replay logs with in *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -818,8 +572,6 @@ Begins the read replica sync loop in topic partitions with Tiered Storage enable *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -836,8 +588,6 @@ CAUTION: This property exists to simplify testing and shouldn't be set in produc *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* boolean @@ -846,14 +596,26 @@ CAUTION: This property exists to simplify testing and shouldn't be set in produc --- +=== cloud_storage_disable_tls + +Disable TLS for all object storage connections. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* boolean + +*Default:* `false` + +--- + === cloud_storage_disable_upload_consistency_checks Disable all upload consistency checks to allow Redpanda to upload logs with gaps and replicate metadata with consistency violations. Normally, this option should be disabled. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -868,8 +630,6 @@ Begins the upload loop in topic partitions with Tiered Storage enabled. The prop *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -885,8 +645,6 @@ When set to `true`, Redpanda can re-upload data for compacted topics to object s *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -902,8 +660,6 @@ When set to `true`, new topics are by default configured to allow reading data d *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -919,8 +675,6 @@ When set to `true`, new topics are by default configured to upload data to objec *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -935,8 +689,6 @@ Enable routine checks (scrubbing) of object storage partitions. The scrubber val *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -955,8 +707,6 @@ Enables adjacent segment merging. The segments are reuploaded if there is an opp *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* boolean @@ -965,6 +715,20 @@ Enables adjacent segment merging. The segments are reuploaded if there is an opp --- +=== cloud_storage_enabled + +Enable object storage. Must be set to `true` to use Tiered Storage or Remote Read Replicas. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* boolean + +*Default:* `false` + +--- + === cloud_storage_full_scrub_interval_ms Interval, in milliseconds, between a final scrub and the next scrub. @@ -973,15 +737,13 @@ Interval, in milliseconds, between a final scrub and the next scrub. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer *Accepted values:* [`-17592186044416`, `17592186044415`] -*Default:* `4320000000` (12h) +*Default:* `43200000` (12h) --- @@ -993,8 +755,6 @@ Timeout for running the cloud storage garbage collection, in milliseconds. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1013,8 +773,6 @@ Time limit on waiting for uploads to complete before a leadership transfer. If *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -1033,15 +791,13 @@ Interval, in milliseconds, between object storage housekeeping tasks. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer *Accepted values:* [`-17592186044416`, `17592186044415`] -*Default:* `5000` +*Default:* `300000` --- @@ -1053,8 +809,6 @@ A segment is divided into chunks. Chunk hydration means downloading the chunk (w *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* number @@ -1075,8 +829,6 @@ Negative doesn't make sense, but it may not be checked-for/enforced. Large is su *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1093,8 +845,6 @@ The object storage request rate threshold for idle state detection. If the avera *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* number @@ -1111,8 +861,6 @@ The timeout, in milliseconds, used to detect the idle state of the object storag *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1131,8 +879,6 @@ Initial backoff time for exponential backoff algorithm (ms). *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1149,8 +895,6 @@ Scrubber uses the latest cloud storage inventory report, if available, to check *Requires restart:* Yes -*Optional:* No - *Visibility:* `tunable` *Type:* boolean @@ -1159,14 +903,12 @@ Scrubber uses the latest cloud storage inventory report, if available, to check --- -=== cloud_storage_inventory_hash_store +=== cloud_storage_inventory_hash_path_directory Directory to store inventory report hashes for use by cloud storage scrubber. *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `user` *Type:* string @@ -1181,8 +923,6 @@ The name of the scheduled inventory job created by Redpanda to generate bucket o *Requires restart:* Yes -*Optional:* No - *Visibility:* `tunable` *Type:* string @@ -1199,8 +939,6 @@ Maximum bytes of hashes held in memory before writing data to disk during invent *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -1215,9 +953,9 @@ Maximum bytes of hashes held in memory before writing data to disk during invent Time interval between checks for a new inventory report in the cloud storage bucket or container. -*Requires restart:* Yes +*Unit:* milliseconds -*Optional:* No +*Requires restart:* Yes *Visibility:* `tunable` @@ -1235,8 +973,6 @@ The prefix to the path in the cloud storage bucket or container where inventory *Requires restart:* Yes -*Optional:* No - *Visibility:* `tunable` *Type:* string @@ -1251,8 +987,6 @@ If enabled, Redpanda will not attempt to create the scheduled report configurati *Requires restart:* Yes -*Optional:* No - *Visibility:* `tunable` *Type:* boolean @@ -1269,8 +1003,6 @@ Amount of memory that can be used to handle Tiered Storage metadata. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1279,7 +1011,7 @@ Amount of memory that can be used to handle Tiered Storage metadata. --- -=== cloud_storage_manifest_cache_ttl_ms +=== cloud_storage_materialized_manifest_ttl_ms The interval, in milliseconds, determines how long the materialized manifest can stay in the cache under contention. This setting is used for performance tuning. When the spillover manifest is materialized and stored in the cache, and the cache needs to evict it, it uses this value as a timeout. The cursor that uses the spillover manifest uses this value as a TTL interval, after which it stops referencing the manifest making it available for eviction. This only affects spillover manifests under contention. @@ -1287,8 +1019,6 @@ The interval, in milliseconds, determines how long the materialized manifest can *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1303,9 +1033,9 @@ The interval, in milliseconds, determines how long the materialized manifest can Minimum interval, in seconds, between partition manifest uploads. Actual time between uploads may be greater than this interval. If this is `null`, metadata is updated after each segment upload. -*Requires restart:* No +*Unit:* seconds -*Optional:* No +*Requires restart:* No *Visibility:* `tunable` @@ -1325,8 +1055,6 @@ Manifest upload timeout, in milliseconds. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1344,8 +1072,6 @@ Maximum concurrent segment hydrations of remote data per CPU core. If unset, va *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -1363,9 +1089,7 @@ This setting reduces resource utilization by closing inactive connections. Adjus *Unit:* milliseconds -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -1377,14 +1101,28 @@ This setting reduces resource utilization by closing inactive connections. Adjus --- +=== cloud_storage_max_connections + +Maximum simultaneous object storage connections per shard, applicable to upload and download activities. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* integer + +*Accepted values:* [`-32768`, `32767`] + +*Default:* `20` + +--- + === cloud_storage_max_segment_readers_per_shard Maximum concurrent I/O cursors of materialized remote segments per CPU core. If unset, the value of `topic_partitions_per_shard` is used, where one segment reader per partition is used if the shard is at its maximum partition capacity. These readers are cached across Kafka consume requests and store a readahead buffer. *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -1401,8 +1139,6 @@ The per-partition limit for the number of segments pending deletion from the clo *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1418,8 +1154,6 @@ This setting limits the Tiered Storage subsystem's throughput per shard, facilit *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -1436,8 +1170,6 @@ Timeout for xref:manage:tiered-storage.adoc[] metadata synchronization. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1454,8 +1186,6 @@ The minimum number of chunks per segment for trimming to be enabled. If the numb *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1474,8 +1204,6 @@ Time interval between two partial scrubs of the same partition. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1494,8 +1222,6 @@ Timeout to check if new data is available for partitions in object storage for r *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1512,8 +1238,6 @@ Retention in bytes for topics created during automated recovery. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1522,15 +1246,90 @@ Retention in bytes for topics created during automated recovery. --- -=== cloud_storage_roles_operation_timeout_ms +=== cloud_storage_recovery_topic_validation_depth -Timeout for IAM role related operations, in milliseconds. +Number of metadata segments to validate, from newest to oldest, when <> is set to `check_manifest_and_segment_metadata`. -*Unit:* milliseconds +*Requires restart:* No + +*Required:* No + +*Visibility:* `tunable` + +*Type:* integer + +*Accepted values:* [`0`, `4294967295`] + +*Default:* `10` + +--- + +=== cloud_storage_recovery_topic_validation_mode + +Validation performed before recovering a topic from object storage. In case of failure, the reason for the failure appears as `ERROR` lines in the Redpanda application log. For each topic, this reports errors for all partitions, but for each partition, only the first error is reported. + +This property accepts the following parameters: + +- `no_check`: Skips the checks for topic recovery. +- `check_manifest_existence`: Runs an existence check on each `partition_manifest`. Fails if there are connection issues to the object storage. +- `check_manifest_and_segment_metadata`: Downloads the manifest and runs a consistency check, comparing the metadata with the cloud storage objects. The process fails if metadata references any missing cloud storage objects. + +Example: Redpanda validates the topic `kafka/panda-topic-recovery-NOT-OK` and stops due to a fatal error on partition 0: + +```bash +ERROR 2024-04-24 21:29:08,166 [shard 1:main] cluster - [fiber11|0|299996ms recovery validation of {kafka/panda-topic-recovery-NOT-OK/0}/24] - manifest metadata check: missing segment, validation not ok +ERROR 2024-04-24 21:29:08,166 [shard 1:main] cluster - topics_frontend.cc:519 - Stopping recovery of {kafka/panda-topic-recovery-NOT-OK} due to validation error +``` + +Each failing partition error message has the following format: + +```bash +ERROR .... [... recovery validation of {}...] - , validation not ok +``` + +At the end of the process, Redpanda outputs a final ERROR message: + +```bash +ERROR ... ... - Stopping recovery of {} due to validation error +``` *Requires restart:* No -*Optional:* Yes +*Required:* No + +*Visibility:* `tunable` + +*Type:* string + +*Default:* `check_manifest_existence` + +*Accepted values:* [`no_check`, `check_manifest_existence`, `check_manifest_and_segment_metadata`] + +*Related topics:* xref:manage:whole-cluster-restore.adoc[] + +--- + +=== cloud_storage_region + +Cloud provider region that houses the bucket or container used for storage. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* string + +*Default:* `null` + +--- + +=== cloud_storage_roles_operation_timeout_ms + +Timeout for IAM role related operations (ms). + +*Unit:* milliseconds + +*Requires restart:* Yes *Visibility:* `tunable` @@ -1550,15 +1349,27 @@ Jitter applied to the object storage scrubbing interval. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer *Accepted values:* [`-17592186044416`, `17592186044415`] -*Default:* `600000` (10min) +*Default:* `600000` + +--- + +=== cloud_storage_secret_key + +Cloud provider secret key. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* string + +*Default:* `null` --- @@ -1566,9 +1377,9 @@ Jitter applied to the object storage scrubbing interval. Time that a segment can be kept locally without uploading it to the object storage, in seconds. -*Requires restart:* No +*Unit:* seconds -*Optional:* No +*Requires restart:* No *Visibility:* `tunable` @@ -1586,8 +1397,6 @@ Smallest acceptable segment size in the object storage. Default: `cloud_storage_ *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -1604,8 +1413,6 @@ Desired segment size in the object storage. The default is set in the topic-leve *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -1622,8 +1429,6 @@ Log segment upload timeout, in milliseconds. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1640,8 +1445,6 @@ Maximum number of segments in the spillover manifest that can be offloaded to th *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -1656,8 +1459,6 @@ The size of the manifest which can be offloaded to the cloud. If the size of the *Requires restart:* No -*Optional:* No - *Visibility:* `tunable` *Type:* integer @@ -1670,9 +1471,9 @@ The size of the manifest which can be offloaded to the cloud. If the size of the Maximum throughput used by Tiered Storage per broker expressed as a percentage of the disk bandwidth. If the server has several disks, Redpanda uses the one that stores the Tiered Storage cache. Even if Tiered Storage is allowed to use the full bandwidth of the disk (100%), it won't necessarily use it in full. The actual usage depends on your workload and the state of the Tiered Storage cache. This setting is a safeguard that prevents Tiered Storage from using too many system resources: it is not a performance tuning knob. -*Requires restart:* No +*Unit:* percent -*Optional:* No +*Requires restart:* No *Visibility:* `tunable` @@ -1690,8 +1491,6 @@ Grace period during which the purger refuses to purge the topic. *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1702,13 +1501,25 @@ Grace period during which the purger refuses to purge the topic. --- +=== cloud_storage_trust_file + +Path to certificate that should be used to validate server certificate during TLS handshake. + +*Requires restart:* Yes + +*Visibility:* `user` + +*Type:* string + +*Default:* `null` + +--- + === cloud_storage_upload_ctrl_d_coeff Derivative coefficient for upload PID controller. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -1722,9 +1533,7 @@ Derivative coefficient for upload PID controller. Maximum number of I/O and CPU shares that archival upload can use. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -1740,9 +1549,7 @@ Maximum number of I/O and CPU shares that archival upload can use. Minimum number of I/O and CPU shares that archival upload can use. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -1758,9 +1565,7 @@ Minimum number of I/O and CPU shares that archival upload can use. Proportional coefficient for upload PID controller. -*Requires restart:* No - -*Optional:* Yes +*Requires restart:* Yes *Visibility:* `tunable` @@ -1778,8 +1583,6 @@ Initial backoff interval when there is nothing to upload for a partition, in mil *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1798,8 +1601,6 @@ Maximum backoff interval when there is nothing to upload for a partition, in mil *Requires restart:* No -*Optional:* Yes - *Visibility:* `tunable` *Type:* integer @@ -1822,8 +1623,6 @@ If neither addressing style works, Redpanda terminates the startup, requiring ma *Requires restart:* Yes -*Optional:* Yes - *Visibility:* `user` *Accepted values:* diff --git a/modules/reference/pages/properties/topic-properties.adoc b/modules/reference/pages/properties/topic-properties.adoc index 6070a80ac9..dadf9abd59 100644 --- a/modules/reference/pages/properties/topic-properties.adoc +++ b/modules/reference/pages/properties/topic-properties.adoc @@ -504,10 +504,62 @@ CAUTION: Setting `redpanda.remote.readreplica` together with either `redpanda.re --- +=== Apache Iceberg integration properties + +Integrate Redpanda topics as Iceberg tables. + +==== redpanda.iceberg.mode + +Enable the Iceberg integration for the topic. You can choose one of three modes. + +**Default**: `disabled` + +**Values**: + +- `key_value`: Creates an Iceberg table with a `Key` column and a `Value` column. Redpanda stores the raw topic content in the `Value` column. +- `value_schema_id_prefix`: Creates an Iceberg table whose structure matches the Redpanda schema for the topic, with columns corresponding to each field. Redpanda parses the topic values per field and stores them in the corresponding table columns. +- `disabled`: Disables writing to an Iceberg table for the topic. + +**Related topics**: + +- xref:manage:topic-iceberg-integration.adoc[] + +==== redpanda.iceberg.delete + +Whether the corresponding Iceberg table is deleted upon deleting the topic. + +**Default**: `true` + +**Related topics**: + +- xref:manage:topic-iceberg-integration.adoc[] + === Redpanda topic properties Configure Redpanda-specific topic properties. +[[redpandaleaderspreference]] +==== redpanda.leaders.preference + +The preferred location (rack) for partition leaders of a topic. + +This property inherits the value from the config_ref:default_leaders_preference,true,properties/cluster-properties[] cluster configuration property. You may override the cluster-wide setting by specifying the value for individual topics. + +If the cluster configuration property config_ref:enable_rack_awareness,true,properties/cluster-properties[] is set to `false`, Leader Pinning is disabled across the cluster. + +**Default**: `none` + +**Values**: + +- `none`: Opt out the topic from Leader Pinning. +- `racks:[,,...]`: Specify the preferred location (rack) of all topic partition leaders. The list can contain one or more rack IDs. If you specify multiple IDs, Redpanda tries to distribute the partition leader locations equally across brokers in these racks. + +**Related topics**: + +- xref:develop:produce-data/leader-pinning.adoc[Leader pinning] + +--- + ==== replication.factor The number of replicas of a topic to save in different nodes (brokers) of a cluster. @@ -525,6 +577,25 @@ NOTE: Although `replication.factor` isn't returned or displayed by xref:referenc --- +[[deleteretentionms]] +==== delete.retention.ms + +The retention time for tombstone records in a compacted topic. Redpanda removes tombstone records after the retention limit is exceeded. + +If you have enabled Tiered Storage and set <> or <> for the topic, you cannot enable tombstone removal. + +If both `delete.retention.ms` and the cluster property config_ref:tombstone_retention_ms,true,properties/cluster-properties[] are set, `delete.retention.ms` overrides the cluster level tombstone retention for an individual topic. + +*Unit:* milliseconds + +**Default**: null + +**Related topics**: + +- xref:manage:cluster-maintenance/compaction-settings.adoc#tombstone-record-removal[Tombstone record removal] + +--- + == Related topics - xref:develop:produce-data/configure-producers.adoc[Configure Producers] diff --git a/modules/reference/pages/public-metrics-reference.adoc b/modules/reference/pages/public-metrics-reference.adoc index 07ffe3330b..172736e956 100644 --- a/modules/reference/pages/public-metrics-reference.adoc +++ b/modules/reference/pages/public-metrics-reference.adoc @@ -19,7 +19,7 @@ Number of configured, fully commissioned brokers in a cluster. *Type*: gauge -*How to monitor*: Create an alert for when this gauge dips below a steady-state threshold, as a node(s) has become unresponsive. +*Usage*: Create an alert for when this gauge dips below a steady-state threshold, as a node(s) has become unresponsive. --- @@ -49,7 +49,17 @@ Number of requests dropped by a cluster controller log due to exceeding < from Tiered Storage to the cluster in the my-namespace: + +[,bash] +---- +rpk cluster storage mount +---- + + +Mount topic `` from Tiered Storage to the cluster in the `` with `` as the new topic name: + +[,bash] +---- +rpk cluster storage mount / --to / +---- + +== Flags + +[cols="1m,1a,2a"] +|=== +|*Value* |*Type* |*Description* + +|--to |string |New namespace/topic name for the mounted topic (optional). + +|-h, --help |- |Help for mount. + +|--config |string |Redpanda or `rpk` config file; default search paths are `/var/lib/redpanda/.config/rpk/rpk.yaml`, `$PWD/redpanda.yaml`, and `/etc/redpanda/redpanda.yaml`. + +|-X, --config-opt |stringArray |Override `rpk` configuration settings. See xref:reference:rpk/rpk-x-options.adoc[`rpk -X`] or execute `rpk -X help` for inline detail or `rpk -X list` for terser detail. + +|--profile |string |Profile to use. See xref:reference:rpk/rpk-profile.adoc[`rpk profile`] for more details. + +|-v, --verbose |- |Enable verbose logging. +|=== \ No newline at end of file diff --git a/modules/reference/pages/rpk/rpk-cluster/rpk-cluster-storage-status-mount.adoc b/modules/reference/pages/rpk/rpk-cluster/rpk-cluster-storage-status-mount.adoc new file mode 100644 index 0000000000..fffa7934a3 --- /dev/null +++ b/modules/reference/pages/rpk/rpk-cluster/rpk-cluster-storage-status-mount.adoc @@ -0,0 +1,45 @@ += rpk cluster storage status mount + +Status of mount/unmount operation on topic in a Redpanda cluster from glossterm:tiered-storage[]. + +== Usage + +[,bash] +---- +rpk cluster storage status-mount [MIGRATION ID] [flags] +---- + +== Aliases + +[,bash] +---- +status-mount, status-unmount +---- + +== Examples + +Status for a mount/unmount operation: + +[,bash] +---- +rpk cluster storage status-mount 123 +---- + +== Flags + +[cols="1m,1a,2a"] +|=== +|*Value* |*Type* |*Description* + +|--format |string |Output format. Possible values: `json`, `yaml`, `text`, `wide`, `help`. Default: `text`. + +|-h, --help |- |Help for status-mount. + +|--config |string |Redpanda or `rpk` config file; default search paths are `/var/lib/redpanda/.config/rpk/rpk.yaml`, `$PWD/redpanda.yaml`, and `/etc/redpanda/redpanda.yaml`. + +|-X, --config-opt |stringArray |Override `rpk` configuration settings. See xref:reference:rpk/rpk-x-options.adoc[`rpk -X`] or execute `rpk -X help` for inline detail or `rpk -X list` for terser detail. + +|--profile |string |Profile to use. See xref:reference:rpk/rpk-profile.adoc[`rpk profile`] for more details. + +|-v, --verbose |- |Enable verbose logging. +|=== \ No newline at end of file diff --git a/modules/reference/pages/rpk/rpk-cluster/rpk-cluster-storage-unmount.adoc b/modules/reference/pages/rpk/rpk-cluster/rpk-cluster-storage-unmount.adoc new file mode 100644 index 0000000000..7844cb490b --- /dev/null +++ b/modules/reference/pages/rpk/rpk-cluster/rpk-cluster-storage-unmount.adoc @@ -0,0 +1,52 @@ += rpk cluster storage unmount + +Unmount a topic from the Redpanda cluster and secure it in glossterm:tiered-storage[]. + +This command performs an operation that: + +1. Rejects all writes to the topic. + +2. Flushes data to Tiered Storage. + +3. Removes the topic from the cluster. + +Key Points: + +- During unmounting, any attempted writes or reads will receive an `UNKNOWN_TOPIC_OR_PARTITION` error. + +- The unmount operation works independently of other topic configurations like `remote.delete=false`. + +- After unmounting, the topic can be remounted to this cluster or a different cluster if the log segments are moved to that cluster's Tiered Storage. + +== Usage + +[,bash] +---- +rpk cluster storage unmount [TOPIC] [flags] +---- + +== Examples + +Unmount topic '' from the cluster in the '': + +[,bash] +---- +rpk cluster storage unmount / +---- + +== Flags + +[cols="1m,1a,2a"] +|=== +|*Value* |*Type* |*Description* + +|-h, --help |- |Help for unmount. + +|--config |string |Redpanda or `rpk` config file; default search paths are `/var/lib/redpanda/.config/rpk/rpk.yaml`, `$PWD/redpanda.yaml`, and `/etc/redpanda/redpanda.yaml`. + +|-X, --config-opt |stringArray |Override `rpk` configuration settings. See xref:reference:rpk/rpk-x-options.adoc[`rpk -X`] or execute `rpk -X help` for inline detail or `rpk -X list` for terser detail. + +|--profile |string |Profile to use. See xref:reference:rpk/rpk-profile.adoc[`rpk profile`] for more details. + +|-v, --verbose |- |Enable verbose logging. +|=== \ No newline at end of file diff --git a/modules/reference/pages/rpk/rpk-container/rpk-container-start.adoc b/modules/reference/pages/rpk/rpk-container/rpk-container-start.adoc index dbd4149300..4858fbdc70 100644 --- a/modules/reference/pages/rpk/rpk-container/rpk-container-start.adoc +++ b/modules/reference/pages/rpk/rpk-container/rpk-container-start.adoc @@ -73,7 +73,7 @@ rpk container start --admin-ports 9644,9645,9646 |-h, --help |- |Help for start. -|--image |string |An arbitrary container Redpanda image to use (default `redpandadata/redpanda:v{full-version}`). +|--image |string |An arbitrary container Redpanda image to use (default `redpandadata/redpanda:{latest-redpanda-tag}`). |--kafka-ports |strings |Kafka protocol ports to listen on; check help text for more information. diff --git a/modules/reference/pages/rpk/rpk-debug/rpk-debug-bundle.adoc b/modules/reference/pages/rpk/rpk-debug/rpk-debug-bundle.adoc index 0a635028db..cb6d91316c 100644 --- a/modules/reference/pages/rpk/rpk-debug/rpk-debug-bundle.adoc +++ b/modules/reference/pages/rpk/rpk-debug/rpk-debug-bundle.adoc @@ -116,6 +116,8 @@ capturing the second snapshot of the metrics endpoints, for example because some metrics are counters that need values at two points in time. Default: `12s`. +|--metrics-samples |int |Number of metrics samples to take (at the interval of `--metrics-interval`). Must be higher or equals 2 (default 2). + |-n, --namespace |string |The Kubernetes namespace in which the Redpanda cluster is running. Default: `redpanda` + *Kubernetes only*. @@ -151,13 +153,15 @@ The files and directories in the diagnostics bundle differ depending on the envi Linux:: + -- -include::reference:partial$bundle-contents-linux.adoc[] +include::reference:partial$bundle-contents.adoc[] -- Kubernetes:: + -- -include::reference:partial$bundle-contents-k8s.adoc[] +:env-kubernetes: true + +include::reference:partial$bundle-contents.adoc[] -- ==== diff --git a/modules/reference/pages/rpk/rpk-topic/rpk-topic-list.adoc b/modules/reference/pages/rpk/rpk-topic/rpk-topic-list.adoc index b38c19d816..9179bd1b0d 100644 --- a/modules/reference/pages/rpk/rpk-topic/rpk-topic-list.adoc +++ b/modules/reference/pages/rpk/rpk-topic/rpk-topic-list.adoc @@ -41,6 +41,8 @@ list, ls |-d, --detailed |- |Print per-partition information for topics. +|--format |string |Output format. Possible values: `json`, `yaml`, `text`, `wide`, `help`. Default: `text`. + |-h, --help |- |Help for list. |-i, --internal |- |Print internal topics. diff --git a/modules/reference/pages/rpk/rpk-version.adoc b/modules/reference/pages/rpk/rpk-version.adoc index ddd440274a..f1b3e0bc6f 100644 --- a/modules/reference/pages/rpk/rpk-version.adoc +++ b/modules/reference/pages/rpk/rpk-version.adoc @@ -6,6 +6,8 @@ Prints the current `rpk` and Redpanda version and allows you to list the Redpand To list the Redpanda version of each broker in your cluster you may pass the Admin API hosts via flags, profile, or environment variables. +To get only the rpk version, use `rpk --version`. + == Usage [,bash] diff --git a/modules/reference/partials/bundle-contents-k8s.adoc b/modules/reference/partials/bundle-contents-k8s.adoc deleted file mode 100644 index 7d51c7379b..0000000000 --- a/modules/reference/partials/bundle-contents-k8s.adoc +++ /dev/null @@ -1,52 +0,0 @@ -NOTE: Redpanda collects some data from the Kubernetes API. -To communicate with the Kubernetes API, Redpanda requires a ClusterRole attached to the default ServiceAccount for the Pods. -The files and directories that are generated only when the ClusterRole exists are labeled *Requires ClusterRole*. - -|=== -| File or Directory | Description - -| `/admin` -| Cluster and broker configurations, cluster health data, and license key information. + -*Requires ClusterRole*. - -| `/controller` -| Binary-encoded replicated logs that contain the history of configuration changes as well as internal settings. + -Redpanda can replay the events that took place in the cluster to arrive at a similar state. - -| `data-dir.txt` -| Metadata for the Redpanda data directory of the broker on which the `rpk debug bundle` command was executed. - -| `du.txt` -| The disk usage of the data directory of the broker on which the `rpk debug bundle` command was executed, as output by the `du` command. - -| `/k8s` -| Kubernetes manifests for all resources in the given Kubernetes namespace. + -*Requires ClusterRole*. - -| `kafka.json` -| Kafka metadata, such as broker configuration, topic configuration, offsets, groups, and group commits. - -| `/logs` -| Logs of each Pod in the given Kubernetes namespace. + -If `--logs-since` is passed, only the logs within the given timeframe are included. + -*Requires ClusterRole*. - -| `/metrics` -| Prometheus metrics from both the `/metrics` endpoint and the `public_metrics` endpoint. + -One directory for each broker's metrics. + -*Requires ClusterRole*. - -| `ntp.txt` -| The NTP clock delta (using https://www.ntppool.org/en/[`ntppool`] as a reference) and round trip time of the broker on which the `rpk debug bundle` command was executed. - -| `/proc` -| CPU details of the broker on which the `rpk debug bundle` command was executed. + -The directory includes a `cpuinfo` file with CPU information such as processor model, core count, cache size, frequency, as well as an `interrupts` file that contains IRQ distribution across CPU cores. - -| `redpanda.yaml` -| The Redpanda configuration file of the broker on which the `rpk debug bundle` command was executed. + -Sensitive data is removed and replaced with `(REDACTED)`. - -| `resource-usage.json` -| Redpanda resource usage data, such as CPU usage and free memory available. -|=== diff --git a/modules/reference/partials/bundle-contents-linux.adoc b/modules/reference/partials/bundle-contents-linux.adoc deleted file mode 100644 index e7c450886f..0000000000 --- a/modules/reference/partials/bundle-contents-linux.adoc +++ /dev/null @@ -1,60 +0,0 @@ -NOTE: For some data, Redpanda requires the `rpk debug bundle` command to be run with root privileges. -The names of the files or directories that are generated only with root privileges are labeled *Requires root privileges*. - -[cols="1,1", options="header"] -|=== -|File or Directory |Description - -|`data-dir.txt` -|Metadata for the Redpanda data directory of the broker on which the `rpk debug bundle` command was executed. - -|`dig.txt` -|The DNS information, as output by the `dig` command, using the hosts in the `/etc/resolv.conf` file. - -|`dmidecode.txt` -|The contents of the DMI table (system management BIOS or SMBIOS). + -*Requires root privileges* - -|`du.txt` -|The disk usage of the data directory of the broker on which the `rpk debug bundle` command was executed, as output by the `du` command. - -|`ip.txt` -|Network configuration, as output by the `ip addr` command. - -|`kafka.json` -|Kafka metadata, such as broker configuration, topic configuration, offsets, groups, and group commits. - -|`lspci.txt` -|PCI buses and the devices connected to them. - -|`ntp.txt` -|The NTP clock delta (using https://www.ntppool.org/en/[`ntppool`] as a reference) and round trip time of the broker on which the `rpk debug bundle` command was executed. - -|`/proc` -|CPU details of the broker on which the `rpk debug bundle` command was executed. + -The directory includes a `cpuinfo` file with CPU information such as processor model, core count, cache size, and frequency, as well as an `interrupts` file that contains IRQ distribution across CPU cores. - -|`redpanda.log` -|The Redpanda logs written to journald. If `--logs-since` and/or `--logs-until` are passed, then only the logs within the given timeframe are included. - -|`prometheus-metrics.txt` -|The local broker's Prometheus metrics, fetched through its admin API. - -|`redpanda.yaml` -|The Redpanda configuration file of the broker on which the `rpk debug bundle` command was executed.
Sensitive data is removed and replaced with `(REDACTED)`. - -|`resource-usage.json` -|Redpanda resource usage data, such as CPU usage and free memory available. - -|`ss.txt` -|Data about active sockets, as output by the `ss` command. - -|`syslog.txt` -|The kernel logs ring buffer, as output by the `syslog` command. - -|`top.txt` -|Information about the running processes, as output by the `top` command. Check system processes. - -|`vmstat.txt` -|Virtual memory statistics, as output by the `vmstat` command. -|=== diff --git a/modules/reference/partials/bundle-contents.adoc b/modules/reference/partials/bundle-contents.adoc new file mode 100644 index 0000000000..591ca9c4e1 --- /dev/null +++ b/modules/reference/partials/bundle-contents.adoc @@ -0,0 +1,79 @@ +ifdef::env-kubernetes[] +NOTE: Redpanda collects some data from the Kubernetes API. +To communicate with the Kubernetes API, Redpanda requires a ClusterRole attached to the default ServiceAccount for the Pods. +The files and directories that are generated only when the ClusterRole exists are labeled *Requires ClusterRole*. +endif::[] + +|=== +| File or Directory | Description + +| `/admin` +| Cluster and broker configurations, cluster health data, and license key information. + +ifdef::env-kubernetes[] +*Requires ClusterRole*. +endif::[] + +| `/controller` +| Binary-encoded replicated logs that contain the history of configuration changes as well as internal settings. + +Redpanda can replay the events that took place in the cluster to arrive at a similar state. + +| `data-dir.txt` +| Metadata for the Redpanda data directory of the broker on which the `rpk debug bundle` command was executed. + +ifdef::env-kubernetes[] +| `/k8s` +| Kubernetes manifests for all resources in the given Kubernetes namespace. + +*Requires ClusterRole*. +endif::[] + +| `kafka.json` +| Kafka metadata, such as broker configuration, topic configuration, offsets, groups, and group commits. + +ifndef::env-kubernetes[] +| `redpanda.log` +|Redpanda logs for the broker. + +If `--logs-since` is passed, only the logs within the given timeframe are included. + +endif::[] + +ifdef::env-kubernetes[] +| `/logs` +|Logs from the Pods that run Redpanda in the given Kubernetes namespace. + +If `--logs-since` is passed, only the logs within the given timeframe are included. + +*Requires ClusterRole*. +endif::[] + +| `/metrics` +| Prometheus metrics from both the `/metrics` endpoint and the `public_metrics` endpoint. + +ifdef::env-kubernetes[] +*Requires ClusterRole*. +endif::[] + +| `/proc` +| CPU details of the broker on which the `rpk debug bundle` command was executed. + +The directory includes a `cpuinfo` file with CPU information such as processor model, core count, cache size, frequency, as well as an `interrupts` file that contains IRQ distribution across CPU cores. + +| `redpanda.yaml` +| The Redpanda configuration file of the broker on which the `rpk debug bundle` command was executed. + +Sensitive data is removed and replaced with `(REDACTED)`. + +| `resource-usage.json` +| Redpanda resource usage data, such as CPU usage and free memory available. + +| `/utils` +a| Data from the node on which the broker is running. This directory includes: + +- `du.txt`: The disk usage of the data directory of the broker on which the `rpk debug bundle` command was executed, as output by the `du` command. +- `ntp.txt`: The NTP clock delta (using https://www.ntppool.org/en/[`ntppool`] as a reference) and round trip time of the broker on which the `rpk debug bundle` command was executed. +- `uname.txt`: System information, such as the kernel version, hostname, and architecture, as output by the `uname` command. +ifndef::env-kubernetes[] +- `dig.txt`: The DNS resolution information for the node, as output by the `dig` command. +- `dmidecode.txt`: System hardware information from the node, as output by the the `dmidecode` command. *Requires root privileges*. +- `free.txt`: The amount of free and used memory on the node, as output by the `free` command. +- `ip.txt`: Network interface information, including IP addresses and network configuration, as output by the `ip` command. +- `lspci.txt`: Information about PCI devices on the node, as output by the `lspci` command. +- `ss.txt`: Active socket connections, as output by the `ss` command, showing network connections, listening ports, and more. +- `sysctl.txt`: Kernel parameters of the system, as output by the `sysctl` command. +- `top.txt`: The top processes by CPU and memory usage, as output by the `top` command. +- `vmstat.txt`: Virtual memory statistics, including CPU usage, memory, and IO operations, as output by the `vmstat` command. +endif::[] +|=== diff --git a/modules/shared/attachments/redpanda-console-config.yaml b/modules/shared/attachments/redpanda-console-config.yaml index 4b0be5c7dd..4323b4b238 100644 --- a/modules/shared/attachments/redpanda-console-config.yaml +++ b/modules/shared/attachments/redpanda-console-config.yaml @@ -169,8 +169,8 @@ kafka: #maxRetryInterval: 60s #backoffMultiplier: 2 -# To mount an Enterprise license, set either license or licenseFilepath. -# This is only required if you want to use an Enterprise feature +# To mount an enterprise license, set either license or licenseFilepath. +# This is only required if you want to use an enterprise feature # such as SSO or RBAC. # Filepath to your redpanda.license file #licenseFilepath: "" @@ -178,7 +178,7 @@ kafka: #license: "" # The enterprise block contains configurations for features that -# can only be used with a valid Redpanda Enterprise license. +# can only be used with a valid Redpanda enterprise license. #enterprise: #rbac: # Whether or not RBAC is used. This must be enabled @@ -254,7 +254,7 @@ kafka: # Login contains all configurations in order to protect Redpanda Console # with a login screen. To support SSO, configure one or more of the identity # providers below. -# This feature requires an Enterprise license. +# This feature requires an enterprise license. #login: #enabled: false # jwtSecret is the secret key you must use to sign and encrypt the JSON diff --git a/modules/shared/images/iceberg-integration.png b/modules/shared/images/iceberg-integration.png new file mode 100644 index 0000000000..a11db5230d Binary files /dev/null and b/modules/shared/images/iceberg-integration.png differ diff --git a/modules/shared/partials/enterprise-and-console.adoc b/modules/shared/partials/enterprise-and-console.adoc index 06c96248c7..451da64eb8 100644 --- a/modules/shared/partials/enterprise-and-console.adoc +++ b/modules/shared/partials/enterprise-and-console.adoc @@ -1 +1,3 @@ -This section pertains to Redpanda Console in a self-managed deployment, and this feature requires an xref:get-started:licenses.adoc[Enterprise license]. To upgrade, contact https://redpanda.com/try-redpanda?section=enterprise-trial[Redpanda sales^]. \ No newline at end of file +This feature requires an xref:get-started:licenses.adoc[enterprise license]. To get a trial license key or extend your trial period, https://redpanda.com/try-enterprise[generate a new trial license key^]. To purchase a license, contact https://redpanda.com/upgrade[Redpanda Sales^]. + +If Redpanda Console has enterprise features enabled and it cannot find a valid license either in its xref:console:config/enterprise-license.adoc[local configuration] or in the xref:get-started:licensing/add-license-redpanda/index.adoc[connected Redpanda cluster], it redirects you to the license expiration landing page, and all other access is restricted. \ No newline at end of file diff --git a/modules/shared/partials/enterprise-license.adoc b/modules/shared/partials/enterprise-license.adoc index ceb0a8952c..fcafee78e5 100644 --- a/modules/shared/partials/enterprise-license.adoc +++ b/modules/shared/partials/enterprise-license.adoc @@ -1 +1,3 @@ -This feature requires an xref:get-started:licenses.adoc[Enterprise license] for self-managed deployments. To upgrade, contact https://redpanda.com/try-redpanda?section=enterprise-trial[Redpanda sales^]. +This feature requires an xref:get-started:licenses.adoc[enterprise license]. To get a trial license key or extend your trial period, https://redpanda.com/try-enterprise[generate a new trial license key^]. To purchase a license, contact https://redpanda.com/upgrade[Redpanda Sales^]. + +If Redpanda has enterprise features enabled and it cannot find a valid license, xref:get-started:licenses.adoc#self-managed[restrictions] apply. diff --git a/modules/troubleshoot/pages/cluster-diagnostics/diagnose-issues.adoc b/modules/troubleshoot/pages/cluster-diagnostics/diagnose-issues.adoc new file mode 100644 index 0000000000..afa5e0e8fa --- /dev/null +++ b/modules/troubleshoot/pages/cluster-diagnostics/diagnose-issues.adoc @@ -0,0 +1,7 @@ += Run Cluster Diagnostics in Linux +:description: Use this guide to diagnose and troubleshoot issues in a Redpanda cluster running in Linux. +:page-aliases: manage:cluster-maintenance/cluster-diagnostics.adoc + +{description} + +include::troubleshoot:partial$cluster-diagnostics.adoc[] \ No newline at end of file diff --git a/modules/troubleshoot/pages/cluster-diagnostics/index.adoc b/modules/troubleshoot/pages/cluster-diagnostics/index.adoc new file mode 100644 index 0000000000..9ac521e650 --- /dev/null +++ b/modules/troubleshoot/pages/cluster-diagnostics/index.adoc @@ -0,0 +1,5 @@ += Cluster Diagnostics +:description: Discover tools and tests to diagnose issues in Redpanda clusters across multiple environments. +:page-layout: index + +{description} \ No newline at end of file diff --git a/modules/troubleshoot/pages/cluster-diagnostics/k-diagnose-issues.adoc b/modules/troubleshoot/pages/cluster-diagnostics/k-diagnose-issues.adoc new file mode 100644 index 0000000000..2368451009 --- /dev/null +++ b/modules/troubleshoot/pages/cluster-diagnostics/k-diagnose-issues.adoc @@ -0,0 +1,7 @@ += Run Cluster Diagnostics in Kubernetes +:env-kubernetes: true +:description: Use this guide to diagnose and troubleshoot issues in a Redpanda cluster running in Kubernetes. + +{description} + +include::troubleshoot:partial$cluster-diagnostics.adoc[] \ No newline at end of file diff --git a/modules/troubleshoot/pages/debug-bundle/generate-debug-bundle.adoc b/modules/troubleshoot/pages/debug-bundle/generate-debug-bundle.adoc new file mode 100644 index 0000000000..287804f5d1 --- /dev/null +++ b/modules/troubleshoot/pages/debug-bundle/generate-debug-bundle.adoc @@ -0,0 +1,7 @@ += Generate a Debug Bundle in Linux +:page-categories: Management, Troubleshooting +:description: pass:q[Use `rpk` or Redpanda Console to generate a debug bundle to diagnose issues yourself, or send it to the Redpanda support team to help resolve your issue.] + +{description} + +include::troubleshoot:partial$debug-bundle.adoc[] \ No newline at end of file diff --git a/modules/troubleshoot/pages/debug-bundle/index.adoc b/modules/troubleshoot/pages/debug-bundle/index.adoc new file mode 100644 index 0000000000..5ea32ada34 --- /dev/null +++ b/modules/troubleshoot/pages/debug-bundle/index.adoc @@ -0,0 +1,5 @@ += Generate a Debug Bundle +:description: Generate a ZIP file with data that can help debug and diagnose issues with a Redpanda cluster, a broker, or the machines on which the brokers are running. +:page-layout: index + +{description} \ No newline at end of file diff --git a/modules/troubleshoot/pages/debug-bundle/k-generate-debug-bundle.adoc b/modules/troubleshoot/pages/debug-bundle/k-generate-debug-bundle.adoc new file mode 100644 index 0000000000..7023eaa25d --- /dev/null +++ b/modules/troubleshoot/pages/debug-bundle/k-generate-debug-bundle.adoc @@ -0,0 +1,9 @@ += Generate a Debug Bundle in Kubernetes +:page-aliases: manage:kubernetes/troubleshooting/diagnostics-bundle.adoc, manage:kubernetes/troubleshooting/k-diagnostics-bundle.adoc +:page-categories: Management, Troubleshooting +:env-kubernetes: true +:description: pass:q[Use `rpk` or Redpanda Console to generate a debug bundle to diagnose issues yourself, or send it to the Redpanda support team to help resolve your issue.] + +{description} + +include::troubleshoot:partial$debug-bundle.adoc[] \ No newline at end of file diff --git a/modules/troubleshoot/pages/errors-solutions/index.adoc b/modules/troubleshoot/pages/errors-solutions/index.adoc new file mode 100644 index 0000000000..23d9603962 --- /dev/null +++ b/modules/troubleshoot/pages/errors-solutions/index.adoc @@ -0,0 +1,5 @@ += Error Messages and Solutions +:description: Find solutions to common Redpanda errors, including error messages, explanations, and troubleshooting steps for Linux and Kubernetes deployments. +:page-layout: index + +{description} diff --git a/modules/troubleshoot/pages/errors-solutions/k-resolve-errors.adoc b/modules/troubleshoot/pages/errors-solutions/k-resolve-errors.adoc new file mode 100644 index 0000000000..b78a03cfd0 --- /dev/null +++ b/modules/troubleshoot/pages/errors-solutions/k-resolve-errors.adoc @@ -0,0 +1,12 @@ += Resolve Errors in Kubernetes +:description: This section describes errors or issues you might encounter while deploying Redpanda in Kubernetes and explains how to troubleshoot them. +:page-categories: Management, Troubleshooting +:page-aliases: manage:kubernetes/troubleshooting/troubleshoot.adoc, manage:kubernetes/troubleshooting/k-troubleshoot.adoc +:page-categories: Management, Troubleshooting +:env-kubernetes: true +:include-categories: true + +{description} + +include::troubleshoot:partial$errors-and-solutions.adoc[] + diff --git a/modules/troubleshoot/pages/errors-solutions/resolve-errors.adoc b/modules/troubleshoot/pages/errors-solutions/resolve-errors.adoc new file mode 100644 index 0000000000..ef9793ff54 --- /dev/null +++ b/modules/troubleshoot/pages/errors-solutions/resolve-errors.adoc @@ -0,0 +1,8 @@ += Resolve Errors in Linux +:description: This section describes errors or issues you might encounter while deploying Redpanda in Linux and explains how to troubleshoot them. +:page-categories: Management, Troubleshooting +:include-categories: true + +{description} + +include::troubleshoot:partial$errors-and-solutions.adoc[] \ No newline at end of file diff --git a/modules/troubleshoot/pages/index.adoc b/modules/troubleshoot/pages/index.adoc new file mode 100644 index 0000000000..3a6d892d71 --- /dev/null +++ b/modules/troubleshoot/pages/index.adoc @@ -0,0 +1,7 @@ += Troubleshoot Redpanda +:description: Find advice on diagnosing issues and handling errors when using Redpanda. +:page-layout: index +:page-categories: Management, Troubleshooting +:page-aliases: manage:kubernetes/troubleshooting/index.adoc + +Use these guides to troubleshoot Redpanda. Navigate through cluster diagnostics for specific environments or see the list of error messages and solutions. diff --git a/modules/troubleshoot/partials/cluster-diagnostics.adoc b/modules/troubleshoot/partials/cluster-diagnostics.adoc new file mode 100644 index 0000000000..7c8f8bb818 --- /dev/null +++ b/modules/troubleshoot/partials/cluster-diagnostics.adoc @@ -0,0 +1,372 @@ +ifdef::env-kubernetes[] +:link-errors: troubleshoot:errors-solutions/k-resolve-errors.adoc +:link-bundle: troubleshoot:debug-bundle/k-generate-debug-bundle.adoc +endif::[] +ifndef::env-kubernetes[] +:link-errors: troubleshoot:errors-solutions/resolve-errors.adoc +:link-bundle: troubleshoot:debug-bundle/generate-debug-bundle.adoc +endif::[] + +ifdef::env-kubernetes[] +== Prerequisites + +Before troubleshooting Redpanda, ensure that Kubernetes isn't the cause of the issue. For information about debugging applications in a Kubernetes cluster, see the https://kubernetes.io/docs/tasks/debug/[Kubernetes documentation^]. +endif::[] + +== Collect all debugging data + +For a comprehensive diagnostic snapshot, generate a debug bundle that collects detailed data for cluster, broker, or node analysis. + +See xref:{link-bundle}[] for details on generating a debug bundle. + +ifdef::env-kubernetes[] +== View Helm chart configuration + +To check the overrides that were applied to your deployment: + +[,bash] +---- +helm get values --namespace +---- + +If you're using the Redpanda Operator, the chart name matches the name of your Redpanda resource. + +To check all the values that were set in the Redpanda Helm chart, including any overrides: + +[,bash] +---- +helm get values --namespace --all +---- + +== View recent events + +To understand the latest events that occurred in your Redpanda cluster's namespace, you can sort events by their creation timestamp: + +[,bash] +---- +kubectl get events --namespace --sort-by='.metadata.creationTimestamp' +---- + +== View Redpanda logs + +Logs are crucial for monitoring and troubleshooting your Redpanda clusters. Redpanda brokers output logs to STDOUT, making them accessible via `kubectl`. + +To access logs for a specific Pod: + +. List all Pods to find the names of those that are running Redpanda brokers: ++ +[source,bash] +---- +kubectl get pods --namespace +---- + +. View logs for a particular Pod by replacing `` with the name of your Pod: ++ +[source,bash] +---- +kubectl logs --namespace +---- ++ +[TIP] +==== +For a comprehensive overview, you can view aggregated logs from all Pods in the StatefulSet: + +[,bash] +---- +kubectl logs --namespace -l app.kubernetes.io/component=redpanda-statefulset +---- +==== + +=== Change the default log level + +To change the default log level for all Redpanda subsystems, use the `logging.logLevel` configuration. Valid values are `trace`, `debug`, `info`, `warn`, `error`. + +Changing the default log level to `debug` can provide more detailed logs for diagnostics. This logging level increases the volume of generated logs. + +NOTE: To set different log levels for individual subsystems, see <>. + +[tabs] +====== +Helm + Operator:: ++ +-- +Apply the new log level: + +.`redpanda-cluster.yaml` +[source,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + chartRef: {} + clusterSpec: + logging: + logLevel: debug +---- + +Then, apply this configuration: + +[source,bash] +---- +kubectl apply -f redpanda-cluster.yaml --namespace +---- +-- + +Helm:: ++ +-- +Choose between using a custom values file or setting values directly: +[tabs] +==== +--values:: ++ +Specify logging settings in `logging.yaml`, then upgrade: ++ +.`logging.yaml` +[source,yaml] +---- +logging: + logLevel: debug +---- ++ +[source,bash] +---- +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --values logging.yaml --reuse-values +---- +--set:: ++ +Directly set the log level during upgrade: ++ +[source,bash] +---- +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --set logging.logLevel=debug +---- +==== +-- +====== + +After applying these changes, verify the log level by <> for the Redpanda Pods. + +=== Override the default log level for Redpanda subsystems + +You can override the log levels for individual subsystems, such as `rpc` and `kafka`, for more detailed logging control. Overrides exist for the entire length of the running Redpanda process. + +TIP: To temporarily override the log level for individual subsystems, you can use the xref:reference:rpk/rpk-redpanda/rpk-redpanda-admin-config-log-level-set.adoc[`rpk redpanda admin config log-level set`] command. + +. List all available subsystem loggers: ++ +[source,bash] +---- +kubectl exec -it --namespace -c redpanda -- rpk redpanda start --help-loggers +---- + +. Set the log level for one or more subsystems. In this example, the `rpc` and `kafka` subsystem loggers are set to `debug`. ++ +[tabs] +====== +Helm + Operator:: ++ +-- +Apply the new log level: + +.`redpanda-cluster.yaml` +[source,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + chartRef: {} + clusterSpec: + statefulset: + additionalRedpandaCmdFlags: + - '--logger-log-level=rpc=debug:kafka=debug' +---- + +Then, apply this configuration: + +[source,bash] +---- +kubectl apply -f redpanda-cluster.yaml --namespace +---- +-- + +Helm:: ++ +-- +Choose between using a custom values file or setting values directly: +[tabs] +==== +--values:: ++ +Specify logging settings in `logging.yaml`, then upgrade: ++ +.`logging.yaml` +[source,yaml] +---- +statefulset: + additionalRedpandaCmdFlags: + - '--logger-log-level=rpc=debug:kafka=debug' +---- ++ +[source,bash] +---- +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --values logging.yaml --reuse-values +---- +--set:: ++ +Directly set the log level during upgrade: ++ +[source,bash] +---- +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --set statefulset.additionalRedpandaCmdFlags="{--logger-log-level=rpc=debug:kafka=debug}" +---- +==== +-- +====== + +Overriding the log levels for specific subsystems provides enhanced visibility into Redpanda's internal operations, facilitating better debugging and monitoring. + +== View Redpanda Operator logs + +To learn what's happening with the Redpanda Operator and the associated Redpanda resources, you can inspect a combination of Kubernetes events and the resource manifests. By monitoring these events and resources, you can troubleshoot any issues that arise during the lifecycle of a Redpanda deployment. + +[,bash] +---- +kubectl logs -l app.kubernetes.io/name=operator -c manager --namespace +---- + +=== Inspect Helm releases + +The Redpanda Operator uses Flux to deploy the Redpanda Helm chart. By inspecting the `helmreleases.helm.toolkit.fluxcd.io` resource, you can get detailed information about the Helm installation process for your Redpanda resource: + +[,bash] +---- +kubectl get helmreleases.helm.toolkit.fluxcd.io -o yaml --namespace +---- + +To check the Redpanda resource: + +[,bash] +---- +kubectl get redpandas.cluster.redpanda.com -o yaml --namespace +---- + +In both the HelmRelease and the Redpanda resource, the condition section reveals the ongoing status of the Helm installation. These conditions provide information on the success, failure, or pending status of various operations. +endif::[] + +[[self-test]] +== Self-test benchmarks + +When anomalous behavior arises in a cluster, you can determine if it's caused by issues with hardware, such as disk drives or network interfaces (NICs) by running xref:reference:rpk/rpk-cluster/rpk-cluster-self-test.adoc[`rpk cluster self-test`] to assess their performance and compare it to vendor specifications. + +The `rpk cluster self-test` command runs a set of benchmarks to gauge the maximum performance of a machine's disks and network connections: + +* **Disk tests**: Measures throughput and latency by performing concurrent sequential operations. +* **Network tests**: Selects unique pairs of Redpanda brokers as client/server pairs and runs throughput tests between them. + +Each benchmark runs for a configurable duration and returns IOPS, throughput, and latency metrics. This helps you determine if hardware performance aligns with expected vendor specifications. + +=== Cloud storage tests + +You can also use the self-test command to confirm your cloud storage is configured correctly for xref:manage:tiered-storage.adoc[Tiered Storage]. + +Self-test performs the following checks to validate cloud storage configuration: + +include::reference:partial$rpk-self-test-cloud-tests.adoc[] + +For more information on cloud storage test details, see the xref:reference:rpk/rpk-cluster/rpk-cluster-self-test-start.adoc[`rpk cluster self-test start`] reference. + +=== Start self-test + +To start using self-test, run the `self-test start` command. Only initiate `self-test start` when system resources are available, as this operation can be resource-intensive. + +[,bash] +---- +rpk cluster self-test start +---- + +For command help, run `rpk cluster self-test start -h`. For additional command flags, see the xref:reference:rpk/rpk-cluster/rpk-cluster-self-test-start.adoc[rpk cluster self-test start] reference. + +Before `self-test start` begins, it requests your confirmation to run its potentially large workload. + +Example start output: + +[.no-copy] +---- +? Redpanda self-test will run benchmarks of disk and network hardware that will consume significant system resources. Do not start self-test if large workloads are already running on the system. (Y/n) +Redpanda self-test has started, test identifier: "031be460-246b-46af-98f2-5fc16f03aed3", To check the status run: +rpk cluster self-test status +---- + +The `self-test start` command returns immediately, and self-test runs its benchmarks asynchronously. + +=== Check self-test status + +To check the status of self-test, run the `self-test status` command. + +[,bash] +---- +rpk cluster self-test status +---- + +For command help, run `rpk cluster self-test status -h`. For additional command flags, see the xref:reference:rpk/rpk-cluster/rpk-cluster-self-test-status.adoc[rpk cluster self-test status] reference. + +If benchmarks are currently running, `self-test status` returns a test-in-progress message. + +Example status output: + +[.no-copy] +---- +$ rpk cluster self-test status +Nodes [0 1 2] are still running jobs +---- + +[TIP] +==== +The `status` command can output results in JSON format for automated checks or script integration. Use the `--format=json` option: + +[,bash] +---- +rpk cluster self-test status --format=json +---- +==== + +If benchmarks have completed, `self-test status` returns their results. + +include::reference:partial$rpk-self-test-descriptions.adoc[] + +.Example status output: test results +include::reference:partial$rpk-self-test-status-output.adoc[] + +=== Stop self-test + +To stop a running self-test, run the `self-test stop` command. + +[,bash] +---- +rpk cluster self-test stop +---- + +Example stop output: + +[.no-copy] +---- +$ rpk cluster self-test stop +All self-test jobs have been stopped +---- + +For command help, run `rpk cluster self-test stop -h`. For additional command flags, see the xref:reference:rpk/rpk-cluster/rpk-cluster-self-test-stop.adoc[rpk cluster self-test stop] reference. + +For more details about self-test, including command flags, see xref:reference:rpk/rpk-cluster/rpk-cluster-self-test.adoc[rpk cluster self-test]. + +== Next steps + +Learn how to xref:{link-errors}[resolve common errors]. \ No newline at end of file diff --git a/modules/troubleshoot/partials/debug-bundle.adoc b/modules/troubleshoot/partials/debug-bundle.adoc new file mode 100644 index 0000000000..9c8d2a1c05 --- /dev/null +++ b/modules/troubleshoot/partials/debug-bundle.adoc @@ -0,0 +1,433 @@ +ifndef::env-kubernetes[] +== Prerequisites + +You must have xref:get-started:rpk-install.adoc[`rpk` installed] on your host machine. +endif::[] + +== Generate a debug bundle with `rpk` + +To generate a debug bundle with `rpk`, you can run the xref:reference:rpk/rpk-debug/rpk-debug-bundle.adoc[`rpk debug bundle`] command on each broker in the cluster. + +ifdef::env-kubernetes[] +. Create a ClusterRole to allow Redpanda to collect information from the Kubernetes API: ++ +[tabs] +====== +Helm + Operator:: ++ +-- +.`redpanda-cluster.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + chartRef: {} + clusterSpec: + serviceAccount: + create: true + rbac: + enabled: true +---- + +```bash +kubectl apply -f redpanda-cluster.yaml --namespace +``` + +NOTE: You must deploy the Redpanda Operator with the `--set rbac.createRPKBundleCRs=true` flag to give it the required ClusterRoles. + +-- +Helm:: ++ +-- +[tabs] +==== +--values:: ++ +.`serviceaccount.yaml` +[,yaml] +---- +serviceAccount: + create: true +rbac: + enabled: true +---- ++ +```bash +helm upgrade --install redpanda redpanda/redpanda --namespace redpanda --create-namespace \ + --values serviceaccount.yaml --reuse-values +``` + +--set:: ++ +```bash +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --set serviceAccount.create=true \ + --set rbac.enabled=true +``` +==== +-- +====== ++ +[TIP] +==== +If you aren't using the Helm chart, you can create the ClusterRole manually: + +```bash +kubectl create clusterrolebinding redpanda --clusterrole=view --serviceaccount=redpanda:default +``` +==== +endif::[] + + +. Execute the xref:reference:rpk/rpk-debug/rpk-debug-bundle.adoc[`rpk debug bundle`] command on a broker: ++ +[,bash] +---- +ifdef::env-kubernetes[] +kubectl exec -it --namespace redpanda-0 -c redpanda -- rpk debug bundle --namespace +endif::[] +ifndef::env-kubernetes[] +rpk debug bundle +endif::[] +---- ++ +If you have an upload URL from the Redpanda support team, +provide it in the `--upload-url` flag to upload your debug bundle to Redpanda. ++ +[,bash] +---- +ifdef::env-kubernetes[] +kubectl exec -it --namespace redpanda-0 -c redpanda -- rpk debug bundle \ + --upload-url \ + --namespace +endif::[] +ifndef::env-kubernetes[] +rpk debug bundle \ + --upload-url +endif::[] + +---- ++ +Example output: ++ +[.no-copy] +---- +Creating bundle file... + +Debug bundle saved to "/var/lib/redpanda/1675440652-bundle.zip" +---- + + +. On your host machine, make a directory in which to save the debug bundle: ++ +```bash +mkdir debug-bundle +``` + +. Copy the debug bundle ZIP file to the `debug-bundle` directory on your host machine. +ifdef::env-kubernetes[] ++ +Replace `` with the name of your ZIP file. ++ +```bash +kubectl cp /redpanda-0:/var/lib/redpanda/ debug-bundle/.zip +``` +endif::[] + +. Unzip the file on your host machine. ++ +```bash +cd debug-bundle +unzip .zip +``` + +. Remove the debug bundle from the Redpanda broker: ++ +[,bash] +---- +ifdef::env-kubernetes[] +kubectl exec redpanda-0 -c redpanda --namespace -- rm /var/lib/redpanda/.zip +endif::[] +ifndef::env-kubernetes[] +rm /var/lib/redpanda/.zip +endif::[] +---- ++ +TIP: To avoid manually deleting debug bundles, you can configure the `debug_bundle_auto_removal_seconds` property to automatically remove them after a period of time. See <>. + +When you've finished troubleshooting, remove the debug bundle from your host machine: + +```bash +rm -r debug-bundle +``` + +For a description of the files and directories, see <>. + +== Generate a debug bundle with Redpanda Console + +See xref:console:ui/generate-bundle.adoc[]. + +== Automatically remove debug bundles + +To avoid manually deleting debug bundles, you can configure the xref:reference:properties/cluster-properties.adoc#debug_bundle_auto_removal_seconds[`debug_bundle_auto_removal_seconds`] property. This cluster configuration property automatically deletes debug bundles after the specified number of seconds. By default, this property is not set, meaning debug bundles are retained indefinitely. + +NOTE: Only one debug bundle can exist at a time. If you generate a new debug bundle, any existing bundle from a previous run will be automatically deleted. + +Changes to this property take effect immediately and do not require a cluster restart. + +ifndef::env-kubernetes[] +To set this property, use the xref:reference:rpk/rpk-cluster/rpk-cluster-config.adoc[`rpk cluster config`] command: +[,bash] +---- +rpk cluster config set debug_bundle_auto_removal_seconds +---- + +For example, to retain debug bundles for 1 day: +[,bash] +---- +rpk cluster config set debug_bundle_auto_removal_seconds 86400 +---- +endif::[] +ifdef::env-kubernetes[] +To set this property, use the `config.cluster.debug_bundle_auto_removal_seconds` field: + +[tabs] +====== +Helm + Operator:: ++ +-- +.`redpanda-cluster.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + chartRef: {} + clusterSpec: + config: + cluster: + debug_bundle_auto_removal_seconds: +---- + +For example, to retain debug bundles for 1 day: + +.`redpanda-cluster.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + chartRef: {} + clusterSpec: + config: + cluster: + debug_bundle_auto_removal_seconds: 86400 +---- + +Apply the changes with: + +```bash +kubectl apply -f redpanda-cluster.yaml --namespace +``` + +-- +Helm:: ++ +-- +Update the `values.yaml` file or use the `--set` flag to specify the property: + +[tabs] +==== +--values:: ++ +.`cloud-storage.yaml` +[,yaml] +---- +config: + cluster: + debug_bundle_auto_removal_seconds: +---- ++ +For example, to retain debug bundles for 1 day: ++ +.`cloud-storage.yaml` +[,yaml] +---- +config: + cluster: + debug_bundle_auto_removal_seconds: 86400 +---- ++ +Apply the changes with: ++ +```bash +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --values cloud-storage.yaml --reuse-values +``` + +--set:: ++ +```bash +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --set config.cluster.debug_bundle_auto_removal_seconds= +``` ++ +For example, to retain debug bundles for 1 day: ++ +```bash +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --set config.cluster.debug_bundle_auto_removal_seconds=86400 +``` + +==== +-- +====== +endif::[] + +== Choose where the debug bundle is saved + +The xref:reference:properties/cluster-properties.adoc#debug_bundle_storage_dir[`debug_bundle_storage_dir`] property allows you to specify a custom directory for storing debug bundles. By default, debug bundles are stored in the Redpanda data directory. Configuring a custom storage directory can help manage storage capacity and isolate debug data from operational data. + +Changes to this property take effect immediately and do not require a cluster restart. + +Before you change this property: + +* Ensure that your chosen directory has sufficient storage capacity to handle debug bundles. ++ +Factors such as the volume of logs can increase the bundle size. While it is difficult to define an exact storage requirement due to variability in bundle size, 200 MB should be sufficient for most cases. +* Verify the directory's permissions to ensure Redpanda can write to it. By default, Redpanda operates as the `redpanda` user within the `redpanda` group. + +ifndef::env-kubernetes[] +To set a custom debug bundle storage directory, use the `rpk cluster config` command: + +[,bash] +---- +rpk cluster config set debug_bundle_storage_dir +---- + +For example, to store debug bundles in `/var/log/redpanda/debug_bundles`: + +[,bash] +---- +rpk cluster config set debug_bundle_storage_dir /var/log/redpanda/debug_bundles +---- +endif::[] + +ifdef::env-kubernetes[] +To set this property, use the `config.cluster.debug_bundle_storage_dir` field: + +[tabs] +====== +Helm + Operator:: ++ +-- +.`redpanda-cluster.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + chartRef: {} + clusterSpec: + config: + cluster: + debug_bundle_storage_dir: +---- + +For example: + +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + chartRef: {} + clusterSpec: + config: + cluster: + debug_bundle_storage_dir: /var/log/redpanda/debug_bundles +---- + +Apply the changes with: + +[,bash] +---- +kubectl apply -f redpanda-cluster.yaml --namespace +---- + +-- +Helm:: ++ +-- + +Update the `values.yaml` file or use the `--set` flag to specify the property: + +[tabs] +==== +--values:: ++ +[,yaml] +---- +config: + cluster: + debug_bundle_storage_dir: +---- ++ +For example, to store debug bundles in `/var/log/redpanda/debug_bundles`: ++ +[,yaml] +---- +config: + cluster: + debug_bundle_storage_dir: /var/log/redpanda/debug_bundles +---- ++ +Apply the changes with: ++ +[,bash] +---- +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --values values.yaml --reuse-values +---- + +--set:: ++ +```bash +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --set config.cluster.debug_bundle_storage_dir= +``` ++ +For example: ++ +```bash +helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ + --set config.cluster.debug_bundle_storage_dir=/var/log/redpanda/debug_bundles +``` + +==== +-- +====== + +include::troubleshoot:partial$inspect-bundle.adoc[] + +== Contents of the debug bundle + +The debug bundle includes the following files and directories: + +include::reference:partial$bundle-contents.adoc[] + +include::shared:partial$suggested-reading.adoc[] + +- xref:reference:rpk/rpk-debug/rpk-debug-bundle.adoc[`rpk debug bundle`] +- https://stedolan.github.io/jq/manual/#Basicfilters[jq manual^] diff --git a/modules/manage/pages/kubernetes/troubleshooting/k-troubleshoot.adoc b/modules/troubleshoot/partials/errors-and-solutions.adoc similarity index 68% rename from modules/manage/pages/kubernetes/troubleshooting/k-troubleshoot.adoc rename to modules/troubleshoot/partials/errors-and-solutions.adoc index 3f34483fb6..4ae8a3e078 100644 --- a/modules/manage/pages/kubernetes/troubleshooting/k-troubleshoot.adoc +++ b/modules/troubleshoot/partials/errors-and-solutions.adoc @@ -1,269 +1,12 @@ -= Troubleshoot Redpanda in Kubernetes -:description: Find advice on how to diagnose and troubleshoot issues while deploying Redpanda in Kubernetes. -:tags: ["Kubernetes"] -:page-aliases: manage:kubernetes/troubleshooting/troubleshoot.adoc -:page-categories: Management, Troubleshooting -:env-kubernetes: true +ifdef::env-kubernetes+include-categories[] +== Kubernetes-related issues -This topic provides guidance on how to diagnose and troubleshoot issues with Redpanda deployments in Kubernetes. - -== Prerequisites - -Before troubleshooting Redpanda, ensure that Kubernetes isn't the cause of the issue. For information about debugging applications in a Kubernetes cluster, see the https://kubernetes.io/docs/tasks/debug/[Kubernetes documentation^]. - -== Collect all debugging data - -If you're unsure of what is wrong, you can generate a diagnostics bundle that contains a wide range of data to help debug and diagnose a Redpanda cluster or the nodes on which the brokers are running. - -See xref:./k-diagnostics-bundle.adoc[Diagnostics Bundles in Kubernetes]. - -== View Helm chart configuration - -To check the overrides that were applied to your deployment: - -[,bash] ----- -helm get values --namespace ----- - -If you're using the Redpanda Operator, the chart name matches the name of your Redpanda resource. - -To check all the values that were set in the Redpanda Helm chart, including any overrides: - -[,bash] ----- -helm get values --namespace --all ----- - -== View recent events - -To understand the latest events that occurred in your Redpanda cluster's namespace, you can sort events by their creation timestamp: - -[,bash] ----- -kubectl get events --namespace --sort-by='.metadata.creationTimestamp' ----- - -== View Redpanda logs - -Logs are crucial for monitoring and troubleshooting your Redpanda clusters. Redpanda brokers output logs to STDOUT, making them accessible via `kubectl`. - -To access logs for a specific Pod: - -. List all Pods to find the names of those that are running Redpanda brokers: -+ -[source,bash] ----- -kubectl get pods --namespace ----- - -. View logs for a particular Pod by replacing `` with the name of your Pod: -+ -[source,bash] ----- -kubectl logs --namespace ----- -+ -[TIP] -==== -For a comprehensive overview, you can view aggregated logs from all Pods in the StatefulSet: - -[source,bash] ----- -kubectl logs --namespace -l app.kubernetes.io/component=redpanda-statefulset ----- -==== - -=== Change the default log level - -To change the default log level for all Redpanda subsystems, use the `logging.logLevel` configuration. Valid values are `trace`, `debug`, `info`, `warn`, `error`. - -Changing the default log level to `debug` can provide more detailed logs for diagnostics. This logging level increases the volume of generated logs. - -NOTE: To set different log levels for individual subsystems, see <>. - -[tabs] -====== -Helm + Operator:: -+ --- -Apply the new log level: - -.`redpanda-cluster.yaml` -[source,yaml] ----- -apiVersion: cluster.redpanda.com/v1alpha2 -kind: Redpanda -metadata: - name: redpanda -spec: - chartRef: {} - clusterSpec: - logging: - logLevel: debug ----- - -Then, apply this configuration: - -[source,bash] ----- -kubectl apply -f redpanda-cluster.yaml --namespace ----- --- - -Helm:: -+ --- -Choose between using a custom values file or setting values directly: -[tabs] -==== ---values:: -+ -Specify logging settings in `logging.yaml`, then upgrade: -+ -.`logging.yaml` -[source,yaml] ----- -logging: - logLevel: debug ----- -+ -[source,bash] ----- -helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ - --values logging.yaml --reuse-values ----- ---set:: -+ -Directly set the log level during upgrade: -+ -[source,bash] ----- -helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ - --set logging.logLevel=debug ----- -==== --- -====== - -After applying these changes, verify the log level by <> for the Redpanda Pods. - -=== Override the default log level for Redpanda subsystems - -You can override the log levels for individual subsystems, such as `rpc` and `kafka`, for more detailed logging control. Overrides exist for the entire length of the running Redpanda process. - -TIP: To temporarily override the log level for individual subsystems, you can use the xref:reference:rpk/rpk-redpanda/rpk-redpanda-admin-config-log-level-set.adoc[`rpk-redpanda-admin-config-log-level-set`] command. - -. List all available subsystem loggers: -+ -[source,bash] ----- -kubectl exec -it --namespace -c redpanda -- rpk redpanda start --help-loggers ----- - -. Set the log level for one or more subsystems. In this example, the `rpc` and `kafka` subsystem loggers are set to `debug`. -+ -[tabs] -====== -Helm + Operator:: -+ --- -Apply the new log level: - -.`redpanda-cluster.yaml` -[source,yaml] ----- -apiVersion: cluster.redpanda.com/v1alpha2 -kind: Redpanda -metadata: - name: redpanda -spec: - chartRef: {} - clusterSpec: - statefulset: - additionalRedpandaCmdFlags: - - '--logger-log-level=rpc=debug:kafka=debug' ----- - -Then, apply this configuration: - -[source,bash] ----- -kubectl apply -f redpanda-cluster.yaml --namespace ----- --- - -Helm:: -+ --- -Choose between using a custom values file or setting values directly: -[tabs] -==== ---values:: -+ -Specify logging settings in `logging.yaml`, then upgrade: -+ -.`logging.yaml` -[source,yaml] ----- -statefulset: - additionalRedpandaCmdFlags: - - '--logger-log-level=rpc=debug:kafka=debug' ----- -+ -[source,bash] ----- -helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ - --values logging.yaml --reuse-values ----- ---set:: -+ -Directly set the log level during upgrade: -+ -[source,bash] ----- -helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ - --set statefulset.additionalRedpandaCmdFlags="{--logger-log-level=rpc=debug:kafka=debug}" ----- -==== --- -====== - -Overriding the log levels for specific subsystems provides enhanced visibility into Redpanda's internal operations, facilitating better debugging and monitoring. - -== View Redpanda Operator logs - -To learn what's happening with the Redpanda Operator and the associated Redpanda resources, you can inspect a combination of Kubernetes events and the resource manifests. By monitoring these events and resources, you can troubleshoot any issues that arise during the lifecycle of a Redpanda deployment. - -[,bash] ----- -kubectl logs -l app.kubernetes.io/name=operator -c manager --namespace ----- - -=== Inspect Helm releases - -The Redpanda Operator uses Flux to deploy the Redpanda Helm chart. By inspecting the `helmreleases.helm.toolkit.fluxcd.io` resource, you can get detailed information about the Helm installation process for your Redpanda resource: - -[,bash] ----- -kubectl get helmreleases.helm.toolkit.fluxcd.io -o yaml --namespace ----- - -To check the Redpanda resource: - -[,bash] ----- -kubectl get redpandas.cluster.redpanda.com -o yaml --namespace ----- - -In both the HelmRelease and the Redpanda resource, the condition section reveals the ongoing status of the Helm installation. These conditions provide information on the success, failure, or pending status of various operations. - -== Troubleshoot known issues - -This section describes issues you might encounter while deploying Redpanda in Kubernetes and explains how to troubleshoot them. +This section addresses common issues that may occur when deploying Redpanda in a Kubernetes environment. +endif::[] //tag::deployment[] //tag::deployment-helm-release-not-ready[] +ifdef::env-kubernetes[] === HelmRelease is not ready If you are using the Redpanda Operator, you may see the following message while waiting for a Redpanda custom resource to be deployed: @@ -337,51 +80,6 @@ flux resume helmrelease --namespace ``` //end::deployment-retries-exhausted[] -//tag::crashloopbackoff[] -=== Crash loop backoffs - -If a broker crashes after startup, or gets stuck in a crash loop, it could produce progressively more stored state that uses additional disk space and takes more time for each restart to process. - -To prevent infinite crash loops, the Redpanda Helm chart sets the `crash_loop_limit` node property to 5. The crash loop limit is the number of consecutive crashes that can happen within one hour of each other. After Redpanda reaches this limit, it will not start until its internal consecutive crash counter is reset to zero. In Kubernetes, the Pod running Redpanda remains in a `CrashLoopBackoff` state until its internal consecutive crash counter is reset to zero. - -To troubleshoot a crash loop backoff: - -. Check the Redpanda logs from the most recent crashes: -+ -[,bash] ----- -kubectl logs --namespace ----- -+ -NOTE: Kubernetes retains logs only for the current and the previous instance of a container. This limitation makes it difficult to access logs from earlier crashes, which may contain vital clues about the root cause of the issue. Given these log retention limitations, setting up a centralized logging system is crucial. Systems such as https://grafana.com/docs/loki/latest/[Loki] or https://www.datadoghq.com/product/log-management/[Datadog] can capture and store logs from all containers, ensuring you have access to historical data. - -. Resolve the issue that led to the crash loop backoff. - -. Reset the crash counter to zero to allow Redpanda to restart. You can do any of the following to reset the counter: -+ -- Update the redpanda.yaml configuration file. You can make changes to any of the following sections in the Redpanda Helm chart to trigger an update: -* `config.cluster` -* `config.node` -* `config.tunable` - -- Delete the `startup_log` file in the broker's data directory. -+ -[,bash] ----- -kubectl exec --namespace -- rm /var/lib/redpanda/data/startup_log ----- -+ -NOTE: It might be challenging to execute this command within a Pod that is in a `CrashLoopBackoff` state due to the limited time during which the Pod is available before it restarts. Wrapping the command in a loop might work. - -- Wait one hour since the last crash. The crash counter resets after one hour. - -To avoid future crash loop backoffs and manage the accumulation of small segments effectively: - -* xref:manage:kubernetes/monitoring/k-monitor-redpanda.adoc[Monitor] the size and number of segments regularly. -* Optimize your Redpanda configuration for segment management. -* Consider implementing xref:manage:kubernetes/storage/tiered-storage/k-tiered-storage.adoc[Tiered Storage] to manage data more efficiently. -//end::crashloopbackoff[] - //tag::deployment-pod-pending[] === StatefulSet never rolls out @@ -686,9 +384,137 @@ After clearing the `pending-rollback` state: * *Retry the upgrade*: Restart the upgrade process. You should investigate the initial failure to avoid getting into the `pending-rollback` state again. * *Perform a rollback*: If you need to roll back to a previous release, use `helm rollback ` to revert to a specific, stable release version. //end::pending-rollback[] + +endif::[] + +ifdef::include-categories[] +== Deployment issues + +This section addresses common deployment issues encountered during Redpanda setup or upgrades. +endif::[] + +//tag::crashloopbackoff[] +ifdef::env-kubernetes[] +=== Crash loop backoffs + +If a broker crashes after startup, or gets stuck in a crash loop, it could produce progressively more stored state that uses additional disk space and takes more time for each restart to process. + +To prevent infinite crash loops, the Redpanda Helm chart sets the `crash_loop_limit` node property to 5. The crash loop limit is the number of consecutive crashes that can happen within one hour of each other. After Redpanda reaches this limit, it will not start until its internal consecutive crash counter is reset to zero. In Kubernetes, the Pod running Redpanda remains in a `CrashLoopBackoff` state until its internal consecutive crash counter is reset to zero. + +To troubleshoot a crash loop backoff: + +. Check the Redpanda logs from the most recent crashes: ++ +[,bash] +---- +kubectl logs --namespace +---- ++ +NOTE: Kubernetes retains logs only for the current and the previous instance of a container. This limitation makes it difficult to access logs from earlier crashes, which may contain vital clues about the root cause of the issue. Given these log retention limitations, setting up a centralized logging system is crucial. Systems such as https://grafana.com/docs/loki/latest/[Loki] or https://www.datadoghq.com/product/log-management/[Datadog] can capture and store logs from all containers, ensuring you have access to historical data. + +. Resolve the issue that led to the crash loop backoff. + +. Reset the crash counter to zero to allow Redpanda to restart. You can do any of the following to reset the counter: ++ +- Make changes to any of the following sections in the Redpanda Helm chart to trigger an update: ++ +-- +* `config.node` +* `config.tunable` +-- ++ +For example: ++ +```yaml +config: + node: + crash_loop_limit: +``` + +- Delete the `startup_log` file in the broker's data directory. ++ +[,bash] +---- +kubectl exec --namespace -- rm /var/lib/redpanda/data/startup_log +---- ++ +NOTE: It might be challenging to execute this command within a Pod that is in a `CrashLoopBackoff` state due to the limited time during which the Pod is available before it restarts. Wrapping the command in a loop might work. + +- Wait one hour since the last crash. The crash counter resets after one hour. + +To avoid future crash loop backoffs and manage the accumulation of small segments effectively: + +* xref:manage:kubernetes/monitoring/k-monitor-redpanda.adoc[Monitor] the size and number of segments regularly. +* Optimize your Redpanda configuration for segment management. +* Consider implementing xref:manage:kubernetes/storage/tiered-storage/k-tiered-storage.adoc[Tiered Storage] to manage data more efficiently. + +endif::[] +//end::crashloopbackoff[] + +=== A Redpanda Enterprise Edition license is required + +During a Redpanda upgrade, if enterprise features are enabled and a valid Enterprise Edition license is missing, Redpanda logs a warning and aborts the upgrade process on the first broker. This issue prevents a successful upgrade. + +[.no-copy] +---- +A Redpanda Enterprise Edition license is required to use the currently enabled features. To apply your license, downgrade this broker to the pre-upgrade version and provide a valid license key via rpk using 'rpk cluster license set ', or via Redpanda Console. To request an enterprise license, please visit . To try Redpanda Enterprise for 30 days, visit . For more information, see . +---- + +If you encounter this message, follow these steps to recover: + +ifdef::env-kubernetes[] +. xref:upgrade:k-rolling-upgrade.adoc#roll-back[Roll back the affected broker to the original version]. +endif::[] +ifndef::env-kubernetes[] +. Roll back the affected broker to the original version. +endif::[] +. Do one of the following: +- xref:get-started:licensing/add-license-redpanda/index.adoc[Apply a valid Redpanda Enterprise Edition license] to the cluster. +- Disable enterprise features. ++ +If you do not have a valid license and want to proceed without using enterprise features, you can disable the enterprise features in your Redpanda configuration. + +. Retry the upgrade. + //end::deployment[] +ifdef::include-categories[] +ifdef::env-kubernetes[] +== Networking issues + +This section provides insights into diagnosing network-related errors, such as connection timeouts, DNS misconfigurations, and network stability. +endif::[] +endif::[] + +// tag::networking[] +ifdef::env-kubernetes[] +=== I/O timeout + +This error appears when your worker nodes are unreachable through the given address. + +Check the following: + +* The address and port are correct. +* Your DNS records point to addresses that resolve to your worker nodes. + +endif::[] +//end::networking[] + +ifdef::include-categories[] +== TLS issues + +This section covers common TLS errors, their causes, and solutions, including certificate issues and correct client configuration. + +endif::[] + //tag::tls[] +ifdef::env-kubernetes[] +=== Redpanda not applying TLS changes + +include::manage:partial$kubernetes/tls-update-note.adoc[] + +endif::[] + === Invalid large response size This error appears when your cluster is configured to use TLS, but you don't specify that you are connecting over TLS. @@ -698,14 +524,15 @@ This error appears when your cluster is configured to use TLS, but you don't spe unable to request metadata: invalid large response size 352518912 > limit 104857600; the first three bytes received appear to be a tls alert record for TLS v1.2; is this a plaintext connection speaking to a tls endpoint? ---- -If you're using rpk, ensure to add the `-X tls.enabled` flag, and any other necessary TLS flags such as the TLS certificate: +If you're using `rpk`, ensure to add the `-X tls.enabled` flag, and any other necessary TLS flags such as the TLS certificate: [,bash] ---- -kubectl exec -c redpanda --namespace -- rpk cluster info -X brokers=.: -X tls.enabled=true +ifdef::env-kubernetes[kubectl exec -c redpanda --namespace -- \] +rpk cluster info -X tls.enabled=true ---- -For all available flags, see the xref:reference:rpk/index.adoc[rpk command reference]. +For all available flags, see the xref:reference:rpk/rpk-x-options.adoc[`rpk` options reference]. === Malformed HTTP response @@ -716,9 +543,9 @@ This error appears when a cluster has TLS enabled, and you try to access the adm Retrying POST for error: Post "http://127.0.0.1:9644/v1/security/users": net/http: HTTP/1.x transport connection broken: malformed HTTP response "\x15\x03\x03\x00\x02\x02" ---- -If you're using rpk, ensure to include the TLS flags. +If you're using `rpk`, ensure to include the TLS flags. -For all available flags, see the xref:reference:rpk/index.adoc[rpk command reference]. +For all available flags, see the xref:reference:rpk/rpk-x-options.adoc[`rpk` options reference]. === x509: certificate signed by unknown authority @@ -729,7 +556,9 @@ Check the following: - Ensure you have installed the root CA certificate correctly on your local system. - If using a self-signed certificate, ensure it is properly configured and included in your system's trust store. - If you are using a certificate issued by a CA, ensure the issuing CA is included in your system's trust store. +ifdef::env-kubernetes[] - If you are using cert-manager, ensure it is correctly configured and running properly. +endif::[] - Check the validity of your certificates. They might have expired. === x509: certificate is not valid for any names @@ -746,56 +575,52 @@ This error appears if you are using a CA certificate when you try to establish a unable to request metadata: unable to dial: x509: cannot validate certificate for 127.0.0.1 because it doesn't contain any IP SANs ``` -To fix this error, you must either specify the public domain or use self-signed certificates: +To fix this error, you must either specify the URL with a public domain or use self-signed certificates: -```bash -kubectl exec redpanda-0 -c redpanda --namespace -- \ - rpk cluster info \ - -X brokers=.: \ - -X tls.enabled=true -``` - -=== Redpanda not applying TLS changes +[,bash] +---- +ifdef::env-kubernetes[kubectl exec redpanda-0 -c redpanda --namespace -- \] +rpk cluster info \ +-X brokers=: \ +-X tls.enabled=true +---- -include::manage:partial$kubernetes/tls-update-note.adoc[] //end::tls[] -//tag::networking[] -=== I/O timeout +ifdef::include-categories[] +== SASL issues -This error appears when your worker nodes are unreachable through the given address. - -Check the following: +This section addresses errors related to SASL (Simple Authentication and Security Layer), focusing on connection and authentication problems. -* The address and port are correct. -* Your DNS records point to addresses that resolve to your worker nodes. +endif::[] -//end::networking[] //tag::sasl[] -=== Is SASL missing? +ifdef::env-kubernetes[] +=== Unable to continue with update: Secret -This error appears when you try to interact with a cluster that has SASL enabled without passing a user's credentials. +When you use a YAML list to specify superusers, the Helm chart creates a Secret using the value of `auth.sasl.secretRef` as the Secret's name, and stores those superusers in the Secret. If the Secret already exists in the namespace when you deploy Redpanda, the following error is displayed: [.no-copy] ---- -unable to request metadata: broker closed the connection immediately after a request was issued, which happens when SASL is required but not provided: is SASL missing? +Error: UPGRADE FAILED: rendered manifests contain a resource that already exists. Unable to continue with update: Secret ---- -If you're using rpk, ensure to specify the `-X user`, `-X pass`, and `-X sasl.mechanism` flags. +To fix this error, ensure that you use only one of the following methods to create superusers: -For all available flags, see the xref:reference:rpk/index.adoc[rpk command reference]. +- `auth.sasl.secretRef` +- `auth.sasl.users` -=== Unable to continue with update: Secret +endif::[] +=== Is SASL missing? -When you use a YAML list to specify superusers, the Helm chart creates a Secret using the value of `auth.sasl.secretRef` as the Secret's name, and stores those superusers in the Secret. If the Secret already exists in the namespace when you deploy Redpanda, the following error is displayed: +This error appears when you try to interact with a cluster that has SASL enabled without passing a user's credentials. [.no-copy] ---- -Error: UPGRADE FAILED: rendered manifests contain a resource that already exists. Unable to continue with update: Secret +unable to request metadata: broker closed the connection immediately after a request was issued, which happens when SASL is required but not provided: is SASL missing? ---- -To fix this error, ensure that you use only one of the following methods to create superusers: +If you're using `rpk`, ensure to specify the `-X user`, `-X pass`, and `-X sasl.mechanism` flags. -- `auth.sasl.secretRef` -- `auth.sasl.users` +For all available flags, see the xref:reference:rpk/rpk-x-options.adoc[`rpk` options reference]. //end::sasl[] diff --git a/modules/manage/pages/kubernetes/troubleshooting/k-diagnostics-bundle.adoc b/modules/troubleshoot/partials/inspect-bundle.adoc similarity index 72% rename from modules/manage/pages/kubernetes/troubleshooting/k-diagnostics-bundle.adoc rename to modules/troubleshoot/partials/inspect-bundle.adoc index cf9b40aee2..541d929f2d 100644 --- a/modules/manage/pages/kubernetes/troubleshooting/k-diagnostics-bundle.adoc +++ b/modules/troubleshoot/partials/inspect-bundle.adoc @@ -1,156 +1,8 @@ -= Diagnostics Bundles in Kubernetes -:description: Use the diagnostics bundle to debug issues yourself, or you can send it to the Redpanda support team to help resolve your issue. -:tags: ["Kubernetes"] -:page-aliases: manage:kubernetes/troubleshooting/diagnostics-bundle.adoc -:page-categories: Management, Troubleshooting -:env-kubernetes: true +== Inspect the debug bundle -A diagnostics bundle is a ZIP file with data that can help debug and diagnose issues with a Redpanda cluster, a broker, or the machines on which the brokers are running. You can use this file to debug issues yourself, or you can send it to the Redpanda support team to help resolve your issue. +After downloading the debug bundle files, you can inspect the contents to debug your cluster. This section provides some useful data points to check while troubleshooting. -== Prerequisites - -Most files in the diagnostics bundle are JSON files. To make it easier to read these files, this guide uses jq. To install jq, see the https://stedolan.github.io/jq/download/[jq downloads page^]. - -These examples use the default values in Helm chart. If you've customized the Helm chart, you may need to provide custom values and/or flags. - -== Generate a diagnostics bundle - -. Create a ClusterRole to allow Redpanda to collect information from the Kubernetes API: -+ -[tabs] -====== -Helm + Operator:: -+ --- -.`redpanda-cluster.yaml` -[,yaml] ----- -apiVersion: cluster.redpanda.com/v1alpha2 -kind: Redpanda -metadata: - name: redpanda -spec: - chartRef: {} - clusterSpec: - serviceAccount: - create: true - rbac: - enabled: true ----- - -```bash -kubectl apply -f redpanda-cluster.yaml --namespace -``` - -NOTE: You must deploy the Redpanda Operator with the `--set rbac.createRPKBundleCRs=true` flag to give it the required ClusterRoles. - --- -Helm:: -+ --- -[tabs] -==== ---values:: -+ -.`serviceaccount.yaml` -[,yaml] ----- -serviceAccount: - create: true -rbac: - enabled: true ----- -+ -```bash -helm upgrade --install redpanda redpanda/redpanda --namespace redpanda --create-namespace \ - --values serviceaccount.yaml --reuse-values -``` - ---set:: -+ -```bash -helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ - --set serviceAccount.create=true \ - --set rbac.enabled=true -``` -==== --- -====== -+ -[TIP] -==== -If you aren't using the Helm chart, you can create the ClusterRole manually: - -```bash -kubectl create clusterrolebinding redpanda --clusterrole=view --serviceaccount=redpanda:default -``` -==== - -. Execute the xref:reference:rpk/rpk-debug/rpk-debug-bundle.adoc[`rpk debug bundle`] command inside a Pod container that's running a Redpanda broker. -+ -In this example, the command is executed on a Pod called `redpanda-0`. -+ -```bash -kubectl exec -it --namespace redpanda-0 -c redpanda -- rpk debug bundle -``` -+ -If you have an upload URL from the Redpanda support team, -provide it in the `--upload-url` flag to upload your diagnostics bundle to Redpanda. -+ -```bash -kubectl exec -it --namespace redpanda-0 -c redpanda -- rpk debug bundle \ - --upload-url \ - --namespace -``` -+ -Example output: -+ -[.no-copy] ----- -Creating bundle file... - -Debug bundle saved to "/var/lib/redpanda/1675440652-bundle.zip" ----- - -. On your host machine, make a directory in which to save the diagnostics bundle: -+ -```bash -mkdir diagnostics-bundle -``` - -. Copy the diagnostics bundle from the Pod to your host machine: -+ -Replace `` with the name of your ZIP file. -+ -```bash -kubectl cp redpanda/redpanda-0:/var/lib/redpanda/ diagnostics-bundle/ -``` - -. Unzip the file on your host machine. -+ -```bash -cd diagnostics-bundle -unzip -``` - -. Remove the diagnostics bundle from the Redpanda container: -+ -```bash -kubectl exec redpanda-0 -c redpanda --namespace -- rm /var/lib/redpanda/ -``` - - -When you've finished troubleshooting, remove the diagnostics bundle from your host machine: - -```bash -rm -r diagnostics-bundle -``` - -For a description of the files and directories, see <>. - -== Inspect the diagnostics bundle - -This section provides some useful data points to check while troubleshooting. +Most files in the debug bundle are JSON files. To make it easier to read these files, this section uses jq. To install jq, see the https://stedolan.github.io/jq/download/[jq downloads page^]. === View the version of Redpanda on all brokers @@ -160,11 +12,11 @@ cat admin/brokers.json | jq '.[] | .version' Example output: -[,json,role=no-copy] +[,json,role=no-copy,subs=attributes+] ---- -"v23.1.1" -"v23.1.1" -"v23.1.1" +"{latest-redpanda-tag}" +"{latest-redpanda-tag}" +"{latest-redpanda-tag}" ---- === View the maintenance status of all brokers @@ -254,7 +106,7 @@ cat admin/cluster_config.json | jq ``` ==== -=== Check Enterprise license keys +=== Check Enterprise Edition license keys ```bash cat admin/license.json | jq @@ -279,12 +131,19 @@ cat admin/license.json | jq === View metadata about the Redpanda data directory +You can inspect the size of directories in the Redpanda data directory and identify anomalies using a pre-generated report. This is useful for troubleshooting issues such as imbalances in partition sizes or unexpected data growth. + To check the size of the directories and look for anomalies: ```bash -cat du.txt +cat utils/du.txt ``` +The `du.txt` file provides information about the size of each directory. Anomalies to look for include: + +- One partition of a topic being significantly larger than others. +- Directories containing more data than expected for a specific topic or partition. + .Example output [%collapsible] ==== @@ -627,14 +486,21 @@ cat kafka.json | jq '.[1:]' === View the Redpanda logs +ifdef::env-kubernetes[] ```bash cat logs/redpanda-0.txt # logs/redpanda-1.txt logs/redpanda-2.txt ``` +endif::[] +ifndef::env-kubernetes[] +```bash +cat redpanda.log +``` +endif::[] === Check for clock drift ```bash -cat ntp.txt | jq +cat utils/ntp.txt | jq ``` Use the output to check for clock drift. For details about how NTP works, see the http://www.ntp.org/ntpfaq/NTP-s-algo.htm[NTP documentation^]. @@ -654,6 +520,7 @@ Use the output to check for clock drift. For details about how NTP works, see th ``` ==== +ifdef::env-kubernetes[] === View Kubernetes manifests ```bash @@ -678,13 +545,4 @@ k8s ``` ==== -== Contents of the diagnostics bundle - -The diagnostics bundle includes the following files and directories: - -include::reference:partial$bundle-contents-k8s.adoc[] - -include::shared:partial$suggested-reading.adoc[] - -- xref:reference:rpk/rpk-debug/rpk-debug-bundle.adoc[`rpk debug bundle`] -- https://stedolan.github.io/jq/manual/#Basicfilters[jq manual^] +endif::[] \ No newline at end of file diff --git a/modules/upgrade/pages/k-rolling-upgrade.adoc b/modules/upgrade/pages/k-rolling-upgrade.adoc index 30194be9c1..f90f5db935 100644 --- a/modules/upgrade/pages/k-rolling-upgrade.adoc +++ b/modules/upgrade/pages/k-rolling-upgrade.adoc @@ -20,10 +20,6 @@ include::partial$incompat-changes.adoc[] include::partial$rolling-upgrades/restart-impact.adoc[leveloffset=+1] -== Incompatible changes - -Patch releases in https://github.com/redpanda-data/redpanda/discussions/9522[22.3.14^] and https://github.com/redpanda-data/redpanda/discussions/9523[23.1.2^] changed the behavior when remote read is disabled and the requested Raft term falls below the local log's beginning. In earlier versions, Redpanda returned an offset -1. With the patch, when you request a value older than the lowest offset, Redpanda returns the lowest offset, not -1. - == Check your current Redpanda version Before you perform a rolling upgrade: @@ -41,12 +37,12 @@ kubectl exec --namespace -c redpanda -- \ [%collapsible] ==== The Redpanda version for each broker is listed under `BROKER-VERSION`. -[.no-copy] +[role="no-copy",subs="attributes+"] ``` NODE-ID BROKER-VERSION -0 v22.2.10 -1 v22.2.10 -2 v22.2.10 +0 {latest-redpanda-tag} +1 {latest-redpanda-tag} +2 {latest-redpanda-tag} ``` ==== - xref:upgrade:k-compatibility.adoc[Review the Kubernetes compatibility matrix] to find out if you need to upgrade the Helm chart or the Redpanda Operator to use your chosen version of Redpanda. @@ -56,6 +52,12 @@ If your current version of Redpanda is more than one feature release behind the + To find information about what has changed between Redpanda versions, check the https://github.com/redpanda-data/redpanda/releases[release notes^]. +== Check license status + +If you use xref:get-started:licensing/overview.adoc#self-managed[enterprise features] of Redpanda, make sure that you have a valid Enterprise Edition license key. From version 24.3, Redpanda fails to upgrade to a new feature release if you have enterprise features enabled without a valid license key. + +include::get-started:partial$licensing/check-license.adoc[] + == Prepare your cluster Before you upgrade, you must make sure that your cluster is in a healthy state and that your topics are configured to limit outages during the upgrade process. @@ -258,7 +260,7 @@ which means all Pods in the StatefulSet are restarted in reverse-ordinal order. Helm + Operator:: + -- -The Redpanda Operator rolls back automatically after three failed attempts to upgrade the cluster. +Revert the change that you made to the version and reapply the manifest. -- Helm:: + @@ -311,7 +313,7 @@ Leaderless partitions: [] == Troubleshooting -include::manage:kubernetes/troubleshooting/k-troubleshoot.adoc[tags=deployment] +include::troubleshoot:partial$errors-and-solutions.adoc[tags=deployment] include::shared:partial$suggested-reading.adoc[] diff --git a/modules/upgrade/pages/migrate/kubernetes/helm-to-operator.adoc b/modules/upgrade/pages/migrate/kubernetes/helm-to-operator.adoc index ff19b5a78a..c1190f752c 100644 --- a/modules/upgrade/pages/migrate/kubernetes/helm-to-operator.adoc +++ b/modules/upgrade/pages/migrate/kubernetes/helm-to-operator.adoc @@ -181,7 +181,7 @@ After completing these steps, the Redpanda Operator is no longer managing your H While the deployment process can sometimes take a few minutes, a prolonged 'not ready' status may indicate an issue. -include::manage:kubernetes/troubleshooting/k-troubleshoot.adoc[tags=deployment] +include::troubleshoot:partial$errors-and-solutions.adoc[tags=deployment] For more troubleshooting steps, see xref:manage:kubernetes/troubleshooting/k-troubleshoot.adoc[Troubleshoot Redpanda in Kubernetes]. diff --git a/modules/upgrade/pages/rolling-upgrade.adoc b/modules/upgrade/pages/rolling-upgrade.adoc index 1cd25b15af..55a2328995 100644 --- a/modules/upgrade/pages/rolling-upgrade.adoc +++ b/modules/upgrade/pages/rolling-upgrade.adoc @@ -21,6 +21,8 @@ include::partial$rolling-upgrades/upgrade-limitations.adoc[] include::partial$incompat-changes.adoc[] +include::partial$rolling-upgrades/restart-impact.adoc[leveloffset=+1] + == Find a new version Before you upgrade, find out which Redpanda version you are currently running, whether you can upgrade straight to the new version, and what's changed since your original version. To find your current version, run: @@ -66,7 +68,7 @@ Example output: [,bash,subs="attributes+"] ---- -v{full-version} (rev {latest-release-commit}) +{latest-redpanda-tag} (rev {latest-release-commit}) ---- [NOTE] @@ -81,7 +83,11 @@ curl -s 'https://hub.docker.com/v2/repositories/redpandadata/redpanda/tags/?orde Check the https://github.com/redpanda-data/redpanda/releases[release notes^] to find information about what has changed between Redpanda versions. -include::partial$rolling-upgrades/restart-impact.adoc[leveloffset=+1] +== Check license status + +If you use xref:get-started:licensing/overview.adoc#self-managed[enterprise features] of Redpanda, make sure that you have a valid Enterprise Edition license key. Redpanda fails to upgrade to a new feature release if you have enterprise features enabled without a valid license key. + +include::get-started:partial$licensing/check-license.adoc[] == Perform a rolling upgrade @@ -99,6 +105,10 @@ include::partial$rolling-upgrades/disable-maintenance-mode.adoc[] include::partial$rolling-upgrades/post-upgrade-tasks.adoc[] +== Troubleshooting + +include::troubleshoot:partial$errors-and-solutions.adoc[tags=deployment] + include::shared:partial$suggested-reading.adoc[] * xref:manage:monitoring.adoc[] diff --git a/modules/upgrade/partials/rolling-upgrades/upgrade-limitations.adoc b/modules/upgrade/partials/rolling-upgrades/upgrade-limitations.adoc index 719c1c0d00..c81a51ba60 100644 --- a/modules/upgrade/partials/rolling-upgrades/upgrade-limitations.adoc +++ b/modules/upgrade/partials/rolling-upgrades/upgrade-limitations.adoc @@ -1,11 +1,12 @@ == Limitations -The following limitations ensure a smooth transition between versions and help to maintain the stability of your cluster. +The following limitations ensure a smooth transition between versions and help maintain your cluster's stability. * *Broker upgrades*: ** New features are enabled only after upgrading all brokers in the cluster. ** You can upgrade only one feature release at a time, for example from 22.2 to 22.3. Skipping feature releases is not supported. +** If you have xref:get-started:licensing/overview.adoc[enterprise features] enabled without a valid license, Redpanda fails to upgrade to new feature releases, starting from version 24.3. * *Rollbacks*: You can roll back to the original version only if at least one broker is still running the original version (not yet upgraded) and the cluster hasn't yet restarted. @@ -19,4 +20,4 @@ CAUTION: In a mixed-version state, the cluster could run out of disk space. If y * *Remote Read Replicas*: Upgrade the Remote Read Replica cluster first, ensuring it's on the same version as the origin cluster or one feature release ahead of the origin cluster. -When upgrading to Redpanda 23.2, metadata from object storage is not synchronized until all brokers in the cluster are upgraded. If you need to force a mixed-version cluster to sync read replicas, transfer partition leadership to brokers running the original version. \ No newline at end of file +When upgrading to Redpanda 23.2, metadata from object storage is not synchronized until all brokers in the cluster are upgraded. If you need to force a mixed-version cluster to sync read replicas, transfer partition leadership to brokers running the original version. diff --git a/package-lock.json b/package-lock.json index d8327120d4..44c3fa0e5f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2773,9 +2773,10 @@ } }, "node_modules/@redpanda-data/docs-extensions-and-macros": { - "version": "3.9.1", - "resolved": "https://registry.npmjs.org/@redpanda-data/docs-extensions-and-macros/-/docs-extensions-and-macros-3.9.1.tgz", - "integrity": "sha512-qEs/4KDsFv+P8VZSD41P5VS9oeYtH8MOKJwu6OIVypzBpbBUg6f1MbqaLoy9F8Halh/CIEbwZwpCe72rDijXcg==", + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/@redpanda-data/docs-extensions-and-macros/-/docs-extensions-and-macros-3.10.0.tgz", + "integrity": "sha512-H50KCLTBg+qiZdD+k+PHv1m6BDJPAjBLOAdXZBt36oGZucfwCEqwW/6pUVxbbjuHmvfL1MTfDPgqfZRLJpNpRQ==", + "license": "ISC", "dependencies": { "@asciidoctor/tabs": "^1.0.0-beta.6", "@octokit/core": "^6.1.2", diff --git a/package.json b/package.json index 05cc1d7044..8929c32548 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "serve": "wds --node-resolve --open / --watch --root-dir docs --port 5002", "start": "cross-env-shell LIVERELOAD=true npx gulp", "test-console-docs": "cd setup-tests && npx doc-detective runTests --input ../modules/console/pages/quickstart.adoc -l debug", + "test-get-started-docs": "cd setup-tests && npx doc-detective runTests --input ../modules/get-started/pages/quick-start.adoc -l debug", "test-docs": "cd setup-tests && npx doc-detective runTests --input ../modules -l debug" }, "dependencies": { diff --git a/setup-tests/fetch-versions-and-rpk.json b/setup-tests/fetch-versions-and-rpk.json index 5e0ee1efb7..63b9d3932e 100644 --- a/setup-tests/fetch-versions-and-rpk.json +++ b/setup-tests/fetch-versions-and-rpk.json @@ -14,7 +14,11 @@ "setVariables": [ { "name": "REDPANDA_CONSOLE_VERSION", - "regex": ".*" + "regex": "(?<=CONSOLE_VERSION=)(.*)" + }, + { + "name": "CONSOLE_DOCKER_REPO", + "regex": "(?<=CONSOLE_DOCKER_REPO=)(.*)" } ] }, @@ -24,7 +28,11 @@ "setVariables": [ { "name": "REDPANDA_VERSION", - "regex": ".*" + "regex": "(?<=REDPANDA_VERSION=)(.*)" + }, + { + "name": "REDPANDA_DOCKER_REPO", + "regex": "(?<=REDPANDA_DOCKER_REPO=)(.*)" } ] }, diff --git a/setup-tests/redpanda-versions/fetch-console.js b/setup-tests/redpanda-versions/fetch-console.js index 262a25acfe..6cf7590648 100644 --- a/setup-tests/redpanda-versions/fetch-console.js +++ b/setup-tests/redpanda-versions/fetch-console.js @@ -1,10 +1,32 @@ -const semver = require("semver"); +// Import the version fetcher module +const GetLatestConsoleVersion = require('../../node_modules/@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/get-latest-console-version.js'); +const yaml = require('../../node_modules/js-yaml/index.js'); +const fs = require('fs'); + const owner = 'redpanda-data'; const repo = 'console'; +const CONSOLE_DOCKER_REPO = 'console' + +function getPrereleaseFromAntora() { + try { + const fileContents = fs.readFileSync('../antora.yml', 'utf8'); + const antoraConfig = yaml.load(fileContents); + return antoraConfig.prerelease === true; + } catch (error) { + console.error("Error reading antora.yml:", error); + return false; + } +} + +// Set beta based on the prerelease field in antora.yml or fallback to environment variable +const beta = getPrereleaseFromAntora() +// GitHub Octokit initialization async function loadOctokit() { const { Octokit } = await import('@octokit/rest'); - if (!process.env.REDPANDA_GITHUB_TOKEN) return new Octokit() + if (!process.env.REDPANDA_GITHUB_TOKEN) { + return new Octokit(); + } return new Octokit({ auth: process.env.REDPANDA_GITHUB_TOKEN, }); @@ -13,28 +35,26 @@ async function loadOctokit() { (async () => { try { const github = await loadOctokit(); - // Fetch the latest 10 releases - const releases = await github.rest.repos.listReleases({ - owner, - repo, - per_page: 10, - }); - // Filter valid semver tags and sort them - const sortedReleases = releases.data - .map(release => release.tag_name.replace(/^v/, '')) - .filter(tag => semver.valid(tag)) - // Sort in descending order to get the highest version first - .sort(semver.rcompare); - - if (sortedReleases.length > 0) { - console.log(sortedReleases[0]); - } else { - console.log("No valid semver releases found."); - process.exit(1) + const results = await Promise.allSettled([ + GetLatestConsoleVersion(github, owner, repo), + ]); + const LatestConsoleVersion = results[0].status === 'fulfilled' ? results[0].value : null; + if (!LatestConsoleVersion) { + throw new Error('Failed to fetch the latest Redpanda version'); } + // Determine the release version based on the beta flag, with a fallback to stable release if RC is null + const latestConsoleReleaseVersion = beta + ? (LatestConsoleVersion.latestBetaRelease + ? LatestConsoleVersion.latestBetaRelease + : `${LatestConsoleVersion.latestStableRelease}`) + : `${LatestConsoleVersion.latestStableRelease}`; + + // Print both version and Docker repo for Doc Detective to capture + console.log(`CONSOLE_VERSION=${latestConsoleReleaseVersion}`); + console.log(`CONSOLE_DOCKER_REPO=${CONSOLE_DOCKER_REPO}`); } catch (error) { console.error(error); - process.exit(1) + process.exit(1); } -})() \ No newline at end of file +})(); \ No newline at end of file diff --git a/setup-tests/redpanda-versions/fetch-redpanda.js b/setup-tests/redpanda-versions/fetch-redpanda.js index 514d346d0a..99fc5b0a01 100644 --- a/setup-tests/redpanda-versions/fetch-redpanda.js +++ b/setup-tests/redpanda-versions/fetch-redpanda.js @@ -1,10 +1,36 @@ +// Import the version fetcher module +const GetLatestRedpandaVersion = require('../../node_modules/@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/get-latest-redpanda-version.js'); +const yaml = require('../../node_modules/js-yaml/index.js'); +const fs = require('fs'); + // Fetch the latest release version from GitHub const owner = 'redpanda-data'; +function getPrereleaseFromAntora() { + try { + const fileContents = fs.readFileSync('../antora.yml', 'utf8'); + const antoraConfig = yaml.load(fileContents); + return antoraConfig.prerelease === true; + } catch (error) { + console.error("Error reading antora.yml:", error); + return false; + } +} + +// Set beta based on the prerelease field in antora.yml or fallback to environment variable +const beta = getPrereleaseFromAntora() +// Conditionally set DOCKER_REPO for subsequent test steps such as the Docker Compose file +if (beta) { + REDPANDA_DOCKER_REPO = 'redpanda-unstable'; +} else { + REDPANDA_DOCKER_REPO = 'redpanda'; +} const repo = 'redpanda'; async function loadOctokit() { const { Octokit } = await import('@octokit/rest'); - if (!process.env.REDPANDA_GITHUB_TOKEN) return new Octokit() + if (!process.env.REDPANDA_GITHUB_TOKEN) { + return new Octokit(); + } return new Octokit({ auth: process.env.REDPANDA_GITHUB_TOKEN, }); @@ -13,22 +39,29 @@ async function loadOctokit() { (async () => { try { const github = await loadOctokit(); - // Fetch the latest release - const release = await github.rest.repos.getLatestRelease({ owner, repo }); - const tag = release.data.tag_name; - latestRedpandaReleaseVersion = tag.replace('v', ''); + const results = await Promise.allSettled([ + GetLatestRedpandaVersion(github, owner, repo), + ]); + + const LatestRedpandaVersion = results[0].status === 'fulfilled' ? results[0].value : null; - // Get reference of the tag - const tagRef = await github.rest.git.getRef({ owner, repo, ref: `tags/${tag}` }); - const releaseSha = tagRef.data.object.sha; + if (!LatestRedpandaVersion) { + throw new Error('Failed to fetch the latest Redpanda version'); + } + // Determine the release version based on the beta flag, with a fallback to stable release if RC is null + const latestRedpandaReleaseVersion = beta + ? (LatestRedpandaVersion.latestRcRelease && LatestRedpandaVersion.latestRcRelease.version + ? LatestRedpandaVersion.latestRcRelease.version + : `${LatestRedpandaVersion.latestRedpandaRelease.version}`) + : `${LatestRedpandaVersion.latestRedpandaRelease.version}`; - // Get the tag object to extract the commit hash - const tagData = await github.rest.git.getTag({ owner, repo, tag_sha: releaseSha }); - latestRedpandaReleaseCommitHash = tagData.data.object.sha.substring(0, 7); + if (!LatestRedpandaVersion.latestRcRelease) REDPANDA_DOCKER_REPO = 'redpanda' - console.log(latestRedpandaReleaseVersion); + // Print both version and Docker repo for Doc Detective to capture + console.log(`REDPANDA_VERSION=${latestRedpandaReleaseVersion}`); + console.log(`REDPANDA_DOCKER_REPO=${REDPANDA_DOCKER_REPO}`); } catch (error) { console.error(error); - process.exit(1) + process.exit(1); } -})() \ No newline at end of file +})(); diff --git a/tools/add-caret-external-links.py b/tools/add-caret-external-links.py index 804d1bfb36..4910d5ef82 100644 --- a/tools/add-caret-external-links.py +++ b/tools/add-caret-external-links.py @@ -40,8 +40,7 @@ def replace_link(match): 'reference\\pages\\k-console-helm-spec.adoc', 'reference\\pages\\crd.adoc', 'reference\\pages\\k-redpanda-helm-spec.adoc', - 'reference\\partials\\bundle-contents-k8s.adoc', - 'reference\\partials\\bundle-contents-linux.adoc' + 'reference\\partials\\bundle-contents.adoc' ] # Function to process all .adoc files in a directory