diff --git a/.asf.yaml b/.asf.yaml
index 5a7f5ff722117..5d88fe28742fb 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -29,16 +29,16 @@ notifications:
# Read more here: https://github.com/apache/infrastructure-asfyaml
github:
collaborators:
- - brandboat
- - FrankYang0529
- - gongxuanzhang
- m1a2st
- - mingyen066
- - ShivsundarR
- smjn
- TaiJuWu
- - xijiu
+ - brandboat
- Yunyung
+ - xijiu
+ - chirag-wadhwa5
+ - mingyen066
+ - ShivsundarR
+ - Rancho-7
enabled_merge_buttons:
squash: true
squash_commit_message: PR_TITLE_AND_DESC
diff --git a/.github/actions/run-gradle/action.yml b/.github/actions/run-gradle/action.yml
index 8422defb273cf..9c8e0945184bb 100644
--- a/.github/actions/run-gradle/action.yml
+++ b/.github/actions/run-gradle/action.yml
@@ -83,6 +83,9 @@ runs:
RUN_FLAKY_TESTS: ${{ inputs.run-flaky-tests }}
TEST_XML_OUTPUT_DIR: ${{ inputs.test-xml-output }}
TEST_VERBOSE: ${{ inputs.test-verbose }}
+ # This build step is invoked by build.yml to run junit tests only,
+ # Spotbugs is being run by that workflow via the "check" task and does not need to also be run here,
+ # since that is redundant.
run: |
set +e
./.github/scripts/thread-dump.sh &
@@ -97,6 +100,8 @@ runs:
-Pkafka.cluster.test.repeat=$TEST_REPEAT \
-Pkafka.test.verbose=$TEST_VERBOSE \
-PcommitId=xxxxxxxxxxxxxxxx \
+ -x spotbugsMain \
+ -x spotbugsTest \
$TEST_TASK
exitcode="$?"
echo "exitcode=$exitcode" >> $GITHUB_OUTPUT
diff --git a/.github/actions/setup-gradle/action.yml b/.github/actions/setup-gradle/action.yml
index 1a5b0902703ab..57d363471dd29 100644
--- a/.github/actions/setup-gradle/action.yml
+++ b/.github/actions/setup-gradle/action.yml
@@ -37,12 +37,12 @@ runs:
using: "composite"
steps:
- name: Setup Java
- uses: actions/setup-java@v4
+ uses: actions/setup-java@v5
with:
distribution: temurin
java-version: ${{ inputs.java-version }}
- name: Setup Gradle
- uses: gradle/actions/setup-gradle@94baf225fe0a508e581a564467443d0e2379123b # v4.3.0
+ uses: gradle/actions/setup-gradle@4d9f0ba0025fe599b4ebab900eb7f3a1d93ef4c2 # v5.0.0
env:
GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true
with:
diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml
index d7e326314c1be..10c55f6e083b5 100644
--- a/.github/actions/setup-python/action.yml
+++ b/.github/actions/setup-python/action.yml
@@ -22,7 +22,7 @@ runs:
using: "composite"
steps:
- name: Setup Python
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v6
with:
python-version: 3.12
- name: Pip install
diff --git a/.github/configs/labeler.yml b/.github/configs/labeler.yml
index 24a7a643c9042..b988967f33788 100644
--- a/.github/configs/labeler.yml
+++ b/.github/configs/labeler.yml
@@ -92,6 +92,12 @@ transactions:
- any-glob-to-any-file:
- 'transaction-coordinator/**'
+group-coordinator:
+ - changed-files:
+ - any-glob-to-any-file:
+ - 'group-coordinator/**'
+ - 'coordinator-common/**'
+
kip-932:
- changed-files:
- any-glob-to-any-file:
diff --git a/.github/scripts/junit.py b/.github/scripts/junit.py
index 95b5d4e4610de..550ea4935116d 100644
--- a/.github/scripts/junit.py
+++ b/.github/scripts/junit.py
@@ -361,7 +361,7 @@ def split_report_path(base_path: str, report_path: str) -> Tuple[str, str]:
failure_messages.append(f"Gradle task had a failure exit code. Failing this script.")
if thread_dump_url:
- failure_messages.append(f"Thread dump available at {thread_dump_url}. Failing this script.")
+ failure_messages.append(f"Thread dump available at {thread_dump_url} and the script will now fail.")
if junit_report_url:
report_md = f"Download [JUnit HTML report]({junit_report_url})"
diff --git a/.github/scripts/pr-format.py b/.github/scripts/pr-format.py
index 7793f8c8378db..d2da5e3e5bff3 100644
--- a/.github/scripts/pr-format.py
+++ b/.github/scripts/pr-format.py
@@ -97,6 +97,8 @@ def split_paragraphs(text: str):
else:
if line[0] in ("#", "*", "-", "=") or line[0].isdigit():
markdown = True
+ if "```" in line:
+ markdown = True
paragraph.append(line)
yield paragraph, markdown
diff --git a/.github/scripts/requirements.txt b/.github/scripts/requirements.txt
index d59455f79dac6..d3fcf50bb7400 100644
--- a/.github/scripts/requirements.txt
+++ b/.github/scripts/requirements.txt
@@ -12,6 +12,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+# Note: Ensure the 'requests' version here matches the version in tests/setup.py
PyYAML~=6.0
pytz==2024.2
-requests==2.32.3
+requests==2.32.4
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index fc5ee1bd7558b..59c2fabaaed25 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -66,12 +66,12 @@ jobs:
name: Load Test Catalog
steps:
- name: Checkout main
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
with:
persist-credentials: false
- name: Checkout test-catalog
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
with:
ref: 'test-catalog'
persist-credentials: false
@@ -118,7 +118,7 @@ jobs:
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
with:
persist-credentials: false
ref: ${{ github.sha }} # this is the default, just being explicit.
@@ -127,7 +127,7 @@ jobs:
- name: Setup Gradle
uses: ./.github/actions/setup-gradle
with:
- java-version: 23
+ java-version: 17
gradle-cache-read-only: ${{ !inputs.is-trunk }}
gradle-cache-write-only: ${{ inputs.is-trunk }}
develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
@@ -181,7 +181,7 @@ jobs:
fail-fast: false
matrix:
# If we change these, make sure to adjust ci-complete.yml
- java: [ 23, 17 ]
+ java: [ 25, 17 ]
run-flaky: [ true, false ]
run-new: [ true, false ]
exclude:
@@ -192,7 +192,7 @@ jobs:
name: JUnit tests Java ${{ matrix.java }}${{ matrix.run-flaky == true && ' (flaky)' || '' }}${{ matrix.run-new == true && ' (new)' || '' }}
steps:
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
with:
persist-credentials: false
ref: ${{ needs.configure.outputs.sha }}
@@ -210,7 +210,7 @@ jobs:
# the overall workflow, so we'll continue here without a test catalog.
- name: Load Test Catalog
id: load-test-catalog
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
continue-on-error: true
with:
name: combined-test-catalog
@@ -270,7 +270,7 @@ jobs:
python .github/scripts/junit.py \
--path build/junit-xml >> $GITHUB_STEP_SUMMARY
- # This job downloads all the JUnit XML files and thread dumps from the JDK 23 test runs.
+ # This job downloads all the JUnit XML files and thread dumps from the JDK 25 test runs.
# If any test job fails, we will not run this job. Also, if any thread dump artifacts
# are present, this means there was a timeout in the tests and so we will not proceed
# with catalog creation.
@@ -282,13 +282,13 @@ jobs:
uploaded-test-catalog: ${{ steps.archive-test-catalog.outcome == 'success' }}
steps:
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
with:
persist-credentials: false
- name: Download Thread Dumps
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
with:
- pattern: junit-thread-dumps-23-*
+ pattern: junit-thread-dumps-25-*
path: thread-dumps
merge-multiple: true
- name: Check For Thread Dump
@@ -300,9 +300,9 @@ jobs:
exit 1;
fi
- name: Download JUnit XMLs
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
with:
- pattern: junit-xml-23-* # Only look at JDK 23 tests for the test catalog
+ pattern: junit-xml-25-* # Only look at JDK 25 tests for the test catalog
path: junit-xml
merge-multiple: true
- name: Collate Test Catalog
@@ -334,7 +334,7 @@ jobs:
contents: write
steps:
- name: Checkout Test Catalog
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
with:
persist-credentials: true # Needed to commit and push later
ref: test-catalog
@@ -342,7 +342,7 @@ jobs:
run: |
rm -rf test-catalog
- name: Download Test Catalog
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
with:
name: test-catalog
path: test-catalog
diff --git a/.github/workflows/ci-complete.yml b/.github/workflows/ci-complete.yml
index 8855c998df28b..1dbd0871e9d1d 100644
--- a/.github/workflows/ci-complete.yml
+++ b/.github/workflows/ci-complete.yml
@@ -38,13 +38,13 @@ run-name: Build Scans for ${{ github.event.workflow_run.display_title}}
jobs:
upload-build-scan:
# Skip this workflow if the CI run was skipped or cancelled
- if: (github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure')
+ if: (github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure') && github.event.workflow_run.head_branch != '4.0'
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
- # Make sure these match build.yml
- java: [ 23, 17 ]
+ # Make sure these match build.yml and also keep in mind that GitHub Actions build will always use this file from the trunk branch.
+ java: [ 25, 17 ]
run-flaky: [ true, false ]
run-new: [ true, false ]
exclude:
@@ -61,7 +61,7 @@ jobs:
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
with:
persist-credentials:
false
@@ -72,7 +72,7 @@ jobs:
develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
- name: Download build scan archive
id: download-build-scan
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
continue-on-error: true # Don't want this step to fail the overall workflow
with:
github-token: ${{ github.token }}
diff --git a/.github/workflows/deflake.yml b/.github/workflows/deflake.yml
index f58408a04739d..3a2fbb56345b2 100644
--- a/.github/workflows/deflake.yml
+++ b/.github/workflows/deflake.yml
@@ -42,7 +42,7 @@ jobs:
name: Deflake JUnit tests
steps:
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
with:
persist-credentials: false
diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml
index 67acdf9fb7424..8358f10433a42 100644
--- a/.github/workflows/docker_build_and_test.yml
+++ b/.github/workflows/docker_build_and_test.yml
@@ -32,9 +32,9 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- name: Set up Python 3.10
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v6
with:
python-version: "3.10"
- name: Setup Docker Compose
@@ -54,7 +54,7 @@ jobs:
run: |
python docker_build_test.py kafka/test -tag=test -type=$IMAGE_TYPE -u=$KAFKA_URL
- name: Run CVE scan
- uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0
+ uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1
with:
image-ref: 'kafka/test:test'
format: 'table'
diff --git a/.github/workflows/docker_official_image_build_and_test.yml b/.github/workflows/docker_official_image_build_and_test.yml
index 58866a19d6cab..1c67ef584720c 100644
--- a/.github/workflows/docker_official_image_build_and_test.yml
+++ b/.github/workflows/docker_official_image_build_and_test.yml
@@ -31,9 +31,9 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- name: Set up Python 3.10
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v6
with:
python-version: "3.10"
- name: Setup Docker Compose
@@ -53,7 +53,7 @@ jobs:
run: |
python docker_official_image_build_test.py kafka/test -tag=test -type=$IMAGE_TYPE -v=$KAFKA_VERSION
- name: Run CVE scan
- uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0
+ uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1
with:
image-ref: 'kafka/test:test'
format: 'table'
diff --git a/.github/workflows/docker_promote.yml b/.github/workflows/docker_promote.yml
index 3b9a6f1d4fb53..e6f8779de6976 100644
--- a/.github/workflows/docker_promote.yml
+++ b/.github/workflows/docker_promote.yml
@@ -31,11 +31,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Set up QEMU
- uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
+ uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
- name: Set up Docker Buildx
- uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
+ uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Login to Docker Hub
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+ uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/docker_rc_release.yml b/.github/workflows/docker_rc_release.yml
index 1f824b39b977a..8ec489fb12b0b 100644
--- a/.github/workflows/docker_rc_release.yml
+++ b/.github/workflows/docker_rc_release.yml
@@ -37,9 +37,9 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
- name: Set up Python 3.10
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v6
with:
python-version: "3.10"
- name: Install dependencies
@@ -47,11 +47,11 @@ jobs:
python -m pip install --upgrade pip
pip install -r docker/requirements.txt
- name: Set up QEMU
- uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
+ uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
- name: Set up Docker Buildx
- uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
+ uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Login to Docker Hub
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+ uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/docker_scan.yml b/.github/workflows/docker_scan.yml
index db729c5c3ae01..ea8cc95a30346 100644
--- a/.github/workflows/docker_scan.yml
+++ b/.github/workflows/docker_scan.yml
@@ -26,10 +26,10 @@ jobs:
strategy:
matrix:
# This is an array of supported tags. Make sure this array only contains the supported tags
- supported_image_tag: ['latest', '3.7.2', '3.8.1', '3.9.0', '4.0.0']
+ supported_image_tag: ['latest', '3.9.1', '4.0.0', '4.1.0']
steps:
- name: Run CVE scan
- uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0
+ uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1
if: always()
with:
image-ref: apache/kafka:${{ matrix.supported_image_tag }}
diff --git a/.github/workflows/generate-reports.yml b/.github/workflows/generate-reports.yml
index a3ae34fcecb80..dee7094c27c15 100644
--- a/.github/workflows/generate-reports.yml
+++ b/.github/workflows/generate-reports.yml
@@ -32,7 +32,7 @@ jobs:
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Run Report
diff --git a/.github/workflows/pr-labeled.yml b/.github/workflows/pr-labeled.yml
index 87b39a659ec0c..b5695825861ff 100644
--- a/.github/workflows/pr-labeled.yml
+++ b/.github/workflows/pr-labeled.yml
@@ -35,7 +35,7 @@ jobs:
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
with:
persist-credentials:
false
diff --git a/.github/workflows/pr-labels-cron.yml b/.github/workflows/pr-labels-cron.yml
index 5faaca72ed36b..420d80498baa6 100644
--- a/.github/workflows/pr-labels-cron.yml
+++ b/.github/workflows/pr-labels-cron.yml
@@ -35,7 +35,7 @@ jobs:
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
- name: Remove label
- uses: actions/github-script@v7
+ uses: actions/github-script@v8
continue-on-error: true
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -77,7 +77,7 @@ jobs:
issues: write
pull-requests: write
steps:
- - uses: actions/stale@v9
+ - uses: actions/stale@v10
with:
debug-only: ${{ inputs.dryRun || false }}
operations-per-run: ${{ inputs.operationsPerRun || 500 }}
diff --git a/.github/workflows/pr-linter.yml b/.github/workflows/pr-linter.yml
index f19efbfabe2c1..d38a9659a01f9 100644
--- a/.github/workflows/pr-linter.yml
+++ b/.github/workflows/pr-linter.yml
@@ -31,12 +31,12 @@ jobs:
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
with:
persist-credentials: false
- name: Load PR Number
id: load-pr-number
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v5
with:
github-token: ${{ github.token }}
run-id: ${{ github.event.workflow_run.id }}
diff --git a/.github/workflows/pr-update.yml b/.github/workflows/pr-update.yml
index e1cd7214d6c36..7b45a15d19126 100644
--- a/.github/workflows/pr-update.yml
+++ b/.github/workflows/pr-update.yml
@@ -37,8 +37,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
- uses: actions/checkout@v4
- - uses: actions/labeler@v5
+ uses: actions/checkout@v5
+ - uses: actions/labeler@v6
with:
configuration-path: .github/configs/labeler.yml
- name: check small label
diff --git a/.github/workflows/prepare_docker_official_image_source.yml b/.github/workflows/prepare_docker_official_image_source.yml
index 32f21a0afd0bf..82204b9b93597 100644
--- a/.github/workflows/prepare_docker_official_image_source.yml
+++ b/.github/workflows/prepare_docker_official_image_source.yml
@@ -31,9 +31,9 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v5
- name: Set up Python 3.10
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v6
with:
python-version: "3.10"
- name: Install dependencies
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 9382d4173e94c..74de2a967b59f 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -38,7 +38,7 @@ jobs:
stale:
runs-on: ubuntu-latest
steps:
- - uses: actions/stale@v9
+ - uses: actions/stale@v10
with:
debug-only: ${{ inputs.dryRun || false }}
operations-per-run: ${{ inputs.operationsPerRun || 500 }}
diff --git a/.github/workflows/workflow-requested.yml b/.github/workflows/workflow-requested.yml
index 6463c72ab0acc..aae00f9ece7b0 100644
--- a/.github/workflows/workflow-requested.yml
+++ b/.github/workflows/workflow-requested.yml
@@ -38,7 +38,7 @@ jobs:
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@v5
with:
persist-credentials:
false
diff --git a/LICENSE-binary b/LICENSE-binary
index 6175d3ed7d479..91c8865931ea2 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -205,53 +205,54 @@
This project bundles some components that are also licensed under the Apache
License Version 2.0:
-- caffeine-3.1.8
-- commons-beanutils-1.9.4
+- caffeine-3.2.0
+- commons-beanutils-1.11.0
- commons-collections-3.2.2
- commons-digester-2.1
-- commons-lang3-3.12.0
-- commons-logging-1.3.2
-- commons-validator-1.9.0
-- error_prone_annotations-2.21.1
-- jackson-annotations-2.16.2
-- jackson-core-2.16.2
-- jackson-databind-2.16.2
-- jackson-dataformat-csv-2.16.2
-- jackson-dataformat-yaml-2.16.2
-- jackson-datatype-jdk8-2.16.2
-- jackson-jakarta-rs-base-2.16.2
-- jackson-jakarta-rs-json-provider-2.16.2
-- jackson-module-blackbird-2.16.2
-- jackson-module-jakarta-xmlbind-annotations-2.16.2
+- commons-lang3-3.18.0
+- commons-logging-1.3.5
+- commons-validator-1.10.0
+- hash4j-0.22.0
+- jackson-annotations-2.19.0
+- jackson-core-2.19.0
+- jackson-databind-2.19.0
+- jackson-dataformat-csv-2.19.0
+- jackson-dataformat-yaml-2.19.0
+- jackson-datatype-jdk8-2.19.0
+- jackson-jakarta-rs-base-2.19.0
+- jackson-jakarta-rs-json-provider-2.19.0
+- jackson-module-blackbird-2.19.0
+- jackson-module-jakarta-xmlbind-annotations-2.19.0
- jakarta.inject-api-2.0.1
- jakarta.validation-api-3.0.2
-- javassist-3.29.2-GA
-- jetty-alpn-client-12.0.15
-- jetty-client-12.0.15
-- jetty-ee10-servlet-12.0.15
-- jetty-ee10-servlets-12.0.15
-- jetty-http-12.0.15
-- jetty-io-12.0.15
-- jetty-security-12.0.15
-- jetty-server-12.0.15
-- jetty-session-12.0.15
-- jetty-util-12.0.15
-- jose4j-0.9.4
-- log4j-api-2.24.3
-- log4j-core-2.24.3
-- log4j-slf4j-impl-2.24.3
-- log4j-1.2-api-2.24.3
+- javassist-3.30.2-GA
+- jetty-alpn-client-12.0.22
+- jetty-client-12.0.22
+- jetty-ee10-servlet-12.0.22
+- jetty-ee10-servlets-12.0.22
+- jetty-http-12.0.22
+- jetty-io-12.0.22
+- jetty-security-12.0.22
+- jetty-server-12.0.22
+- jetty-session-12.0.22
+- jetty-util-12.0.22
+- jose4j-0.9.6
+- jspecify-1.0.0
+- log4j-api-2.25.1
+- log4j-core-2.25.1
+- log4j-slf4j-impl-2.25.1
+- log4j-1.2-api-2.25.1
- lz4-java-1.8.0
- maven-artifact-3.9.6
- metrics-core-2.2.0
-- opentelemetry-proto-1.0.0-alpha
+- opentelemetry-proto-1.3.2-alpha
- plexus-utils-3.5.1
-- rocksdbjni-9.7.3
-- scala-library-2.13.15
+- rocksdbjni-10.1.3
+- scala-library-2.13.17
- scala-logging_2.13-3.9.5
-- scala-reflect-2.13.15
-- snappy-java-1.1.10.5
-- snakeyaml-2.2
+- scala-reflect-2.13.17
+- snappy-java-1.1.10.7
+- snakeyaml-2.4
- swagger-annotations-2.2.25
===============================================================================
@@ -277,12 +278,12 @@ see: licenses/eclipse-public-license-2.0
- hk2-utils-3.0.6
- osgi-resource-locator-1.0.3
- aopalliance-repackaged-3.0.6
-- jersey-client-3.1.9
-- jersey-common-3.1.9
-- jersey-container-servlet-3.1.9
-- jersey-container-servlet-core-3.1.9
-- jersey-hk2-3.1.9
-- jersey-server-3.1.9
+- jersey-client-3.1.10
+- jersey-common-3.1.10
+- jersey-container-servlet-3.1.10
+- jersey-container-servlet-core-3.1.10
+- jersey-hk2-3.1.10
+- jersey-server-3.1.10
---------------------------------------
CDDL 1.1 + GPLv2 with classpath exception
@@ -298,10 +299,10 @@ see: licenses/CDDL+GPL-1.1
MIT License
- argparse4j-0.7.0, see: licenses/argparse-MIT
-- classgraph-4.8.173, see: licenses/classgraph-MIT
+- classgraph-4.8.179, see: licenses/classgraph-MIT
- jopt-simple-5.0.4, see: licenses/jopt-simple-MIT
- slf4j-api-1.7.36, see: licenses/slf4j-MIT
-- pcollections-4.0.1, see: licenses/pcollections-MIT
+- pcollections-4.0.2, see: licenses/pcollections-MIT
---------------------------------------
BSD 2-Clause
@@ -312,11 +313,11 @@ BSD 2-Clause
---------------------------------------
BSD 3-Clause
-- jline-3.25.1, see: licenses/jline-BSD-3-clause
+- jline-3.30.4, see: licenses/jline-BSD-3-clause
- protobuf-java-3.25.5, see: licenses/protobuf-java-BSD-3-clause
- jakarta.activation-2.0.1, see: licenses/jakarta-BSD-3-clause
---------------------------------------
Go License
-- re2j-1.7, see: licenses/re2j-GO
+- re2j-1.8, see: licenses/re2j-GO
diff --git a/README.md b/README.md
index 4c2fc4e1f160b..28da38abe1dab 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@
You need to have [Java](http://www.oracle.com/technetwork/java/javase/downloads/index.html) installed.
-We build and test Apache Kafka with 17 and 23. The `release` parameter in javac is set to `11` for the clients
+We build and test Apache Kafka with 17 and 25. The `release` parameter in javac is set to `11` for the clients
and streams modules, and `17` for the rest, ensuring compatibility with their respective
minimum Java versions. Similarly, the `release` parameter in scalac is set to `11` for the streams modules and `17`
for the rest.
@@ -52,6 +52,7 @@ Follow instructions in https://kafka.apache.org/quickstart
### Running a particular unit/integration test ###
./gradlew clients:test --tests RequestResponseTest
+ ./gradlew streams:integration-tests:test --tests RestoreIntegrationTest
### Repeatedly running a particular unit/integration test with specific times by setting N ###
N=500; I=0; while [ $I -lt $N ] && ./gradlew clients:test --tests RequestResponseTest --rerun --fail-fast; do (( I=$I+1 )); echo "Completed run: $I"; sleep 1; done
@@ -59,6 +60,7 @@ Follow instructions in https://kafka.apache.org/quickstart
### Running a particular test method within a unit/integration test ###
./gradlew core:test --tests kafka.api.ProducerFailureHandlingTest.testCannotSendToInternalTopic
./gradlew clients:test --tests org.apache.kafka.clients.MetadataTest.testTimeToNextUpdate
+ ./gradlew streams:integration-tests:test --tests org.apache.kafka.streams.integration.RestoreIntegrationTest.shouldRestoreNullRecord
### Running a particular unit/integration test with log4j output ###
By default, there will be only small number of logs output while testing. You can adjust it by changing the `log4j2.yaml` file in the module's `src/test/resources` directory.
@@ -230,7 +232,7 @@ Alternatively, use the `allDeps` or `allDepInsight` tasks for recursively iterat
These take the same arguments as the builtin variants.
### Determining if any dependencies could be updated ###
- ./gradlew dependencyUpdates
+ ./gradlew dependencyUpdates --no-parallel
### Common build options ###
diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh
index 3463389d3c005..012f9c27f0aca 100755
--- a/bin/kafka-run-class.sh
+++ b/bin/kafka-run-class.sh
@@ -49,7 +49,7 @@ should_include_file() {
base_dir=$(dirname $0)/..
if [ -z "$SCALA_VERSION" ]; then
- SCALA_VERSION=2.13.15
+ SCALA_VERSION=2.13.17
if [[ -f "$base_dir/gradle.properties" ]]; then
SCALA_VERSION=`grep "^scalaVersion=" "$base_dir/gradle.properties" | cut -d= -f 2`
fi
@@ -282,7 +282,7 @@ fi
# JVM performance options
# MaxInlineLevel=15 is the default since JDK 14 and can be removed once older JDKs are no longer supported
if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then
- KAFKA_JVM_PERFORMANCE_OPTS="-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -XX:MaxInlineLevel=15 -Djava.awt.headless=true"
+ KAFKA_JVM_PERFORMANCE_OPTS="-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -XX:MaxInlineLevel=15"
fi
while [ $# -gt 0 ]; do
diff --git a/bin/windows/kafka-run-class.bat b/bin/windows/kafka-run-class.bat
index ca151e5df96ed..394269a4294e2 100755
--- a/bin/windows/kafka-run-class.bat
+++ b/bin/windows/kafka-run-class.bat
@@ -27,7 +27,7 @@ set BASE_DIR=%CD%
popd
IF ["%SCALA_VERSION%"] EQU [""] (
- set SCALA_VERSION=2.13.15
+ set SCALA_VERSION=2.13.17
)
IF ["%SCALA_BINARY_VERSION%"] EQU [""] (
@@ -177,7 +177,7 @@ IF ["%KAFKA_HEAP_OPTS%"] EQU [""] (
rem JVM performance options
IF ["%KAFKA_JVM_PERFORMANCE_OPTS%"] EQU [""] (
- set KAFKA_JVM_PERFORMANCE_OPTS=-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -Djava.awt.headless=true
+ set KAFKA_JVM_PERFORMANCE_OPTS=-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent
)
IF not defined CLASSPATH (
diff --git a/build.gradle b/build.gradle
index 2e35057165c53..dc3bf215ec88f 100644
--- a/build.gradle
+++ b/build.gradle
@@ -14,8 +14,6 @@
// limitations under the License.
import org.ajoberstar.grgit.Grgit
-import org.gradle.api.JavaVersion
-
import java.nio.charset.StandardCharsets
buildscript {
@@ -31,25 +29,24 @@ buildscript {
}
plugins {
- id 'com.github.ben-manes.versions' version '0.48.0'
+ id 'com.github.ben-manes.versions' version '0.53.0'
id 'idea'
id 'jacoco'
id 'java-library'
- id 'org.owasp.dependencycheck' version '8.2.1'
+ id 'org.owasp.dependencycheck' version '12.1.3'
id 'org.nosphere.apache.rat' version "0.8.1"
id "io.swagger.core.v3.swagger-gradle-plugin" version "${swaggerVersion}"
- id "com.github.spotbugs" version '6.0.25' apply false
+ id "com.github.spotbugs" version '6.2.5' apply false
id 'org.scoverage' version '8.0.3' apply false
- id 'com.gradleup.shadow' version '8.3.6' apply false
- id 'com.diffplug.spotless' version "6.25.0"
+ id 'com.gradleup.shadow' version '8.3.9' apply false
+ id 'com.diffplug.spotless' version "7.2.1"
}
ext {
- gradleVersion = versions.gradle
minClientJavaVersion = 11
minNonClientJavaVersion = 17
- modulesNeedingJava11 = [":clients", ":generator", ":streams", ":streams:test-utils", ":streams-scala", ":test-common:test-common-util"]
+ modulesNeedingJava11 = [":clients", ":generator", ":streams", ":streams:test-utils", ":streams:examples", ":streams-scala", ":test-common:test-common-util"]
buildVersionFileName = "kafka-version.properties"
@@ -74,6 +71,13 @@ ext {
"--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED"
)
+ if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_25)) {
+ // Spotbugs is not compatible with Java 25+ so Gradle related tasks are disabled
+ // until version can be upgraded: https://github.com/spotbugs/spotbugs/issues/3564
+ project.gradle.startParameter.excludedTaskNames.add("spotbugsMain")
+ project.gradle.startParameter.excludedTaskNames.add("spotbugsTest")
+ }
+
maxTestForks = project.hasProperty('maxParallelForks') ? maxParallelForks.toInteger() : Runtime.runtime.availableProcessors()
maxScalacThreads = project.hasProperty('maxScalacThreads') ? maxScalacThreads.toInteger() :
Math.min(Runtime.runtime.availableProcessors(), 8)
@@ -159,7 +163,7 @@ ext {
libs.log4j2Api,
libs.log4j2Core
]
-
+
}
allprojects {
@@ -195,7 +199,8 @@ allprojects {
// ensure we have a single version in the classpath despite transitive dependencies
libs.scalaLibrary,
libs.scalaReflect,
- libs.jacksonAnnotations
+ libs.jacksonAnnotations,
+ libs.commonsLang
)
}
}
@@ -298,7 +303,7 @@ if (repo != null) {
} else {
rat.enabled = false
}
-println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $gradleVersion, Java ${JavaVersion.current()} and Scala ${versions.scala}")
+println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $versions.gradle, Java ${JavaVersion.current()} and Scala ${versions.scala}")
println("Build properties: ignoreFailures=$userIgnoreFailures, maxParallelForks=$maxTestForks, maxScalacThreads=$maxScalacThreads, maxTestRetries=$userMaxTestRetries")
subprojects {
@@ -329,6 +334,16 @@ subprojects {
tasks.register('uploadArchives').configure { dependsOn(publish) }
}
+ tasks.withType(AbstractArchiveTask).configureEach {
+ reproducibleFileOrder = false
+ preserveFileTimestamps = true
+ useFileSystemPermissions()
+ }
+
+ tasks.withType(AbstractTestTask).configureEach {
+ failOnNoDiscoveredTests = false
+ }
+
// apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder.
if (!project.name.equals('connect')) {
apply plugin: 'eclipse'
@@ -369,7 +384,6 @@ subprojects {
// Fix for avoiding inclusion of runtime dependencies marked as 'shadow' in MANIFEST Class-Path.
// https://github.com/GradleUp/shadow/issues/324
- afterEvaluate {
pom.withXml { xml ->
def dependenciesNode = xml.asNode().get('dependencies') ?: xml.asNode().appendNode('dependencies')
project.configurations.shadowed.allDependencies.each {
@@ -380,7 +394,6 @@ subprojects {
dependencyNode.appendNode('scope', 'runtime')
}
}
- }
}
afterEvaluate {
@@ -485,11 +498,36 @@ subprojects {
}
}
+ // Workaround for Mockito Java Agent restrictions in Java 21+
+ // Starting with Java 21, the JDK restricts libraries from attaching a Java agent
+ // to their own JVM. As a result, Mockito’s inline mock maker (mockito-core)
+ // fails without explicit instrumentation, and the JVM consistently emits warnings.
+ // See also: https://javadoc.io/doc/org.mockito/mockito-core/latest/org.mockito/org/mockito/Mockito.html#mockito-instrumentation
+ afterEvaluate { subproject ->
+ def hasMockitoCore = subproject.configurations.findAll {
+ it.canBeResolved
+ }.any { config ->
+ config.incoming.dependencies.any { dependency ->
+ "$dependency" == libs.mockitoCore
+ }
+ }
+
+ if (hasMockitoCore) {
+ subproject.configurations {
+ mockitoAgent {
+ transitive = false
+ }
+ }
+ subproject.dependencies {
+ mockitoAgent libs.mockitoCore
+ }
+ }
+ }
+
// The suites are for running sets of tests in IDEs.
// Gradle will run each test class, so we exclude the suites to avoid redundantly running the tests twice.
def testsToExclude = ['**/*Suite.class']
-
// This task will copy JUnit XML files out of the sub-project's build directory and into
// a top-level build/junit-xml directory. This is necessary to avoid reporting on tests which
// were not run, but instead were restored via FROM-CACHE. See KAFKA-17479 for more details.
@@ -519,6 +557,14 @@ subprojects {
}
test {
+
+ doFirst {
+ def mockitoAgentConfig = configurations.findByName('mockitoAgent')
+ if (mockitoAgentConfig) {
+ jvmArgs("-javaagent:${mockitoAgentConfig.asPath}")
+ }
+ }
+
maxParallelForks = maxTestForks
ignoreFailures = userIgnoreFailures
@@ -552,7 +598,7 @@ subprojects {
maxFailures = userMaxTestRetryFailures
}
}
-
+
finalizedBy("copyTestXml")
}
@@ -894,6 +940,9 @@ project(':server') {
}
dependencies {
+ compileOnly libs.bndlib
+ compileOnly libs.spotbugs
+
implementation project(':clients')
implementation project(':metadata')
implementation project(':server-common')
@@ -906,6 +955,7 @@ project(':server') {
implementation libs.jacksonDatabind
implementation libs.metrics
implementation libs.slf4jApi
+ implementation log4j2Libs
testImplementation project(':clients').sourceSets.test.output
@@ -915,6 +965,7 @@ project(':server') {
testImplementation project(':test-common:test-common-internal-api')
testImplementation project(':test-common:test-common-runtime')
testImplementation project(':storage:storage-api').sourceSets.test.output
+ testImplementation project(':server-common').sourceSets.test.output
testRuntimeOnly runtimeTestLibs
}
@@ -1004,6 +1055,8 @@ project(':core') {
implementation project(':transaction-coordinator')
implementation project(':metadata')
implementation project(':storage:storage-api')
+ // tools-api is automatically included in releaseTarGz via core's runtimeClasspath.
+ // If removed from here, remember to explicitly add it back in the releaseTarGz task.
implementation project(':tools:tools-api')
implementation project(':raft')
implementation project(':storage')
@@ -1024,6 +1077,7 @@ project(':core') {
implementation libs.scalaReflect
implementation libs.scalaLogging
implementation libs.slf4jApi
+ implementation libs.re2j
testImplementation project(':clients').sourceSets.test.output
testImplementation project(':group-coordinator').sourceSets.test.output
@@ -1056,6 +1110,7 @@ project(':core') {
testImplementation libs.junitJupiter
testImplementation libs.caffeine
testImplementation testLog4j2Libs
+ testImplementation libs.mockOAuth2Server
testRuntimeOnly runtimeTestLibs
}
@@ -1143,6 +1198,13 @@ project(':core') {
standardOutput = new File(generatedDocsDir, "topic_config.html").newOutputStream()
}
+ task genGroupConfigDocs(type: JavaExec) {
+ classpath = sourceSets.main.runtimeClasspath
+ mainClass = 'org.apache.kafka.coordinator.group.GroupConfig'
+ if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
+ standardOutput = new File(generatedDocsDir, "group_config.html").newOutputStream()
+ }
+
task genConsumerMetricsDocs(type: JavaExec) {
classpath = sourceSets.test.runtimeClasspath
mainClass = 'org.apache.kafka.clients.consumer.internals.ConsumerMetrics'
@@ -1159,7 +1221,7 @@ project(':core') {
task siteDocsTar(dependsOn: ['genProtocolErrorDocs', 'genProtocolTypesDocs', 'genProtocolApiKeyDocs', 'genProtocolMessageDocs',
'genAdminClientConfigDocs', 'genProducerConfigDocs', 'genConsumerConfigDocs',
- 'genKafkaConfigDocs', 'genTopicConfigDocs',
+ 'genKafkaConfigDocs', 'genTopicConfigDocs', 'genGroupConfigDocs',
':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs',
':connect:runtime:genConnectPredicateDocs',
':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs',
@@ -1217,8 +1279,6 @@ project(':core') {
from(project(':streams:test-utils').configurations.runtimeClasspath) { into("libs/") }
from(project(':streams:examples').jar) { into("libs/") }
from(project(':streams:examples').configurations.runtimeClasspath) { into("libs/") }
- from(project(':tools:tools-api').jar) { into("libs/") }
- from(project(':tools:tools-api').configurations.runtimeClasspath) { into("libs/") }
duplicatesStrategy 'exclude'
}
@@ -1420,6 +1480,7 @@ project(':group-coordinator') {
implementation libs.hdrHistogram
implementation libs.re2j
implementation libs.slf4jApi
+ implementation libs.hash4j
testImplementation project(':clients').sourceSets.test.output
testImplementation project(':server-common').sourceSets.test.output
@@ -1762,11 +1823,11 @@ project(':generator') {
implementation libs.jacksonJDK8Datatypes
implementation libs.jacksonJakartarsJsonProvider
- implementation 'org.eclipse.jgit:org.eclipse.jgit:6.4.0.202211300538-r'
+ implementation 'org.eclipse.jgit:org.eclipse.jgit:7.2.0.202503040940-r'
// SSH support for JGit based on Apache MINA sshd
- implementation 'org.eclipse.jgit:org.eclipse.jgit.ssh.apache:6.4.0.202211300538-r'
+ implementation 'org.eclipse.jgit:org.eclipse.jgit.ssh.apache:7.2.0.202503040940-r'
// GPG support for JGit based on BouncyCastle (commit signing)
- implementation 'org.eclipse.jgit:org.eclipse.jgit.gpg.bc:6.4.0.202211300538-r'
+ implementation 'org.eclipse.jgit:org.eclipse.jgit.gpg.bc:7.2.0.202503040940-r'
testImplementation libs.junitJupiter
@@ -1973,6 +2034,7 @@ project(':clients:clients-integration-tests') {
}
dependencies {
+ testImplementation libs.metrics
testImplementation libs.slf4jApi
testImplementation project(':test-common:test-common-internal-api')
testImplementation project(':test-common:test-common-runtime')
@@ -1982,9 +2044,12 @@ project(':clients:clients-integration-tests') {
testImplementation project(':core').sourceSets.test.output
testImplementation project(':clients').sourceSets.test.output
implementation project(':server-common')
+ testImplementation project(':server-common').sourceSets.test.output
testImplementation project(':metadata')
implementation project(':group-coordinator')
+ implementation project(':group-coordinator:group-coordinator-api')
implementation project(':transaction-coordinator')
+ testImplementation project(':test-common:test-common-util')
testImplementation libs.junitJupiter
testImplementation libs.junitPlatformSuiteEngine
@@ -2230,11 +2295,13 @@ project(':storage') {
}
dependencies {
+ implementation project(':metadata')
implementation project(':storage:storage-api')
implementation project(':server-common')
implementation project(':clients')
implementation(libs.caffeine) {
exclude group: 'org.checkerframework', module: 'checker-qual'
+ exclude group: 'com.google.errorprone', module: 'error_prone_annotations'
}
implementation libs.slf4jApi
implementation libs.jacksonDatabind
@@ -2897,10 +2964,10 @@ project(':streams:examples') {
}
dependencies {
- // this dependency should be removed after we unify data API
- implementation(project(':connect:json'))
implementation project(':streams')
implementation libs.slf4jApi
+ implementation libs.jacksonDatabind
+ implementation libs.jacksonAnnotations
testImplementation project(':streams:test-utils')
testImplementation project(':clients').sourceSets.test.output // for org.apache.kafka.test.IntegrationTest
@@ -2936,6 +3003,7 @@ project(':streams:upgrade-system-tests-0110') {
dependencies {
testImplementation libs.kafkaStreams_0110
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -2951,6 +3019,7 @@ project(':streams:upgrade-system-tests-10') {
dependencies {
testImplementation libs.kafkaStreams_10
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -2966,6 +3035,7 @@ project(':streams:upgrade-system-tests-11') {
dependencies {
testImplementation libs.kafkaStreams_11
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -2981,6 +3051,7 @@ project(':streams:upgrade-system-tests-20') {
dependencies {
testImplementation libs.kafkaStreams_20
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -2996,6 +3067,7 @@ project(':streams:upgrade-system-tests-21') {
dependencies {
testImplementation libs.kafkaStreams_21
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3011,6 +3083,7 @@ project(':streams:upgrade-system-tests-22') {
dependencies {
testImplementation libs.kafkaStreams_22
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3026,6 +3099,7 @@ project(':streams:upgrade-system-tests-23') {
dependencies {
testImplementation libs.kafkaStreams_23
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3041,6 +3115,7 @@ project(':streams:upgrade-system-tests-24') {
dependencies {
testImplementation libs.kafkaStreams_24
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3056,6 +3131,7 @@ project(':streams:upgrade-system-tests-25') {
dependencies {
testImplementation libs.kafkaStreams_25
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3071,6 +3147,7 @@ project(':streams:upgrade-system-tests-26') {
dependencies {
testImplementation libs.kafkaStreams_26
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3086,6 +3163,7 @@ project(':streams:upgrade-system-tests-27') {
dependencies {
testImplementation libs.kafkaStreams_27
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3101,6 +3179,7 @@ project(':streams:upgrade-system-tests-28') {
dependencies {
testImplementation libs.kafkaStreams_28
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3116,6 +3195,7 @@ project(':streams:upgrade-system-tests-30') {
dependencies {
testImplementation libs.kafkaStreams_30
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3131,6 +3211,7 @@ project(':streams:upgrade-system-tests-31') {
dependencies {
testImplementation libs.kafkaStreams_31
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3146,6 +3227,7 @@ project(':streams:upgrade-system-tests-32') {
dependencies {
testImplementation libs.kafkaStreams_32
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3161,6 +3243,7 @@ project(':streams:upgrade-system-tests-33') {
dependencies {
testImplementation libs.kafkaStreams_33
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3176,6 +3259,7 @@ project(':streams:upgrade-system-tests-34') {
dependencies {
testImplementation libs.kafkaStreams_34
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3191,6 +3275,7 @@ project(':streams:upgrade-system-tests-35') {
dependencies {
testImplementation libs.kafkaStreams_35
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3206,6 +3291,7 @@ project(':streams:upgrade-system-tests-36') {
dependencies {
testImplementation libs.kafkaStreams_36
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3221,6 +3307,7 @@ project(':streams:upgrade-system-tests-37') {
dependencies {
testImplementation libs.kafkaStreams_37
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3236,6 +3323,7 @@ project(':streams:upgrade-system-tests-38') {
dependencies {
testImplementation libs.kafkaStreams_38
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3251,6 +3339,7 @@ project(':streams:upgrade-system-tests-39') {
dependencies {
testImplementation libs.kafkaStreams_39
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3266,6 +3355,23 @@ project(':streams:upgrade-system-tests-40') {
dependencies {
testImplementation libs.kafkaStreams_40
testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
+ }
+
+ systemTestLibs {
+ dependsOn testJar
+ }
+}
+
+project(':streams:upgrade-system-tests-41') {
+ base {
+ archivesName = "kafka-streams-upgrade-system-tests-41"
+ }
+
+ dependencies {
+ testImplementation libs.kafkaStreams_41
+ testRuntimeOnly libs.junitJupiter
+ testRuntimeOnly runtimeTestLibs
}
systemTestLibs {
@@ -3291,6 +3397,7 @@ project(':jmh-benchmarks') {
implementation project(':raft')
implementation project(':clients')
implementation project(':coordinator-common')
+ implementation project(':coordinator-common').sourceSets.test.output
implementation project(':group-coordinator')
implementation project(':group-coordinator:group-coordinator-api')
implementation project(':metadata')
@@ -3339,7 +3446,7 @@ project(':jmh-benchmarks') {
if (System.getProperty("jmhArgs")) {
args System.getProperty("jmhArgs").split(' ')
}
- args = [shadowJar.archivePath, *args]
+ args = [shadowJar.archiveFile.get().asFile, *args]
}
}
diff --git a/checkstyle/import-control-clients-integration-tests.xml b/checkstyle/import-control-clients-integration-tests.xml
index 44cf0dba1fbec..3c6f50890d2b8 100644
--- a/checkstyle/import-control-clients-integration-tests.xml
+++ b/checkstyle/import-control-clients-integration-tests.xml
@@ -21,12 +21,14 @@
-
+
+
+
diff --git a/checkstyle/import-control-coordinator-common.xml b/checkstyle/import-control-coordinator-common.xml
index bafffe8069746..7841697cf892a 100644
--- a/checkstyle/import-control-coordinator-common.xml
+++ b/checkstyle/import-control-coordinator-common.xml
@@ -58,9 +58,11 @@
+
+
diff --git a/checkstyle/import-control-group-coordinator.xml b/checkstyle/import-control-group-coordinator.xml
index 8b6a8d99f5eaa..1f0e91de144bc 100644
--- a/checkstyle/import-control-group-coordinator.xml
+++ b/checkstyle/import-control-group-coordinator.xml
@@ -51,6 +51,7 @@
+
@@ -76,6 +77,7 @@
+
diff --git a/checkstyle/import-control-jmh-benchmarks.xml b/checkstyle/import-control-jmh-benchmarks.xml
index 4469ccf3bbeb9..4c11bc3acb42e 100644
--- a/checkstyle/import-control-jmh-benchmarks.xml
+++ b/checkstyle/import-control-jmh-benchmarks.xml
@@ -42,7 +42,6 @@
-
@@ -53,7 +52,7 @@
-
+
diff --git a/checkstyle/import-control-metadata.xml b/checkstyle/import-control-metadata.xml
index c2660674e3259..773635cec8e9c 100644
--- a/checkstyle/import-control-metadata.xml
+++ b/checkstyle/import-control-metadata.xml
@@ -83,7 +83,6 @@
-
@@ -108,6 +107,7 @@
+
@@ -160,9 +160,9 @@
-
+
@@ -198,18 +198,4 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/checkstyle/import-control-server-common.xml b/checkstyle/import-control-server-common.xml
index 8d85dffa341f0..95a014b87e4ec 100644
--- a/checkstyle/import-control-server-common.xml
+++ b/checkstyle/import-control-server-common.xml
@@ -33,6 +33,7 @@
+
@@ -49,6 +50,9 @@
+
+
+
@@ -60,6 +64,7 @@
+
diff --git a/checkstyle/import-control-server.xml b/checkstyle/import-control-server.xml
index 30a7f5fbe761f..b3d1b928cc6db 100644
--- a/checkstyle/import-control-server.xml
+++ b/checkstyle/import-control-server.xml
@@ -86,10 +86,15 @@
+
+
+
+
+
@@ -100,6 +105,10 @@
+
+
+
+
diff --git a/checkstyle/import-control-storage.xml b/checkstyle/import-control-storage.xml
index 639cb6dc1d04c..2a0f74126859a 100644
--- a/checkstyle/import-control-storage.xml
+++ b/checkstyle/import-control-storage.xml
@@ -94,6 +94,8 @@
+
+
@@ -156,4 +158,13 @@
+
+
+
+
+
+
+
+
+
diff --git a/checkstyle/import-control-transaction-coordinator.xml b/checkstyle/import-control-transaction-coordinator.xml
index bf2157750c3a6..810c127c95c32 100644
--- a/checkstyle/import-control-transaction-coordinator.xml
+++ b/checkstyle/import-control-transaction-coordinator.xml
@@ -38,7 +38,9 @@
+
+
diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index ead4111389bb3..c7f9eaad7ea08 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -145,6 +145,7 @@
+
@@ -199,6 +200,8 @@
+
+
@@ -232,6 +235,7 @@
+
@@ -428,6 +432,7 @@
+
@@ -492,6 +497,7 @@
+
diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 4e0f338af5d46..d363b9ed9c0d0 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -41,7 +41,7 @@
files="(KafkaClusterTestKit).java"/>
+ files="(SharePartitionManagerTest|SharePartitionTest|ShareConsumerTest).java"/>
@@ -49,6 +49,7 @@
+
+ files="(KafkaConsumer|ConsumerCoordinator|AbstractFetch|KafkaProducer|AbstractRequest|AbstractResponse|TransactionManager|Admin|KafkaAdminClient|MockAdminClient|KafkaNetworkChannelTest|ClientTelemetryReporterTest).java"/>
@@ -98,7 +99,7 @@
files="(AbstractFetch|ClientTelemetryReporter|ConsumerCoordinator|CommitRequestManager|FetchCollector|OffsetFetcherUtils|KafkaProducer|Sender|ConfigDef|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslFactory|SslTransportLayer|SaslClientAuthenticator|SaslClientCallbackHandler|SaslServerAuthenticator|AbstractCoordinator|TransactionManager|AbstractStickyAssignor|DefaultSslEngineFactory|Authorizer|RecordAccumulator|MemoryRecords|FetchSessionHandler|MockAdminClient).java"/>
+ files="(AbstractRequest|AbstractResponse|KerberosLogin|WorkerSinkTaskTest|TransactionManagerTest|SenderTest|KafkaAdminClient|ConsumerCoordinatorTest|KafkaAdminClientTest|KafkaProducerTest).java"/>
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/ClientRebootstrapTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/ClientRebootstrapTest.java
index 2f11e13377d07..b388744b711ab 100644
--- a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/ClientRebootstrapTest.java
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/ClientRebootstrapTest.java
@@ -24,11 +24,11 @@
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.config.TopicConfig;
import org.apache.kafka.common.test.ClusterInstance;
-import org.apache.kafka.common.test.TestUtils;
import org.apache.kafka.common.test.api.ClusterConfigProperty;
import org.apache.kafka.common.test.api.ClusterTest;
import org.apache.kafka.common.test.api.Type;
import org.apache.kafka.coordinator.group.GroupCoordinatorConfig;
+import org.apache.kafka.test.TestUtils;
import java.time.Duration;
import java.util.List;
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/ClientsTestUtils.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/ClientsTestUtils.java
new file mode 100644
index 0000000000000..a007035e860a8
--- /dev/null
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/ClientsTestUtils.java
@@ -0,0 +1,539 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients;
+
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.consumer.OffsetCommitCallback;
+import org.apache.kafka.clients.consumer.RetriableCommitFailedException;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.ClusterResource;
+import org.apache.kafka.common.ClusterResourceListener;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.internals.Topic;
+import org.apache.kafka.common.record.TimestampType;
+import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.kafka.common.serialization.Serializer;
+import org.apache.kafka.common.test.ClusterInstance;
+import org.apache.kafka.test.TestUtils;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+
+import static org.apache.kafka.clients.ClientsTestUtils.TestClusterResourceListenerDeserializer.UPDATE_CONSUMER_COUNT;
+import static org.apache.kafka.clients.ClientsTestUtils.TestClusterResourceListenerSerializer.UPDATE_PRODUCER_COUNT;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG;
+import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG;
+import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class ClientsTestUtils {
+
+ private static final String KEY_PREFIX = "key ";
+ private static final String VALUE_PREFIX = "value ";
+
+ private ClientsTestUtils() {}
+
+ public static List> consumeRecords(
+ Consumer consumer,
+ int numRecords
+ ) throws InterruptedException {
+ return consumeRecords(consumer, numRecords, Integer.MAX_VALUE);
+ }
+
+ public static List> consumeRecords(
+ Consumer consumer,
+ int numRecords,
+ int maxPollRecords
+ ) throws InterruptedException {
+ List> consumedRecords = new ArrayList<>();
+ TestUtils.waitForCondition(() -> {
+ var records = consumer.poll(Duration.ofMillis(100));
+ records.forEach(consumedRecords::add);
+ assertTrue(records.count() <= maxPollRecords);
+ return consumedRecords.size() >= numRecords;
+ }, 60000, "Timed out before consuming expected " + numRecords + " records.");
+
+ return consumedRecords;
+ }
+
+ public static void consumeAndVerifyRecords(
+ Consumer consumer,
+ TopicPartition tp,
+ int numRecords,
+ int startingOffset,
+ int startingKeyAndValueIndex,
+ long startingTimestamp,
+ long timestampIncrement
+ ) throws InterruptedException {
+ consumeAndVerifyRecords(
+ consumer,
+ tp,
+ numRecords,
+ Integer.MAX_VALUE,
+ startingOffset,
+ startingKeyAndValueIndex,
+ startingTimestamp,
+ timestampIncrement
+ );
+ }
+
+ public static void pollUntilTrue(
+ Consumer consumer,
+ Supplier testCondition,
+ String msg
+ ) throws InterruptedException {
+ pollUntilTrue(consumer, Duration.ofMillis(100), testCondition, 15_000L, msg);
+ }
+
+ public static void pollUntilTrue(
+ Consumer consumer,
+ Supplier testCondition,
+ long waitTimeMs,
+ String msg
+ ) throws InterruptedException {
+ pollUntilTrue(consumer, Duration.ofMillis(100), testCondition, waitTimeMs, msg);
+ }
+
+ public static void pollUntilTrue(
+ Consumer consumer,
+ Duration timeout,
+ Supplier testCondition,
+ long waitTimeMs,
+ String msg
+ ) throws InterruptedException {
+ TestUtils.waitForCondition(() -> {
+ consumer.poll(timeout);
+ return testCondition.get();
+ }, waitTimeMs, msg);
+ }
+
+ public static void consumeAndVerifyRecordsWithTimeTypeLogAppend(
+ Consumer consumer,
+ TopicPartition tp,
+ int numRecords,
+ long startingTimestamp
+ ) throws InterruptedException {
+ var records = consumeRecords(consumer, numRecords, Integer.MAX_VALUE);
+ var now = System.currentTimeMillis();
+ for (var i = 0; i < numRecords; i++) {
+ var record = records.get(i);
+ assertEquals(tp.topic(), record.topic());
+ assertEquals(tp.partition(), record.partition());
+
+ assertTrue(record.timestamp() >= startingTimestamp && record.timestamp() <= now,
+ "Got unexpected timestamp " + record.timestamp() + ". Timestamp should be between [" + startingTimestamp + ", " + now + "]");
+
+ assertEquals(i, record.offset());
+ assertEquals(KEY_PREFIX + i, new String(record.key()));
+ assertEquals(VALUE_PREFIX + i, new String(record.value()));
+ // this is true only because K and V are byte arrays
+ assertEquals((KEY_PREFIX + i).length(), record.serializedKeySize());
+ assertEquals((VALUE_PREFIX + i).length(), record.serializedValueSize());
+ }
+ }
+
+ public static void consumeAndVerifyRecords(
+ Consumer consumer,
+ TopicPartition tp,
+ int numRecords,
+ int maxPollRecords,
+ int startingOffset,
+ int startingKeyAndValueIndex,
+ long startingTimestamp,
+ long timestampIncrement
+ ) throws InterruptedException {
+ var records = consumeRecords(consumer, numRecords, maxPollRecords);
+ for (var i = 0; i < numRecords; i++) {
+ var record = records.get(i);
+ var offset = startingOffset + i;
+
+ assertEquals(tp.topic(), record.topic());
+ assertEquals(tp.partition(), record.partition());
+
+ assertEquals(TimestampType.CREATE_TIME, record.timestampType());
+ var timestamp = startingTimestamp + i * (timestampIncrement > 0 ? timestampIncrement : 1);
+ assertEquals(timestamp, record.timestamp());
+
+ assertEquals(offset, record.offset());
+ var keyAndValueIndex = startingKeyAndValueIndex + i;
+ assertEquals(KEY_PREFIX + keyAndValueIndex, new String(record.key()));
+ assertEquals(VALUE_PREFIX + keyAndValueIndex, new String(record.value()));
+ // this is true only because K and V are byte arrays
+ assertEquals((KEY_PREFIX + keyAndValueIndex).length(), record.serializedKeySize());
+ assertEquals((VALUE_PREFIX + keyAndValueIndex).length(), record.serializedValueSize());
+ }
+ }
+
+ public static void consumeAndVerifyRecords(
+ Consumer consumer,
+ TopicPartition tp,
+ int numRecords,
+ int startingOffset,
+ int startingKeyAndValueIndex,
+ long startingTimestamp
+ ) throws InterruptedException {
+ consumeAndVerifyRecords(consumer, tp, numRecords, startingOffset, startingKeyAndValueIndex, startingTimestamp, -1);
+ }
+
+ public static void consumeAndVerifyRecords(
+ Consumer consumer,
+ TopicPartition tp,
+ int numRecords,
+ int startingOffset
+ ) throws InterruptedException {
+ consumeAndVerifyRecords(consumer, tp, numRecords, startingOffset, 0, 0, -1);
+ }
+
+ public static void sendRecords(
+ ClusterInstance cluster,
+ TopicPartition tp,
+ int numRecords,
+ long startingTimestamp,
+ long timestampIncrement
+ ) {
+ try (Producer producer = cluster.producer()) {
+ for (var i = 0; i < numRecords; i++) {
+ sendRecord(producer, tp, startingTimestamp, i, timestampIncrement);
+ }
+ producer.flush();
+ }
+ }
+
+ public static void sendRecords(
+ ClusterInstance cluster,
+ TopicPartition tp,
+ int numRecords,
+ long startingTimestamp
+ ) {
+ sendRecords(cluster, tp, numRecords, startingTimestamp, -1);
+ }
+
+ public static void sendRecords(
+ ClusterInstance cluster,
+ TopicPartition tp,
+ int numRecords
+ ) {
+ sendRecords(cluster, tp, numRecords, System.currentTimeMillis());
+ }
+
+ public static List> sendRecords(
+ Producer producer,
+ TopicPartition tp,
+ int numRecords,
+ long startingTimestamp,
+ long timestampIncrement
+ ) {
+ List> records = new ArrayList<>();
+ for (var i = 0; i < numRecords; i++) {
+ var record = sendRecord(producer, tp, startingTimestamp, i, timestampIncrement);
+ records.add(record);
+ }
+ producer.flush();
+ return records;
+ }
+
+ public static void sendRecords(
+ Producer producer,
+ TopicPartition tp,
+ int numRecords,
+ long startingTimestamp
+ ) {
+ for (var i = 0; i < numRecords; i++) {
+ sendRecord(producer, tp, startingTimestamp, i, -1);
+ }
+ producer.flush();
+ }
+
+ public static void awaitAssignment(
+ Consumer consumer,
+ Set expectedAssignment
+ ) throws InterruptedException {
+ TestUtils.waitForCondition(() -> {
+ consumer.poll(Duration.ofMillis(100));
+ return consumer.assignment().equals(expectedAssignment);
+ }, "Timed out while awaiting expected assignment " + expectedAssignment + ". " +
+ "The current assignment is " + consumer.assignment()
+ );
+ }
+
+ private static ProducerRecord sendRecord(
+ Producer producer,
+ TopicPartition tp,
+ long startingTimestamp,
+ int numRecord,
+ long timestampIncrement
+ ) {
+ var timestamp = startingTimestamp + numRecord * (timestampIncrement > 0 ? timestampIncrement : 1);
+ var record = new ProducerRecord<>(
+ tp.topic(),
+ tp.partition(),
+ timestamp,
+ (KEY_PREFIX + numRecord).getBytes(),
+ (VALUE_PREFIX + numRecord).getBytes()
+ );
+ producer.send(record);
+ return record;
+ }
+
+ public static void sendAndAwaitAsyncCommit(
+ Consumer consumer,
+ Optional> offsetsOpt
+ ) throws InterruptedException {
+
+ var commitCallback = new RetryCommitCallback<>(consumer, offsetsOpt);
+ sendAsyncCommit(consumer, commitCallback, offsetsOpt);
+
+ TestUtils.waitForCondition(() -> {
+ consumer.poll(Duration.ofMillis(100));
+ return commitCallback.isComplete;
+ }, "Failed to observe commit callback before timeout");
+
+ assertEquals(Optional.empty(), commitCallback.error);
+ }
+
+ public static void awaitRebalance(
+ Consumer consumer,
+ TestConsumerReassignmentListener rebalanceListener
+ ) throws InterruptedException {
+ var numReassignments = rebalanceListener.callsToAssigned;
+ TestUtils.waitForCondition(() -> {
+ consumer.poll(Duration.ofMillis(100));
+ return rebalanceListener.callsToAssigned > numReassignments;
+ }, "Timed out before expected rebalance completed");
+ }
+
+ public static void ensureNoRebalance(
+ Consumer consumer,
+ TestConsumerReassignmentListener rebalanceListener
+ ) throws InterruptedException {
+ // The best way to verify that the current membership is still active is to commit offsets.
+ // This would fail if the group had rebalanced.
+ var initialRevokeCalls = rebalanceListener.callsToRevoked;
+ sendAndAwaitAsyncCommit(consumer, Optional.empty());
+ assertEquals(initialRevokeCalls, rebalanceListener.callsToRevoked);
+ }
+
+
+ public static void waitForPollThrowException(
+ Consumer consumer,
+ Class exceptedException
+ ) throws InterruptedException {
+ TestUtils.waitForCondition(() -> {
+ try {
+ consumer.poll(Duration.ZERO);
+ return false;
+ } catch (Exception e) {
+ return exceptedException.isInstance(e);
+ }
+ }, "Continuous poll not fail");
+ }
+
+ /**
+ * This class is intended to replace the test cases in BaseConsumerTest.scala.
+ * When converting tests that extend from BaseConsumerTest.scala to Java,
+ * we should use the test cases provided in this class.
+ */
+ public static final class BaseConsumerTestcase {
+
+ public static final int BROKER_COUNT = 3;
+ public static final String TOPIC = "topic";
+ public static final TopicPartition TP = new TopicPartition(TOPIC, 0);
+
+ private BaseConsumerTestcase() {
+ }
+
+ public static void testSimpleConsumption(
+ ClusterInstance cluster,
+ Map config
+ ) throws InterruptedException {
+ var numRecords = 10000;
+ var startingTimestamp = System.currentTimeMillis();
+ sendRecords(cluster, TP, numRecords, startingTimestamp);
+ try (Consumer consumer = cluster.consumer(config)) {
+ assertEquals(0, consumer.assignment().size());
+ consumer.assign(List.of(TP));
+ assertEquals(1, consumer.assignment().size());
+ consumer.seek(TP, 0);
+ consumeAndVerifyRecords(consumer, TP, numRecords, 0, 0, startingTimestamp);
+ // check async commit callbacks
+ sendAndAwaitAsyncCommit(consumer, Optional.empty());
+ }
+ }
+
+ public static void testClusterResourceListener(
+ ClusterInstance cluster,
+ Map consumerConfig
+ ) throws InterruptedException {
+ var numRecords = 100;
+ Map producerConfig = Map.of(
+ KEY_SERIALIZER_CLASS_CONFIG, TestClusterResourceListenerSerializer.class,
+ VALUE_SERIALIZER_CLASS_CONFIG, TestClusterResourceListenerSerializer.class
+ );
+ Map consumerConfigOverrides = new HashMap<>(consumerConfig);
+ consumerConfigOverrides.put(KEY_DESERIALIZER_CLASS_CONFIG, TestClusterResourceListenerDeserializer.class);
+ consumerConfigOverrides.put(VALUE_DESERIALIZER_CLASS_CONFIG, TestClusterResourceListenerDeserializer.class);
+ try (Producer producer = cluster.producer(producerConfig);
+ Consumer consumer = cluster.consumer(consumerConfigOverrides)
+ ) {
+ var startingTimestamp = System.currentTimeMillis();
+ sendRecords(producer, TP, numRecords, startingTimestamp, -1);
+
+ consumer.subscribe(List.of(TP.topic()));
+ consumeAndVerifyRecords(consumer, TP, numRecords, 0, 0, startingTimestamp);
+ assertNotEquals(0, UPDATE_PRODUCER_COUNT.get());
+ assertNotEquals(0, UPDATE_CONSUMER_COUNT.get());
+
+ TestClusterResourceListenerSerializer.resetCount();
+ TestClusterResourceListenerDeserializer.resetCount();
+ }
+ }
+
+ public static void testCoordinatorFailover(
+ ClusterInstance cluster,
+ Map consumerConfig
+ ) throws InterruptedException {
+ var listener = new TestConsumerReassignmentListener();
+ try (Consumer consumer = cluster.consumer(consumerConfig)) {
+ consumer.subscribe(List.of(TOPIC), listener);
+ // the initial subscription should cause a callback execution
+ awaitRebalance(consumer, listener);
+ assertEquals(1, listener.callsToAssigned);
+
+ // get metadata for the topic
+ List parts = null;
+ while (parts == null) {
+ parts = consumer.partitionsFor(Topic.GROUP_METADATA_TOPIC_NAME);
+ }
+ assertEquals(1, parts.size());
+ assertNotNull(parts.get(0).leader());
+
+ // shutdown the coordinator
+ int coordinator = parts.get(0).leader().id();
+ cluster.shutdownBroker(coordinator);
+
+ // the failover should not cause a rebalance
+ ensureNoRebalance(consumer, listener);
+ }
+ }
+ }
+
+ public static void sendAsyncCommit(
+ Consumer consumer,
+ OffsetCommitCallback callback,
+ Optional> offsetsOpt
+ ) {
+ offsetsOpt.ifPresentOrElse(
+ offsets -> consumer.commitAsync(offsets, callback),
+ () -> consumer.commitAsync(callback)
+ );
+ }
+
+ public static class TestClusterResourceListenerSerializer implements Serializer, ClusterResourceListener {
+
+ public static final AtomicInteger UPDATE_PRODUCER_COUNT = new AtomicInteger();
+
+ @Override
+ public void onUpdate(ClusterResource clusterResource) {
+ UPDATE_PRODUCER_COUNT.incrementAndGet();
+ }
+
+ @Override
+ public byte[] serialize(String topic, byte[] data) {
+ return data;
+ }
+
+ public static void resetCount() {
+ UPDATE_PRODUCER_COUNT.set(0);
+ }
+ }
+
+ public static class TestClusterResourceListenerDeserializer implements Deserializer, ClusterResourceListener {
+
+ public static final AtomicInteger UPDATE_CONSUMER_COUNT = new AtomicInteger();
+
+ @Override
+ public void onUpdate(ClusterResource clusterResource) {
+ UPDATE_CONSUMER_COUNT.incrementAndGet();
+ }
+
+ @Override
+ public byte[] deserialize(String topic, byte[] data) {
+ return data;
+ }
+
+ public static void resetCount() {
+ UPDATE_CONSUMER_COUNT.set(0);
+ }
+ }
+
+ private static class RetryCommitCallback implements OffsetCommitCallback {
+ boolean isComplete = false;
+ Optional error = Optional.empty();
+ Consumer consumer;
+ Optional> offsetsOpt;
+
+ public RetryCommitCallback(
+ Consumer consumer,
+ Optional> offsetsOpt
+ ) {
+ this.consumer = consumer;
+ this.offsetsOpt = offsetsOpt;
+ }
+
+ @Override
+ public void onComplete(Map offsets, Exception exception) {
+ if (exception instanceof RetriableCommitFailedException) {
+ sendAsyncCommit(consumer, this, offsetsOpt);
+ } else {
+ isComplete = true;
+ error = Optional.ofNullable(exception);
+ }
+ }
+ }
+
+ public static class TestConsumerReassignmentListener implements ConsumerRebalanceListener {
+ public int callsToAssigned = 0;
+ public int callsToRevoked = 0;
+
+ @Override
+ public void onPartitionsAssigned(Collection partitions) {
+ callsToAssigned += 1;
+ }
+
+ @Override
+ public void onPartitionsRevoked(Collection partitions) {
+ callsToRevoked += 1;
+ }
+ }
+}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/CreateTopicsRequestWithPolicyTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/CreateTopicsRequestWithPolicyTest.java
new file mode 100644
index 0000000000000..bfbc4794da573
--- /dev/null
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/CreateTopicsRequestWithPolicyTest.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.clients;
+
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.CreateTopicsOptions;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.InvalidPartitionsException;
+import org.apache.kafka.common.errors.InvalidReplicationFactorException;
+import org.apache.kafka.common.errors.PolicyViolationException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.internals.Topic;
+import org.apache.kafka.common.test.ClusterInstance;
+import org.apache.kafka.common.test.api.ClusterConfigProperty;
+import org.apache.kafka.common.test.api.ClusterTest;
+import org.apache.kafka.common.test.api.ClusterTestDefaults;
+import org.apache.kafka.server.config.ServerLogConfigs;
+import org.apache.kafka.server.policy.CreateTopicPolicy;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@ClusterTestDefaults(
+ brokers = 3,
+ serverProperties = {
+ @ClusterConfigProperty(key = ServerLogConfigs.CREATE_TOPIC_POLICY_CLASS_NAME_CONFIG, value = "org.apache.kafka.clients.CreateTopicsRequestWithPolicyTest$Policy"),
+ }
+)
+public class CreateTopicsRequestWithPolicyTest {
+
+ public static class Policy implements CreateTopicPolicy {
+ private Map configs;
+ private boolean closed = false;
+
+ @Override
+ public void configure(Map configs) {
+ this.configs = configs;
+ }
+
+ @Override
+ public void validate(RequestMetadata requestMetadata) throws PolicyViolationException {
+ if (Topic.isInternal(requestMetadata.topic())) {
+ // Do not verify internal topics
+ return;
+ }
+
+ if (closed) {
+ throw new IllegalStateException("Policy should not be closed");
+ }
+ if (configs == null || configs.isEmpty()) {
+ throw new IllegalStateException("Configure should have been called with non empty configs");
+ }
+
+ if (requestMetadata.numPartitions() != null || requestMetadata.replicationFactor() != null) {
+ assertNotNull(requestMetadata.numPartitions(), "numPartitions should not be null, but it is " + requestMetadata.numPartitions());
+ assertNotNull(requestMetadata.replicationFactor(), "replicationFactor should not be null, but it is " + requestMetadata.replicationFactor());
+ assertNull(requestMetadata.replicasAssignments(), "replicaAssignments should be null, but it is " + requestMetadata.replicasAssignments());
+
+ if (requestMetadata.numPartitions() < 5) {
+ throw new PolicyViolationException("Topics should have at least 5 partitions, received " +
+ requestMetadata.numPartitions());
+ }
+
+ if (requestMetadata.numPartitions() > 10) {
+ String retentionMs = requestMetadata.configs().get(TopicConfig.RETENTION_MS_CONFIG);
+ if (retentionMs == null || Integer.parseInt(retentionMs) > 5000) {
+ throw new PolicyViolationException("RetentionMs should be less than 5000ms if partitions > 10");
+ }
+ } else {
+ assertTrue(requestMetadata.configs().isEmpty(), "Topic configs should be empty, but it is " + requestMetadata.configs());
+ }
+ } else {
+ assertNull(requestMetadata.numPartitions(), "numPartitions should be null, but it is " + requestMetadata.numPartitions());
+ assertNull(requestMetadata.replicationFactor(), "replicationFactor should be null, but it is " + requestMetadata.replicationFactor());
+ assertNotNull(requestMetadata.replicasAssignments(), "replicasAssignments should not be null, but it is " + requestMetadata.replicasAssignments());
+
+ requestMetadata.replicasAssignments().forEach((partitionId, replicas) -> {
+ if (replicas.size() < 2) {
+ throw new PolicyViolationException("Topic partitions should have at least 2 replicas, received " +
+ replicas.size() + " for partition " + partitionId);
+ }
+ });
+ }
+ }
+
+ @Override
+ public void close() {
+ closed = true;
+ }
+ }
+
+ private void validateValidCreateTopicsRequests(NewTopic topic, Admin admin, boolean validateOnly) throws Exception {
+ admin.createTopics(
+ List.of(topic),
+ new CreateTopicsOptions().validateOnly(validateOnly)
+ ).all().get();
+ }
+
+ @ClusterTest
+ public void testValidCreateTopicsRequests(ClusterInstance cluster) throws Exception {
+ try (Admin admin = cluster.admin()) {
+ cluster.createTopic("topic1", 5, (short) 1);
+
+ validateValidCreateTopicsRequests(
+ new NewTopic("topic2", 5, (short) 3),
+ admin,
+ true
+ );
+
+ validateValidCreateTopicsRequests(
+ new NewTopic("topic3", 11, (short) 2)
+ .configs(Map.of(TopicConfig.RETENTION_MS_CONFIG, "4999")),
+ admin,
+ true
+ );
+
+ validateValidCreateTopicsRequests(
+ new NewTopic("topic4", Map.of(
+ 0, List.of(1, 0),
+ 1, List.of(0, 1)
+ )),
+ admin,
+ false
+ );
+ }
+ }
+
+ private void validateErrorCreateTopicsRequests(NewTopic topic, Admin admin, boolean validateOnly, Class expectedExceptionClass, String expectedErrorMessage) {
+ ExecutionException exception = assertThrows(ExecutionException.class, () ->
+ admin.createTopics(List.of(topic), new CreateTopicsOptions().validateOnly(validateOnly)).all().get());
+ assertEquals(
+ expectedExceptionClass,
+ exception.getCause().getClass(),
+ "Expected " + expectedExceptionClass.getSimpleName() + ", but got " + exception.getCause().getClass().getSimpleName()
+ );
+ assertTrue(exception.getMessage().contains(expectedErrorMessage));
+ }
+
+ @ClusterTest
+ public void testErrorCreateTopicsRequests(ClusterInstance cluster) throws Exception {
+ try (Admin admin = cluster.admin()) {
+ String existingTopic = "existing-topic";
+ cluster.createTopic(existingTopic, 5, (short) 1);
+
+ // Policy violations
+ validateErrorCreateTopicsRequests(
+ new NewTopic("policy-topic1", 4, (short) 1),
+ admin,
+ false,
+ PolicyViolationException.class,
+ "Topics should have at least 5 partitions, received 4"
+ );
+
+ validateErrorCreateTopicsRequests(
+ new NewTopic("policy-topic2", 4, (short) 3),
+ admin,
+ true,
+ PolicyViolationException.class,
+ "Topics should have at least 5 partitions, received 4"
+ );
+
+ validateErrorCreateTopicsRequests(
+ new NewTopic("policy-topic3", 11, (short) 2)
+ .configs(Map.of(TopicConfig.RETENTION_MS_CONFIG, "5001")),
+ admin,
+ true,
+ PolicyViolationException.class,
+ "RetentionMs should be less than 5000ms if partitions > 10"
+ );
+
+ validateErrorCreateTopicsRequests(
+ new NewTopic("policy-topic4", Map.of(
+ 0, List.of(1),
+ 1, List.of(0)
+ )).configs(Map.of(TopicConfig.RETENTION_MS_CONFIG, "5001")),
+ admin,
+ true,
+ PolicyViolationException.class,
+ "Topic partitions should have at least 2 replicas, received 1 for partition 0"
+ );
+
+ // Check that basic errors still work
+ validateErrorCreateTopicsRequests(
+ new NewTopic(existingTopic, 5, (short) 1),
+ admin,
+ false,
+ TopicExistsException.class,
+ "Topic 'existing-topic' already exists."
+ );
+
+ validateErrorCreateTopicsRequests(
+ new NewTopic("error-replication", 10, (short) 4),
+ admin,
+ true,
+ InvalidReplicationFactorException.class,
+ "Unable to replicate the partition 4 time(s): The target replication factor of 4 cannot be reached because only 3 broker(s) are registered."
+ );
+
+ validateErrorCreateTopicsRequests(
+ new NewTopic("error-replication2", 10, (short) -2),
+ admin,
+ true,
+ InvalidReplicationFactorException.class,
+ "Replication factor must be larger than 0, or -1 to use the default value."
+ );
+
+ validateErrorCreateTopicsRequests(
+ new NewTopic("error-partitions", -2, (short) 1),
+ admin,
+ true,
+ InvalidPartitionsException.class,
+ "Number of partitions was set to an invalid non-positive value."
+ );
+ }
+ }
+}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/MetadataVersionIntegrationTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/MetadataVersionIntegrationTest.java
new file mode 100644
index 0000000000000..3d560892fe87d
--- /dev/null
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/MetadataVersionIntegrationTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients;
+
+import org.apache.kafka.clients.admin.FeatureUpdate;
+import org.apache.kafka.clients.admin.UpdateFeaturesOptions;
+import org.apache.kafka.common.test.ClusterInstance;
+import org.apache.kafka.common.test.api.ClusterTest;
+import org.apache.kafka.common.test.api.ClusterTests;
+import org.apache.kafka.common.test.api.Type;
+import org.apache.kafka.server.common.MetadataVersion;
+import org.apache.kafka.test.TestUtils;
+
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class MetadataVersionIntegrationTest {
+ @ClusterTests(value = {
+ @ClusterTest(types = Type.KRAFT, metadataVersion = MetadataVersion.IBP_3_3_IV3),
+ @ClusterTest(types = Type.KRAFT, metadataVersion = MetadataVersion.IBP_3_4_IV0),
+ @ClusterTest(types = Type.KRAFT, metadataVersion = MetadataVersion.IBP_3_5_IV0),
+ @ClusterTest(types = Type.KRAFT, metadataVersion = MetadataVersion.IBP_3_6_IV0),
+ @ClusterTest(types = Type.KRAFT, metadataVersion = MetadataVersion.IBP_3_6_IV1)
+ })
+ public void testBasicMetadataVersionUpgrade(ClusterInstance clusterInstance) throws Exception {
+ try (var admin = clusterInstance.admin()) {
+ var describeResult = admin.describeFeatures();
+ var ff = describeResult.featureMetadata().get().finalizedFeatures().get(MetadataVersion.FEATURE_NAME);
+ assertEquals(clusterInstance.config().metadataVersion().featureLevel(), ff.minVersionLevel());
+ assertEquals(clusterInstance.config().metadataVersion().featureLevel(), ff.maxVersionLevel());
+
+ // Update to new version
+ short updateVersion = MetadataVersion.IBP_3_7_IV1.featureLevel();
+ var updateResult = admin.updateFeatures(
+ Map.of("metadata.version", new FeatureUpdate(updateVersion, FeatureUpdate.UpgradeType.UPGRADE)),
+ new UpdateFeaturesOptions());
+ updateResult.all().get();
+
+ // Verify that new version is visible on broker
+ TestUtils.waitForCondition(() -> {
+ try {
+ var describeResult2 = admin.describeFeatures();
+ var ff2 = describeResult2.featureMetadata().get().finalizedFeatures().get(MetadataVersion.FEATURE_NAME);
+ return ff2.minVersionLevel() == updateVersion && ff2.maxVersionLevel() == updateVersion;
+ } catch (Exception e) {
+ return false;
+ }
+ }, "Never saw metadata.version increase on broker");
+ }
+ }
+
+ @ClusterTest(types = Type.KRAFT, metadataVersion = MetadataVersion.IBP_3_9_IV0)
+ public void testUpgradeSameVersion(ClusterInstance clusterInstance) throws Exception {
+ try (var admin = clusterInstance.admin()) {
+ short updateVersion = MetadataVersion.IBP_3_9_IV0.featureLevel();
+ var updateResult = admin.updateFeatures(
+ Map.of("metadata.version", new FeatureUpdate(updateVersion, FeatureUpdate.UpgradeType.UPGRADE)),
+ new UpdateFeaturesOptions());
+ updateResult.all().get();
+ }
+ }
+
+ @ClusterTest(types = Type.KRAFT)
+ public void testDefaultIsLatestVersion(ClusterInstance clusterInstance) throws Exception {
+ try (var admin = clusterInstance.admin()) {
+ var describeResult = admin.describeFeatures();
+ var ff = describeResult.featureMetadata().get().finalizedFeatures().get(MetadataVersion.FEATURE_NAME);
+ assertEquals(MetadataVersion.latestTesting().featureLevel(), ff.minVersionLevel(),
+ "If this test fails, check the default MetadataVersion in the @ClusterTest annotation");
+ assertEquals(MetadataVersion.latestTesting().featureLevel(), ff.maxVersionLevel());
+ }
+ }
+}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/MetricsDuringTopicCreationDeletionTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/MetricsDuringTopicCreationDeletionTest.java
new file mode 100644
index 0000000000000..f79ddabd8caf2
--- /dev/null
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/MetricsDuringTopicCreationDeletionTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients;
+
+import org.apache.kafka.common.test.ClusterInstance;
+import org.apache.kafka.common.test.api.ClusterConfigProperty;
+import org.apache.kafka.common.test.api.ClusterTest;
+import org.apache.kafka.common.test.api.Type;
+import org.apache.kafka.server.config.ReplicationConfigs;
+import org.apache.kafka.server.config.ServerConfigs;
+import org.apache.kafka.server.config.ServerLogConfigs;
+import org.apache.kafka.server.metrics.KafkaYammerMetrics;
+
+import com.yammer.metrics.core.Gauge;
+
+import java.io.Closeable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class MetricsDuringTopicCreationDeletionTest {
+
+ private static final String TOPIC_NAME_PREFIX = "topic";
+ private static final int TOPIC_NUM = 2;
+ private static final int CREATE_DELETE_ITERATIONS = 3;
+ private static final short REPLICATION_FACTOR = 1;
+ private static final int PARTITION_NUM = 3;
+
+ private final ClusterInstance clusterInstance;
+ private final List topics;
+ private volatile boolean running = true;
+
+ public MetricsDuringTopicCreationDeletionTest(ClusterInstance clusterInstance) {
+ this.clusterInstance = clusterInstance;
+ this.topics = new ArrayList<>();
+ for (int n = 0; n < TOPIC_NUM; n++) {
+ topics.add(TOPIC_NAME_PREFIX + n);
+ }
+ }
+
+ /*
+ * Checking all metrics we care in a single test is faster though it would be more elegant to have 3 @Test methods
+ */
+ @ClusterTest(
+ types = {Type.KRAFT},
+ brokers = 1,
+ serverProperties = {
+ @ClusterConfigProperty(key = ServerConfigs.DELETE_TOPIC_ENABLE_CONFIG, value = "true"),
+ @ClusterConfigProperty(key = "log.initial.task.delay.ms", value = "100"),
+ @ClusterConfigProperty(key = "log.segment.delete.delay.ms", value = "1000"),
+ @ClusterConfigProperty(key = ServerLogConfigs.AUTO_CREATE_TOPICS_ENABLE_CONFIG, value = "false"),
+ // speed up the test for UnderReplicatedPartitions, which relies on the ISR expiry thread to execute concurrently with topic creation
+ // But the replica.lag.time.max.ms value still need to consider the slow testing environment
+ @ClusterConfigProperty(key = ReplicationConfigs.REPLICA_LAG_TIME_MAX_MS_CONFIG, value = "4000")
+ }
+ )
+ public void testMetricsDuringTopicCreateDelete() throws Exception {
+
+ final int initialOfflinePartitionsCount = getGauge("OfflinePartitionsCount").value();
+ final int initialPreferredReplicaImbalanceCount = getGauge("PreferredReplicaImbalanceCount").value();
+ final int initialUnderReplicatedPartitionsCount = getGauge("UnderReplicatedPartitions").value();
+
+ CompletableFuture future = CompletableFuture.runAsync(() -> {
+ while (running) {
+ int offlinePartitionsCount = getGauge("OfflinePartitionsCount").value();
+ int preferredReplicaImbalanceCount = getGauge("PreferredReplicaImbalanceCount").value();
+ int underReplicatedPartitionsCount = getGauge("UnderReplicatedPartitions").value();
+
+ if (offlinePartitionsCount != initialOfflinePartitionsCount ||
+ preferredReplicaImbalanceCount != initialPreferredReplicaImbalanceCount ||
+ underReplicatedPartitionsCount != initialUnderReplicatedPartitionsCount) {
+ running = false;
+ }
+
+ try {
+ // Avoid busy loop
+ TimeUnit.MILLISECONDS.sleep(100);
+ } catch (InterruptedException ignored) {
+
+ }
+ }
+ });
+
+ Closeable runThread = () -> {
+ running = false;
+ future.join();
+ };
+
+ try (runThread) {
+ createAndDeleteTopics();
+ }
+
+ final int finalOfflinePartitionsCount = getGauge("OfflinePartitionsCount").value();
+ final int finalPreferredReplicaImbalanceCount = getGauge("PreferredReplicaImbalanceCount").value();
+ final int finalUnderReplicatedPartitionsCount = getGauge("UnderReplicatedPartitions").value();
+
+ assertEquals(initialOfflinePartitionsCount, finalOfflinePartitionsCount,
+ "Expect offlinePartitionsCount to be " + initialOfflinePartitionsCount + ", but got: " + finalOfflinePartitionsCount);
+ assertEquals(initialPreferredReplicaImbalanceCount, finalPreferredReplicaImbalanceCount,
+ "Expect PreferredReplicaImbalanceCount to be " + initialPreferredReplicaImbalanceCount + ", but got: " + finalPreferredReplicaImbalanceCount);
+ assertEquals(initialUnderReplicatedPartitionsCount, finalUnderReplicatedPartitionsCount,
+ "Expect UnderReplicatedPartitionCount to be " + initialUnderReplicatedPartitionsCount + ", but got: " + finalUnderReplicatedPartitionsCount);
+ }
+
+ private void createAndDeleteTopics() {
+ for (int i = 1; i <= CREATE_DELETE_ITERATIONS && running; i++) {
+ for (String topic : topics) {
+ if (!running) return;
+ try {
+ clusterInstance.createTopic(topic, PARTITION_NUM, REPLICATION_FACTOR);
+ } catch (Exception ignored) { }
+ }
+
+ for (String topic : topics) {
+ if (!running) return;
+ try {
+ clusterInstance.deleteTopic(topic);
+ } catch (Exception ignored) { }
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private Gauge getGauge(String metricName) {
+ return KafkaYammerMetrics.defaultRegistry().allMetrics().entrySet().stream()
+ .filter(entry -> entry.getKey().getName().endsWith(metricName))
+ .findFirst()
+ .map(entry -> (Gauge) entry.getValue())
+ .orElseThrow(() -> new AssertionError("Unable to find metric " + metricName));
+ }
+}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/TransactionsExpirationTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/TransactionsExpirationTest.java
index adb55f965afae..01eca9afb971a 100644
--- a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/TransactionsExpirationTest.java
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/TransactionsExpirationTest.java
@@ -34,7 +34,6 @@
import org.apache.kafka.common.header.Header;
import org.apache.kafka.common.header.internals.RecordHeader;
import org.apache.kafka.common.test.ClusterInstance;
-import org.apache.kafka.common.test.TestUtils;
import org.apache.kafka.common.test.api.ClusterConfigProperty;
import org.apache.kafka.common.test.api.ClusterFeature;
import org.apache.kafka.common.test.api.ClusterTest;
@@ -47,6 +46,7 @@
import org.apache.kafka.server.config.ReplicationConfigs;
import org.apache.kafka.server.config.ServerConfigs;
import org.apache.kafka.server.config.ServerLogConfigs;
+import org.apache.kafka.test.TestUtils;
import java.time.Duration;
import java.util.ArrayList;
@@ -165,8 +165,8 @@ private void testFatalErrorAfterInvalidProducerIdMapping(ClusterInstance cluster
private void testTransactionAfterProducerIdExpires(ClusterInstance clusterInstance, boolean isTV2Enabled) throws InterruptedException {
clusterInstance.createTopic(TOPIC1, 4, (short) 3);
- long oldProducerId = 0;
- long oldProducerEpoch = 0;
+ long oldProducerId;
+ long oldProducerEpoch;
try (Producer producer = clusterInstance.producer(Map.of(
ProducerConfig.TRANSACTIONAL_ID_CONFIG, TRANSACTION_ID
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/TransactionsWithMaxInFlightOneTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/TransactionsWithMaxInFlightOneTest.java
index fb27f9be4f03f..d95901249c009 100644
--- a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/TransactionsWithMaxInFlightOneTest.java
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/TransactionsWithMaxInFlightOneTest.java
@@ -27,7 +27,6 @@
import org.apache.kafka.common.header.Header;
import org.apache.kafka.common.header.internals.RecordHeader;
import org.apache.kafka.common.test.ClusterInstance;
-import org.apache.kafka.common.test.TestUtils;
import org.apache.kafka.common.test.api.ClusterConfigProperty;
import org.apache.kafka.common.test.api.ClusterTest;
import org.apache.kafka.common.test.api.ClusterTestDefaults;
@@ -38,6 +37,7 @@
import org.apache.kafka.server.config.ReplicationConfigs;
import org.apache.kafka.server.config.ServerConfigs;
import org.apache.kafka.server.config.ServerLogConfigs;
+import org.apache.kafka.test.TestUtils;
import java.time.Duration;
import java.util.ArrayList;
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/AdminClientWithPoliciesIntegrationTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/AdminClientWithPoliciesIntegrationTest.java
new file mode 100644
index 0000000000000..8bad3b1c9006c
--- /dev/null
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/AdminClientWithPoliciesIntegrationTest.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.clients.admin;
+
+import org.apache.kafka.clients.admin.AlterConfigOp.OpType;
+import org.apache.kafka.common.config.ConfigResource;
+import org.apache.kafka.common.config.SslConfigs;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.InvalidConfigurationException;
+import org.apache.kafka.common.errors.InvalidRequestException;
+import org.apache.kafka.common.errors.PolicyViolationException;
+import org.apache.kafka.common.test.ClusterInstance;
+import org.apache.kafka.common.test.api.ClusterConfigProperty;
+import org.apache.kafka.common.test.api.ClusterTest;
+import org.apache.kafka.common.test.api.ClusterTestDefaults;
+import org.apache.kafka.network.SocketServerConfigs;
+import org.apache.kafka.server.config.ServerConfigs;
+import org.apache.kafka.server.config.ServerLogConfigs;
+import org.apache.kafka.server.policy.AlterConfigPolicy;
+import org.apache.kafka.storage.internals.log.LogConfig;
+
+import org.junit.jupiter.api.BeforeEach;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.kafka.test.TestUtils.assertFutureThrows;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Tests AdminClient calls when the broker is configured with policies - AlterConfigPolicy.
+ */
+
+@ClusterTestDefaults(
+ brokers = 3,
+ serverProperties = {
+ @ClusterConfigProperty(key = ServerLogConfigs.ALTER_CONFIG_POLICY_CLASS_NAME_CONFIG, value = "org.apache.kafka.clients.admin.AdminClientWithPoliciesIntegrationTest$Policy"),
+ }
+)
+public class AdminClientWithPoliciesIntegrationTest {
+ private final ClusterInstance clusterInstance;
+ private static List validations = new ArrayList<>();
+
+ AdminClientWithPoliciesIntegrationTest(ClusterInstance clusterInstance) {
+ this.clusterInstance = clusterInstance;
+ }
+
+ @BeforeEach
+ public void setup() throws InterruptedException {
+ clusterInstance.waitForReadyBrokers();
+ }
+
+ @ClusterTest
+ public void testInvalidAlterConfigsDueToPolicy() throws Exception {
+ try (final Admin adminClient = clusterInstance.admin()) {
+ // Create topics
+ String topic1 = "invalid-alter-configs-due-to-policy-topic-1";
+ String topic2 = "invalid-alter-configs-due-to-policy-topic-2";
+ String topic3 = "invalid-alter-configs-due-to-policy-topic-3";
+ clusterInstance.createTopic(topic1, 1, (short) 1);
+ clusterInstance.createTopic(topic2, 1, (short) 1);
+ clusterInstance.createTopic(topic3, 1, (short) 1);
+
+ ConfigResource topicResource1 = new ConfigResource(ConfigResource.Type.TOPIC, topic1);
+ ConfigResource topicResource2 = new ConfigResource(ConfigResource.Type.TOPIC, topic2);
+ ConfigResource topicResource3 = new ConfigResource(ConfigResource.Type.TOPIC, topic3);
+
+ // Set a mutable broker config
+ ConfigResource brokerResource = new ConfigResource(ConfigResource.Type.BROKER, "0"); // "0" represents the broker ID
+ Map> configOps = Map.of(
+ brokerResource, List.of(new AlterConfigOp(new ConfigEntry(ServerConfigs.MESSAGE_MAX_BYTES_CONFIG, "50000"), OpType.SET))
+ );
+ adminClient.incrementalAlterConfigs(configOps).all().get();
+ assertEquals(Set.of(ServerConfigs.MESSAGE_MAX_BYTES_CONFIG), validationsForResource(brokerResource).get(0).configs().keySet());
+ validations.clear();
+
+ Map> alterConfigs = new HashMap<>();
+ alterConfigs.put(topicResource1, List.of(
+ new AlterConfigOp(new ConfigEntry(TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG, "0.9"), OpType.SET),
+ new AlterConfigOp(new ConfigEntry(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "2"), OpType.SET)
+ ));
+ alterConfigs.put(topicResource2, List.of(new AlterConfigOp(new ConfigEntry(TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG, "0.8"), OpType.SET)));
+ alterConfigs.put(topicResource3, List.of(new AlterConfigOp(new ConfigEntry(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "-1"), OpType.SET)));
+ alterConfigs.put(brokerResource, List.of(new AlterConfigOp(new ConfigEntry(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, "12313"), OpType.SET)));
+
+ // Alter configs: second is valid, the others are invalid
+ AlterConfigsResult alterResult = adminClient.incrementalAlterConfigs(alterConfigs);
+ assertEquals(Set.of(topicResource1, topicResource2, topicResource3, brokerResource), alterResult.values().keySet());
+ assertFutureThrows(PolicyViolationException.class, alterResult.values().get(topicResource1));
+ alterResult.values().get(topicResource2).get();
+ assertFutureThrows(InvalidConfigurationException.class, alterResult.values().get(topicResource3));
+ assertFutureThrows(InvalidRequestException.class, alterResult.values().get(brokerResource));
+ assertTrue(validationsForResource(brokerResource).isEmpty(),
+ "Should not see the broker resource in the AlterConfig policy when the broker configs are not being updated.");
+ validations.clear();
+
+ // Verify that the second resource was updated and the others were not
+ clusterInstance.ensureConsistentMetadata();
+ DescribeConfigsResult describeResult = adminClient.describeConfigs(List.of(topicResource1, topicResource2, topicResource3, brokerResource));
+ var configs = describeResult.all().get();
+ assertEquals(4, configs.size());
+
+ assertEquals(String.valueOf(LogConfig.DEFAULT_MIN_CLEANABLE_DIRTY_RATIO), configs.get(topicResource1).get(TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG).value());
+ assertEquals(String.valueOf(ServerLogConfigs.MIN_IN_SYNC_REPLICAS_DEFAULT), configs.get(topicResource1).get(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG).value());
+
+ assertEquals("0.8", configs.get(topicResource2).get(TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG).value());
+
+ assertNull(configs.get(brokerResource).get(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG).value());
+
+ // Alter configs with validateOnly = true: only second is valid
+ alterConfigs.put(topicResource2, List.of(new AlterConfigOp(new ConfigEntry(TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG, "0.7"), OpType.SET)));
+ alterResult = adminClient.incrementalAlterConfigs(alterConfigs, new AlterConfigsOptions().validateOnly(true));
+
+ assertFutureThrows(PolicyViolationException.class, alterResult.values().get(topicResource1));
+ alterResult.values().get(topicResource2).get();
+ assertFutureThrows(InvalidConfigurationException.class, alterResult.values().get(topicResource3));
+ assertFutureThrows(InvalidRequestException.class, alterResult.values().get(brokerResource));
+ assertTrue(validationsForResource(brokerResource).isEmpty(),
+ "Should not see the broker resource in the AlterConfig policy when the broker configs are not being updated.");
+ validations.clear();
+
+ // Verify that no resources are updated since validate_only = true
+ clusterInstance.ensureConsistentMetadata();
+ describeResult = adminClient.describeConfigs(List.of(topicResource1, topicResource2, topicResource3, brokerResource));
+ configs = describeResult.all().get();
+ assertEquals(4, configs.size());
+
+ assertEquals(String.valueOf(LogConfig.DEFAULT_MIN_CLEANABLE_DIRTY_RATIO), configs.get(topicResource1).get(TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG).value());
+ assertEquals(String.valueOf(ServerLogConfigs.MIN_IN_SYNC_REPLICAS_DEFAULT), configs.get(topicResource1).get(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG).value());
+
+ assertEquals("0.8", configs.get(topicResource2).get(TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG).value());
+
+ assertNull(configs.get(brokerResource).get(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG).value());
+
+ // Do an incremental alter config on the broker, ensure we don't see the broker config we set earlier in the policy
+ alterResult = adminClient.incrementalAlterConfigs(Map.of(
+ brokerResource, List.of(new AlterConfigOp(new ConfigEntry(SocketServerConfigs.MAX_CONNECTIONS_CONFIG, "9999"), OpType.SET))
+ ));
+ alterResult.all().get();
+ assertEquals(Set.of(SocketServerConfigs.MAX_CONNECTIONS_CONFIG), validationsForResource(brokerResource).get(0).configs().keySet());
+ }
+ }
+
+ private static List validationsForResource(ConfigResource resource) {
+ return validations.stream().filter(req -> req.resource().equals(resource)).toList();
+ }
+
+ /**
+ * Used in @ClusterTestDefaults serverProperties, so it may appear unused in the IDE.
+ */
+ public static class Policy implements AlterConfigPolicy {
+ private Map configs;
+ private boolean closed = false;
+
+
+ @Override
+ public void configure(Map configs) {
+ validations.clear();
+ this.configs = configs;
+ }
+
+ @Override
+ public void validate(AlterConfigPolicy.RequestMetadata requestMetadata) {
+ validations.add(requestMetadata);
+ assertFalse(closed, "Policy should not be closed");
+ assertFalse(configs.isEmpty(), "configure should have been called with non empty configs");
+ assertFalse(requestMetadata.configs().isEmpty(), "request configs should not be empty");
+ assertFalse(requestMetadata.resource().name().isEmpty(), "resource name should not be empty");
+ if (requestMetadata.configs().containsKey(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG)) {
+ throw new PolicyViolationException("Min in sync replicas cannot be updated");
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.closed = true;
+ }
+ }
+}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/ClientTelemetryTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/ClientTelemetryTest.java
index 008b6096c3d79..70881ef8d8a37 100644
--- a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/ClientTelemetryTest.java
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/ClientTelemetryTest.java
@@ -46,6 +46,7 @@
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -54,7 +55,6 @@
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
import static java.util.Arrays.asList;
import static org.apache.kafka.clients.admin.AdminClientConfig.METRIC_REPORTER_CLASSES_CONFIG;
@@ -79,7 +79,7 @@ public void testClientInstanceId(ClusterInstance clusterInstance) throws Interru
try (Admin admin = Admin.create(configs)) {
String testTopicName = "test_topic";
admin.createTopics(Collections.singletonList(new NewTopic(testTopicName, 1, (short) 1)));
- clusterInstance.waitForTopic(testTopicName, 1);
+ clusterInstance.waitTopicCreation(testTopicName, 1);
Map producerConfigs = new HashMap<>();
producerConfigs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, clusterInstance.bootstrapServers());
@@ -128,7 +128,7 @@ public void testIntervalMsParser(ClusterInstance clusterInstance) {
List alterOpts = asList("--bootstrap-server", clusterInstance.bootstrapServers(),
"--alter", "--entity-type", "client-metrics", "--entity-name", "test", "--add-config", "interval.ms=bbb");
try (Admin client = clusterInstance.admin()) {
- ConfigCommand.ConfigCommandOptions addOpts = new ConfigCommand.ConfigCommandOptions(toArray(alterOpts));
+ ConfigCommand.ConfigCommandOptions addOpts = new ConfigCommand.ConfigCommandOptions(toArray(Set.of(alterOpts)));
Throwable e = assertThrows(ExecutionException.class, () -> ConfigCommand.alterConfig(client, addOpts));
assertTrue(e.getMessage().contains(InvalidConfigurationException.class.getSimpleName()));
@@ -152,8 +152,8 @@ public void testMetrics(ClusterInstance clusterInstance) {
}
}
- private static String[] toArray(List... lists) {
- return Stream.of(lists).flatMap(List::stream).toArray(String[]::new);
+ private static String[] toArray(Collection> lists) {
+ return lists.stream().flatMap(List::stream).toArray(String[]::new);
}
/**
@@ -161,6 +161,7 @@ private static String[] toArray(List... lists) {
* {@link org.apache.kafka.common.protocol.ApiKeys#GET_TELEMETRY_SUBSCRIPTIONS} command will not be supported
* by the server
**/
+ @SuppressWarnings("unused")
public static class GetIdClientTelemetry implements ClientTelemetry, MetricsReporter {
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/DeleteTopicTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/DeleteTopicTest.java
index fceebc0c1b62c..a803b426f50f6 100644
--- a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/DeleteTopicTest.java
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/DeleteTopicTest.java
@@ -25,7 +25,6 @@
import org.apache.kafka.common.record.MemoryRecords;
import org.apache.kafka.common.record.SimpleRecord;
import org.apache.kafka.common.test.ClusterInstance;
-import org.apache.kafka.common.test.TestUtils;
import org.apache.kafka.common.test.api.ClusterConfigProperty;
import org.apache.kafka.common.test.api.ClusterTest;
import org.apache.kafka.common.test.api.ClusterTestDefaults;
@@ -37,6 +36,7 @@
import org.apache.kafka.storage.internals.log.AppendOrigin;
import org.apache.kafka.storage.internals.log.UnifiedLog;
import org.apache.kafka.storage.internals.log.VerificationGuard;
+import org.apache.kafka.test.TestUtils;
import java.util.ArrayList;
import java.util.Collection;
@@ -70,7 +70,7 @@ public void testDeleteTopicWithAllAliveReplicas(ClusterInstance cluster) throws
try (Admin admin = cluster.admin()) {
admin.createTopics(List.of(new NewTopic(DEFAULT_TOPIC, expectedReplicaAssignment))).all().get();
admin.deleteTopics(List.of(DEFAULT_TOPIC)).all().get();
- cluster.waitForTopic(DEFAULT_TOPIC, 0);
+ cluster.waitTopicDeletion(DEFAULT_TOPIC);
}
}
@@ -93,7 +93,7 @@ public void testResumeDeleteTopicWithRecoveredFollower(ClusterInstance cluster)
"Online replicas have not deleted log.");
follower.startup();
- cluster.waitForTopic(DEFAULT_TOPIC, 0);
+ cluster.waitTopicDeletion(DEFAULT_TOPIC);
}
}
@@ -119,7 +119,7 @@ public void testPartitionReassignmentDuringDeleteTopic(ClusterInstance cluster)
}
follower.startup();
- cluster.waitForTopic(DEFAULT_TOPIC, 0);
+ cluster.waitTopicDeletion(DEFAULT_TOPIC);
}
}
@@ -147,7 +147,7 @@ public void testIncreasePartitionCountDuringDeleteTopic(ClusterInstance cluster)
}
follower.startup();
- cluster.waitForTopic(DEFAULT_TOPIC, 0);
+ cluster.waitTopicDeletion(DEFAULT_TOPIC);
}
}
@@ -166,12 +166,12 @@ public void testDeleteTopicDuringAddPartition(ClusterInstance cluster) throws Ex
"Follower " + followerBrokerId + " was not shutdown");
Map newPartitionSet = Map.of(DEFAULT_TOPIC, NewPartitions.increaseTo(3));
admin.createPartitions(newPartitionSet);
- cluster.waitForTopic(DEFAULT_TOPIC, 3);
+ cluster.waitTopicCreation(DEFAULT_TOPIC, 3);
admin.deleteTopics(List.of(DEFAULT_TOPIC)).all().get();
follower.startup();
// test if topic deletion is resumed
- cluster.waitForTopic(DEFAULT_TOPIC, 0);
+ cluster.waitTopicDeletion(DEFAULT_TOPIC);
waitForReplicaDeleted(cluster.brokers(), newTopicPartition, "Replica logs not for new partition [" + DEFAULT_TOPIC + ",1] not deleted after delete topic is complete.");
}
}
@@ -185,7 +185,7 @@ public void testAddPartitionDuringDeleteTopic(ClusterInstance cluster) throws Ex
admin.deleteTopics(List.of(DEFAULT_TOPIC)).all().get();
Map newPartitionSet = Map.of(DEFAULT_TOPIC, NewPartitions.increaseTo(3));
admin.createPartitions(newPartitionSet);
- cluster.waitForTopic(DEFAULT_TOPIC, 0);
+ cluster.waitTopicDeletion(DEFAULT_TOPIC);
waitForReplicaDeleted(cluster.brokers(), newTopicPartition, "Replica logs not deleted after delete topic is complete");
}
}
@@ -196,12 +196,13 @@ public void testRecreateTopicAfterDeletion(ClusterInstance cluster) throws Excep
admin.createTopics(List.of(new NewTopic(DEFAULT_TOPIC, expectedReplicaAssignment))).all().get();
TopicPartition topicPartition = new TopicPartition(DEFAULT_TOPIC, 0);
admin.deleteTopics(List.of(DEFAULT_TOPIC)).all().get();
- cluster.waitForTopic(DEFAULT_TOPIC, 0);
+ cluster.waitTopicDeletion(DEFAULT_TOPIC);
// re-create topic on same replicas
admin.createTopics(List.of(new NewTopic(DEFAULT_TOPIC, expectedReplicaAssignment))).all().get();
waitForReplicaCreated(cluster.brokers(), topicPartition, "Replicas for topic " + DEFAULT_TOPIC + " not created.");
}
}
+
@ClusterTest
public void testDeleteNonExistingTopic(ClusterInstance cluster) throws Exception {
try (Admin admin = cluster.admin()) {
@@ -217,10 +218,10 @@ public void testDeleteNonExistingTopic(ClusterInstance cluster) throws Exception
}
}, "Topic test2 should not exist.");
- cluster.waitForTopic(topic, 0);
+ cluster.waitTopicDeletion(topic);
waitForReplicaCreated(cluster.brokers(), topicPartition, "Replicas for topic test not created.");
- TestUtils.waitUntilLeaderIsElectedOrChangedWithAdmin(admin, DEFAULT_TOPIC, 0, 1000);
+ cluster.waitUntilLeaderIsElectedOrChangedWithAdmin(admin, DEFAULT_TOPIC, 0, 1000);
}
}
@@ -245,7 +246,7 @@ public void testDeleteTopicWithCleaner(ClusterInstance cluster) throws Exception
server.logManager().cleaner().awaitCleaned(topicPartition, 0, 60000);
admin.deleteTopics(List.of(DEFAULT_TOPIC)).all().get();
- cluster.waitForTopic(DEFAULT_TOPIC, 0);
+ cluster.waitTopicDeletion(DEFAULT_TOPIC);
}
}
@@ -264,7 +265,7 @@ public void testDeleteTopicAlreadyMarkedAsDeleted(ClusterInstance cluster) throw
}
}, "Topic " + DEFAULT_TOPIC + " should be marked for deletion or already deleted.");
- cluster.waitForTopic(DEFAULT_TOPIC, 0);
+ cluster.waitTopicDeletion(DEFAULT_TOPIC);
}
}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/DescribeAuthorizedOperationsTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/DescribeAuthorizedOperationsTest.java
index d4ad50a8f74b6..1464fc06eb10f 100644
--- a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/DescribeAuthorizedOperationsTest.java
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/DescribeAuthorizedOperationsTest.java
@@ -69,8 +69,10 @@ static List generator() {
return List.of(
ClusterConfig.defaultBuilder()
.setTypes(Set.of(Type.KRAFT))
- .setServerProperties(Map.of(GroupCoordinatorConfig.OFFSETS_TOPIC_PARTITIONS_CONFIG, "1"))
- .setServerProperties(Map.of(GroupCoordinatorConfig.OFFSETS_TOPIC_REPLICATION_FACTOR_CONFIG, "1"))
+ .setServerProperties(Map.of(
+ GroupCoordinatorConfig.OFFSETS_TOPIC_PARTITIONS_CONFIG, "1",
+ GroupCoordinatorConfig.OFFSETS_TOPIC_REPLICATION_FACTOR_CONFIG, "1"
+ ))
.setBrokerSecurityProtocol(SecurityProtocol.SASL_PLAINTEXT)
.setControllerSecurityProtocol(SecurityProtocol.SASL_PLAINTEXT)
.build()
@@ -113,10 +115,10 @@ private void setupSecurity(ClusterInstance clusterInstance) throws ExecutionExce
public void testConsumerGroupAuthorizedOperations(ClusterInstance clusterInstance) throws ExecutionException, InterruptedException {
setupSecurity(clusterInstance);
try (Admin admin = clusterInstance.admin(createAdminConfig(JaasUtils.KAFKA_PLAIN_ADMIN, JaasUtils.KAFKA_PLAIN_ADMIN_PASSWORD));
- Admin user1 = clusterInstance.admin(createAdminConfig(JaasUtils.KAFKA_PLAIN_USER1, JaasUtils.KAFKA_PLAIN_USER1_PASSWORD));
+ Admin user1 = clusterInstance.admin(createAdminConfig(JaasUtils.KAFKA_PLAIN_USER1, JaasUtils.KAFKA_PLAIN_USER1_PASSWORD))
) {
admin.createTopics(List.of(new NewTopic("topic1", 1, (short) 1)));
- clusterInstance.waitForTopic("topic1", 1);
+ clusterInstance.waitTopicCreation("topic1", 1);
// create consumers to avoid group not found error
TopicPartition tp = new TopicPartition("topic1", 0);
@@ -191,8 +193,8 @@ public void testTopicAuthorizedOperations(ClusterInstance clusterInstance) throw
new NewTopic(topic1, 1, (short) 1),
new NewTopic(topic2, 1, (short) 1)
));
- clusterInstance.waitForTopic(topic1, 1);
- clusterInstance.waitForTopic(topic2, 1);
+ clusterInstance.waitTopicCreation(topic1, 1);
+ clusterInstance.waitTopicCreation(topic2, 1);
}
try (Admin admin = clusterInstance.admin(createAdminConfig(JaasUtils.KAFKA_PLAIN_USER1, JaasUtils.KAFKA_PLAIN_USER1_PASSWORD))) {
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/DescribeProducersWithBrokerIdTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/DescribeProducersWithBrokerIdTest.java
new file mode 100644
index 0000000000000..979989af1cafa
--- /dev/null
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/DescribeProducersWithBrokerIdTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.admin;
+
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.Node;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.NotLeaderOrFollowerException;
+import org.apache.kafka.common.test.ClusterInstance;
+import org.apache.kafka.common.test.api.ClusterTest;
+import org.apache.kafka.common.test.api.ClusterTestDefaults;
+import org.apache.kafka.test.TestUtils;
+
+import org.junit.jupiter.api.BeforeEach;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+
+@ClusterTestDefaults(
+ brokers = 3
+)
+class DescribeProducersWithBrokerIdTest {
+ private static final String TOPIC_NAME = "test-topic";
+ private static final int NUM_PARTITIONS = 1;
+ private static final short REPLICATION_FACTOR = 3;
+
+ private static final TopicPartition TOPIC_PARTITION = new TopicPartition(TOPIC_NAME, 0);
+
+ private final ClusterInstance clusterInstance;
+
+ public DescribeProducersWithBrokerIdTest(ClusterInstance clusterInstance) {
+ this.clusterInstance = clusterInstance;
+ }
+
+ private static void sendTestRecords(Producer producer) {
+ producer.send(new ProducerRecord<>(TOPIC_NAME, TOPIC_PARTITION.partition(), "key-0".getBytes(), "value-0".getBytes()));
+ producer.flush();
+ }
+
+ @BeforeEach
+ void setUp() throws InterruptedException {
+ clusterInstance.createTopic(TOPIC_NAME, NUM_PARTITIONS, REPLICATION_FACTOR);
+ }
+
+ private List getReplicaBrokerIds(Admin admin) throws Exception {
+ var topicDescription = admin.describeTopics(List.of(TOPIC_PARTITION.topic())).allTopicNames().get().get(TOPIC_PARTITION.topic());
+ return topicDescription.partitions().get(TOPIC_PARTITION.partition()).replicas().stream()
+ .map(Node::id)
+ .toList();
+ }
+
+ private int getNonReplicaBrokerId(Admin admin) throws Exception {
+ var replicaBrokerIds = getReplicaBrokerIds(admin);
+ return clusterInstance.brokerIds().stream()
+ .filter(id -> !replicaBrokerIds.contains(id))
+ .findFirst()
+ .orElseThrow(() -> new IllegalStateException("No non-replica broker found"));
+ }
+
+ private int getFollowerBrokerId(Admin admin) throws Exception {
+ var replicaBrokerIds = getReplicaBrokerIds(admin);
+ var leaderBrokerId = clusterInstance.getLeaderBrokerId(TOPIC_PARTITION);
+ return replicaBrokerIds.stream()
+ .filter(id -> id != leaderBrokerId)
+ .findFirst()
+ .orElseThrow(() -> new IllegalStateException("No follower found for partition " + TOPIC_PARTITION));
+ }
+
+ @ClusterTest
+ void testDescribeProducersDefaultRoutesToLeader() throws Exception {
+ try (Producer producer = clusterInstance.producer();
+ var admin = clusterInstance.admin()) {
+ sendTestRecords(producer);
+
+ var stateWithExplicitLeader = admin.describeProducers(
+ List.of(TOPIC_PARTITION),
+ new DescribeProducersOptions().brokerId(clusterInstance.getLeaderBrokerId(TOPIC_PARTITION))
+ ).partitionResult(TOPIC_PARTITION).get();
+
+ var stateWithDefaultRouting = admin.describeProducers(
+ List.of(TOPIC_PARTITION)
+ ).partitionResult(TOPIC_PARTITION).get();
+
+ assertNotNull(stateWithDefaultRouting);
+ assertFalse(stateWithDefaultRouting.activeProducers().isEmpty());
+ assertEquals(stateWithExplicitLeader.activeProducers(), stateWithDefaultRouting.activeProducers());
+ }
+ }
+
+ @ClusterTest
+ void testDescribeProducersFromFollower() throws Exception {
+ try (Producer producer = clusterInstance.producer();
+ var admin = clusterInstance.admin()) {
+ sendTestRecords(producer);
+
+ var followerState = admin.describeProducers(
+ List.of(TOPIC_PARTITION),
+ new DescribeProducersOptions().brokerId(getFollowerBrokerId(admin))
+ ).partitionResult(TOPIC_PARTITION).get();
+
+ var leaderState = admin.describeProducers(
+ List.of(TOPIC_PARTITION)
+ ).partitionResult(TOPIC_PARTITION).get();
+
+ assertNotNull(followerState);
+ assertFalse(followerState.activeProducers().isEmpty());
+ assertEquals(leaderState.activeProducers(), followerState.activeProducers());
+ }
+ }
+
+ @ClusterTest(brokers = 4)
+ void testDescribeProducersWithInvalidBrokerId() throws Exception {
+ try (Producer producer = clusterInstance.producer();
+ var admin = clusterInstance.admin()) {
+ sendTestRecords(producer);
+
+ TestUtils.assertFutureThrows(NotLeaderOrFollowerException.class,
+ admin.describeProducers(
+ List.of(TOPIC_PARTITION),
+ new DescribeProducersOptions().brokerId(getNonReplicaBrokerId(admin))
+ ).partitionResult(TOPIC_PARTITION));
+ }
+ }
+}
\ No newline at end of file
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/ListOffsetsIntegrationTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/ListOffsetsIntegrationTest.java
index 6b40203c01260..678a3b23b46a7 100644
--- a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/ListOffsetsIntegrationTest.java
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/ListOffsetsIntegrationTest.java
@@ -28,12 +28,12 @@
import org.apache.kafka.common.record.CompressionType;
import org.apache.kafka.common.requests.ListOffsetsResponse;
import org.apache.kafka.common.test.ClusterInstance;
-import org.apache.kafka.common.test.TestUtils;
import org.apache.kafka.common.test.api.ClusterConfigProperty;
import org.apache.kafka.common.test.api.ClusterTest;
import org.apache.kafka.common.test.api.ClusterTestDefaults;
import org.apache.kafka.common.test.api.Type;
import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.test.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -44,13 +44,9 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import scala.jdk.javaapi.CollectionConverters;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -242,9 +238,8 @@ private void verifyListOffsets(String topic, int expectedMaxTimestampOffset) thr
// case 2: test the offsets from recovery path.
// server will rebuild offset index according to log files if the index files are nonexistent
- Set indexFiles = clusterInstance.brokers().values().stream().flatMap(broker ->
- CollectionConverters.asJava(broker.config().logDirs()).stream()
- ).collect(Collectors.toUnmodifiableSet());
+ List indexFiles = clusterInstance.brokers().values().stream().flatMap(broker ->
+ broker.config().logDirs().stream()).toList();
clusterInstance.brokers().values().forEach(KafkaBroker::shutdown);
indexFiles.forEach(root -> {
File[] files = new File(String.format("%s/%s-0", root, topic)).listFiles();
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/RackAwareAutoTopicCreationTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/RackAwareAutoTopicCreationTest.java
index e3b9efb9b6f94..d394dcb627410 100644
--- a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/RackAwareAutoTopicCreationTest.java
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/RackAwareAutoTopicCreationTest.java
@@ -112,9 +112,7 @@ private static Map> getTopicAssignment(Admin admin) throw
private static Map getBrokerToRackMap(ClusterInstance cluster) throws Exception {
Map actualBrokerToRackMap = new HashMap<>();
try (Admin admin = cluster.admin()) {
- admin.describeCluster().nodes().get().forEach(node -> {
- actualBrokerToRackMap.put(node.id(), node.rack());
- });
+ admin.describeCluster().nodes().get().forEach(node -> actualBrokerToRackMap.put(node.id(), node.rack()));
}
return actualBrokerToRackMap;
}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/StaticBrokerConfigTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/StaticBrokerConfigTest.java
index 58240c0711e62..8ba228dd25234 100644
--- a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/StaticBrokerConfigTest.java
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/admin/StaticBrokerConfigTest.java
@@ -17,20 +17,31 @@
package org.apache.kafka.clients.admin;
+import kafka.server.KafkaConfig;
+
+import org.apache.kafka.common.config.ConfigDef;
import org.apache.kafka.common.config.ConfigResource;
import org.apache.kafka.common.config.TopicConfig;
import org.apache.kafka.common.test.ClusterInstance;
import org.apache.kafka.common.test.api.ClusterConfigProperty;
import org.apache.kafka.common.test.api.ClusterTest;
import org.apache.kafka.common.test.api.Type;
+import org.apache.kafka.coordinator.group.GroupConfig;
import org.apache.kafka.coordinator.group.GroupCoordinatorConfig;
+import org.apache.kafka.server.config.KRaftConfigs;
+import org.apache.kafka.server.config.ServerConfigs;
+import org.apache.kafka.server.metrics.ClientMetricsConfigs;
+import org.apache.kafka.storage.internals.log.LogConfig;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
public class StaticBrokerConfigTest {
private static final String TOPIC = "topic";
@@ -108,4 +119,80 @@ public void testTopicConfigsGetImpactedIfStaticConfigsAddToBroker(ClusterInstanc
"Config value should not be custom value since controller doesn't have related static config");
}
}
+
+ @ClusterTest(types = {Type.KRAFT})
+ public void testInternalConfigsDoNotReturnForDescribeConfigs(ClusterInstance cluster) throws Exception {
+ try (
+ Admin admin = cluster.admin();
+ Admin controllerAdmin = cluster.admin(Map.of(), true)
+ ) {
+ ConfigResource brokerResource = new ConfigResource(ConfigResource.Type.BROKER, "0");
+ ConfigResource topicResource = new ConfigResource(ConfigResource.Type.TOPIC, TOPIC);
+ ConfigResource groupResource = new ConfigResource(ConfigResource.Type.GROUP, "testGroup");
+ ConfigResource clientMetricsResource = new ConfigResource(ConfigResource.Type.CLIENT_METRICS, "testClient");
+
+ admin.createTopics(List.of(new NewTopic(TOPIC, 1, (short) 1))).config(TOPIC).get();
+ // make sure the topic metadata exist
+ cluster.waitTopicCreation(TOPIC, 1);
+ Map configResourceMap = admin.describeConfigs(
+ List.of(brokerResource, topicResource, groupResource, clientMetricsResource)).all().get();
+
+ // test for case ConfigResource.Type == BROKER
+ // Notice: since the testing framework actively sets three internal configurations when starting the
+ // broker (see org.apache.kafka.common.test.KafkaClusterTestKit.Builder.createNodeConfig()),
+ // so the API `describeConfigs` will also return these three configurations. However, other internal
+ // configurations will not be returned
+ Set ignoreConfigNames = Set.of(
+ ServerConfigs.UNSTABLE_FEATURE_VERSIONS_ENABLE_CONFIG,
+ ServerConfigs.UNSTABLE_API_VERSIONS_ENABLE_CONFIG,
+ KRaftConfigs.SERVER_MAX_STARTUP_TIME_MS_CONFIG);
+ Config brokerConfig = configResourceMap.get(brokerResource);
+ assertNotContainsInternalConfig(brokerConfig, KafkaConfig.configDef().configKeys(), ignoreConfigNames);
+
+ // test for case ConfigResource.Type == TOPIC
+ Config topicConfig = configResourceMap.get(topicResource);
+ assertNotContainsAnyInternalConfig(topicConfig, LogConfig.configKeys());
+
+ // test for case ConfigResource.Type == GROUP
+ Config groupConfig = configResourceMap.get(groupResource);
+ assertNotContainsAnyInternalConfig(groupConfig, GroupConfig.configDef().configKeys());
+
+ // test for case ConfigResource.Type == CLIENT_METRICS
+ Config clientMetricsConfig = configResourceMap.get(clientMetricsResource);
+ assertNotContainsAnyInternalConfig(clientMetricsConfig, ClientMetricsConfigs.configDef().configKeys());
+
+ // test for controller node, and ConfigResource.Type == BROKER
+ ConfigResource controllerResource = new ConfigResource(ConfigResource.Type.BROKER, "3000");
+ Map controllerConfigMap = controllerAdmin.describeConfigs(List.of(controllerResource)).all().get();
+ Config controllerConfig = controllerConfigMap.get(controllerResource);
+ assertNotContainsInternalConfig(controllerConfig, KafkaConfig.configDef().configKeys(), ignoreConfigNames);
+ }
+ }
+
+ @ClusterTest(types = {Type.KRAFT})
+ public void testInternalConfigsDoNotReturnForCreateTopics(ClusterInstance cluster) throws Exception {
+ try (Admin admin = cluster.admin()) {
+ // test for createTopics API
+ Config config = admin.createTopics(List.of(new NewTopic(TOPIC, 1, (short) 1))).config(TOPIC).get();
+ assertNotContainsAnyInternalConfig(config, LogConfig.configKeys());
+ }
+ }
+
+ private void assertNotContainsAnyInternalConfig(Config config, Map configKeyMap) {
+ assertNotContainsInternalConfig(config, configKeyMap, Set.of());
+ }
+
+ private void assertNotContainsInternalConfig(Config config, Map configKeyMap,
+ Set ignoreConfigNames) {
+ assertFalse(config.entries().isEmpty());
+ for (ConfigEntry topicConfigEntry : config.entries()) {
+ String configName = topicConfigEntry.name();
+ ConfigDef.ConfigKey configKey = configKeyMap.get(configName);
+
+ assertNotNull(configKey, "The ConfigKey of the config named '" + configName + "' should not be null");
+ if (!ignoreConfigNames.contains(configName)) {
+ assertFalse(configKey.internalConfig, "The config named '" + configName + "' is an internal config and should not be returned");
+ }
+ }
+ }
}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerAssignmentPoller.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerAssignmentPoller.java
new file mode 100644
index 0000000000000..a970f7f58d29e
--- /dev/null
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerAssignmentPoller.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.WakeupException;
+import org.apache.kafka.server.util.ShutdownableThread;
+
+import java.time.Duration;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+public class ConsumerAssignmentPoller extends ShutdownableThread {
+ private final Consumer consumer;
+ private final Set partitionsToAssign;
+ private volatile Optional thrownException = Optional.empty();
+ private volatile int receivedMessages = 0;
+ private final Set partitionAssignment = Collections.synchronizedSet(new HashSet<>());
+ private volatile boolean subscriptionChanged = false;
+ private List topicsSubscription;
+ private final ConsumerRebalanceListener rebalanceListener;
+
+ public ConsumerAssignmentPoller(Consumer consumer, List topicsToSubscribe) {
+ this(consumer, topicsToSubscribe, Set.of(), null);
+ }
+
+ public ConsumerAssignmentPoller(Consumer consumer, Set partitionsToAssign) {
+ this(consumer, List.of(), partitionsToAssign, null);
+ }
+
+ public ConsumerAssignmentPoller(Consumer consumer,
+ List topicsToSubscribe,
+ Set partitionsToAssign,
+ ConsumerRebalanceListener userRebalanceListener) {
+ super("daemon-consumer-assignment", false);
+ this.consumer = consumer;
+ this.partitionsToAssign = partitionsToAssign;
+ this.topicsSubscription = topicsToSubscribe;
+
+ this.rebalanceListener = new ConsumerRebalanceListener() {
+ @Override
+ public void onPartitionsAssigned(Collection partitions) {
+ partitionAssignment.addAll(partitions);
+ if (userRebalanceListener != null)
+ userRebalanceListener.onPartitionsAssigned(partitions);
+ }
+
+ @Override
+ public void onPartitionsRevoked(Collection partitions) {
+ partitionAssignment.removeAll(partitions);
+ if (userRebalanceListener != null)
+ userRebalanceListener.onPartitionsRevoked(partitions);
+ }
+ };
+
+ if (partitionsToAssign.isEmpty()) {
+ consumer.subscribe(topicsToSubscribe, rebalanceListener);
+ } else {
+ consumer.assign(List.copyOf(partitionsToAssign));
+ }
+ }
+
+ public Set consumerAssignment() {
+ return Set.copyOf(partitionAssignment);
+ }
+
+ public void subscribe(List newTopicsToSubscribe) {
+ if (subscriptionChanged) {
+ throw new IllegalStateException("Do not call subscribe until the previous subscribe request is processed.");
+ }
+ if (!partitionsToAssign.isEmpty()) {
+ throw new IllegalStateException("Cannot call subscribe when configured to use manual partition assignment");
+ }
+ this.topicsSubscription = newTopicsToSubscribe;
+ subscriptionChanged = true;
+ }
+
+ public boolean isSubscribeRequestProcessed() {
+ return !subscriptionChanged;
+ }
+
+ @Override
+ public boolean initiateShutdown() {
+ boolean res = super.initiateShutdown();
+ consumer.wakeup();
+ return res;
+ }
+
+ @Override
+ public void doWork() {
+ if (subscriptionChanged) {
+ consumer.subscribe(topicsSubscription, rebalanceListener);
+ subscriptionChanged = false;
+ }
+ try {
+ ConsumerRecords records = consumer.poll(Duration.ofMillis(50));
+ receivedMessages += records.count();
+ } catch (WakeupException e) {
+ // ignore for shutdown
+ } catch (Throwable e) {
+ thrownException = Optional.of(e);
+ throw e;
+ }
+ }
+
+ public Optional getThrownException() {
+ return thrownException;
+ }
+
+ public int receivedMessages() {
+ return receivedMessages;
+ }
+}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerBounceTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerBounceTest.java
new file mode 100644
index 0000000000000..2583d09997741
--- /dev/null
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerBounceTest.java
@@ -0,0 +1,814 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer;
+
+import kafka.server.KafkaBroker;
+
+import org.apache.kafka.clients.ClientsTestUtils;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.GroupMaxSizeReachedException;
+import org.apache.kafka.common.message.FindCoordinatorRequestData;
+import org.apache.kafka.common.requests.FindCoordinatorRequest;
+import org.apache.kafka.common.requests.FindCoordinatorResponse;
+import org.apache.kafka.common.test.ClusterInstance;
+import org.apache.kafka.common.test.api.ClusterConfigProperty;
+import org.apache.kafka.common.test.api.ClusterTest;
+import org.apache.kafka.common.test.api.ClusterTestDefaults;
+import org.apache.kafka.common.test.api.Type;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.group.GroupCoordinatorConfig;
+import org.apache.kafka.server.IntegrationTestUtils;
+import org.apache.kafka.server.config.KRaftConfigs;
+import org.apache.kafka.server.config.ReplicationConfigs;
+import org.apache.kafka.server.config.ServerConfigs;
+import org.apache.kafka.server.config.ServerLogConfigs;
+import org.apache.kafka.server.util.ShutdownableThread;
+import org.apache.kafka.test.TestUtils;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.IntStream;
+
+import static org.apache.kafka.test.TestUtils.SEEDED_RANDOM;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Integration tests for the consumer that cover basic usage as well as server failures
+ */
+@ClusterTestDefaults(
+ types = {Type.KRAFT},
+ brokers = ConsumerBounceTest.BROKER_COUNT,
+ serverProperties = {
+ @ClusterConfigProperty(key = GroupCoordinatorConfig.OFFSETS_TOPIC_REPLICATION_FACTOR_CONFIG, value = "3"), // don't want to lose offset
+ @ClusterConfigProperty(key = GroupCoordinatorConfig.OFFSETS_TOPIC_PARTITIONS_CONFIG, value = "1"),
+ @ClusterConfigProperty(key = GroupCoordinatorConfig.GROUP_MIN_SESSION_TIMEOUT_MS_CONFIG, value = "10"), // set small enough session timeout
+ @ClusterConfigProperty(key = GroupCoordinatorConfig.GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, value = "0"),
+
+ // Tests will run for CONSUMER and CLASSIC group protocol, so set the group max size property
+ // required for each.
+ @ClusterConfigProperty(key = GroupCoordinatorConfig.CONSUMER_GROUP_MAX_SIZE_CONFIG, value = ConsumerBounceTest.MAX_GROUP_SIZE),
+ @ClusterConfigProperty(key = GroupCoordinatorConfig.GROUP_MAX_SIZE_CONFIG, value = ConsumerBounceTest.MAX_GROUP_SIZE),
+
+ @ClusterConfigProperty(key = ServerLogConfigs.AUTO_CREATE_TOPICS_ENABLE_CONFIG, value = "false"),
+ @ClusterConfigProperty(key = ServerLogConfigs.LOG_INITIAL_TASK_DELAY_MS_CONFIG, value = "100"),
+ @ClusterConfigProperty(key = ServerConfigs.CONTROLLED_SHUTDOWN_ENABLE_CONFIG, value = "false"),
+ @ClusterConfigProperty(key = TopicConfig.UNCLEAN_LEADER_ELECTION_ENABLE_CONFIG, value = "true"),
+ @ClusterConfigProperty(key = TopicConfig.FILE_DELETE_DELAY_MS_CONFIG, value = "1000"),
+ @ClusterConfigProperty(key = ReplicationConfigs.UNCLEAN_LEADER_ELECTION_INTERVAL_MS_CONFIG, value = "50"),
+
+ @ClusterConfigProperty(key = KRaftConfigs.BROKER_HEARTBEAT_INTERVAL_MS_CONFIG, value = "50"),
+ @ClusterConfigProperty(key = KRaftConfigs.BROKER_SESSION_TIMEOUT_MS_CONFIG, value = "300")
+ }
+)
+public class ConsumerBounceTest {
+
+ private final Logger logger = new LogContext("ConsumerBounceTest").logger(this.getClass());
+
+ public static final int BROKER_COUNT = 3;
+ public static final String MAX_GROUP_SIZE = "5";
+
+ private final Optional gracefulCloseTimeMs = Optional.of(1000L);
+ private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(2);
+ private final String topic = "topic";
+ private final int partition = 0;
+ private final int numPartitions = 3;
+ private final short numReplica = 3;
+ private final TopicPartition topicPartition = new TopicPartition(topic, partition);
+
+ private final ClusterInstance clusterInstance;
+
+ private final List> consumers = new ArrayList<>();
+ private final List consumerPollers = new ArrayList<>();
+
+ ConsumerBounceTest(ClusterInstance clusterInstance) {
+ this.clusterInstance = clusterInstance;
+ }
+
+ @BeforeEach
+ void setUp() throws InterruptedException {
+ clusterInstance.createTopic(topic, numPartitions, numReplica);
+ }
+
+ @AfterEach
+ void tearDown() throws InterruptedException {
+ consumerPollers.forEach(poller -> {
+ try {
+ poller.shutdown();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ executor.shutdownNow();
+ // Wait for any active tasks to terminate to ensure consumer is not closed while being used from another thread
+ assertTrue(executor.awaitTermination(5000, TimeUnit.MILLISECONDS), "Executor did not terminate");
+ consumers.forEach(Consumer::close);
+ }
+
+ @ClusterTest
+ public void testClassicConsumerConsumptionWithBrokerFailures() throws Exception {
+ consumeWithBrokerFailures(10, GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerConsumptionWithBrokerFailures() throws Exception {
+ consumeWithBrokerFailures(10, GroupProtocol.CONSUMER);
+ }
+
+ /**
+ * 1. Produce a bunch of messages
+ * 2. Then consume the messages while killing and restarting brokers at random
+ */
+ private void consumeWithBrokerFailures(int numIters, GroupProtocol groupProtocol) throws InterruptedException {
+ int numRecords = 1000;
+ ClientsTestUtils.sendRecords(clusterInstance, topicPartition, numRecords);
+
+ AtomicInteger consumed = new AtomicInteger(0);
+ try (Consumer consumer = clusterInstance.consumer(Map.of(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name))) {
+
+ consumer.subscribe(List.of(topic));
+
+ BounceBrokerScheduler scheduler = new BounceBrokerScheduler(numIters, clusterInstance);
+ try {
+ scheduler.start();
+
+ while (scheduler.isRunning()) {
+ ConsumerRecords records = consumer.poll(Duration.ofMillis(100));
+
+ records.forEach(record -> {
+ assertEquals(consumed.get(), record.offset());
+ consumed.incrementAndGet();
+ });
+
+ if (!records.isEmpty()) {
+ consumer.commitSync();
+
+ long currentPosition = consumer.position(topicPartition);
+ long committedOffset = consumer.committed(Set.of(topicPartition)).get(topicPartition).offset();
+ assertEquals(currentPosition, committedOffset);
+
+ if (currentPosition == numRecords) {
+ consumer.seekToBeginning(List.of());
+ consumed.set(0);
+ }
+ }
+ }
+ } finally {
+ scheduler.shutdown();
+ }
+ }
+ }
+
+ @ClusterTest
+ public void testClassicConsumerSeekAndCommitWithBrokerFailures() throws InterruptedException {
+ seekAndCommitWithBrokerFailures(5, GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerSeekAndCommitWithBrokerFailures() throws InterruptedException {
+ seekAndCommitWithBrokerFailures(5, GroupProtocol.CONSUMER);
+ }
+
+ private void seekAndCommitWithBrokerFailures(int numIters, GroupProtocol groupProtocol) throws InterruptedException {
+ int numRecords = 1000;
+ ClientsTestUtils.sendRecords(clusterInstance, topicPartition, numRecords);
+
+ try (Consumer consumer = clusterInstance.consumer(Map.of(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name))) {
+ consumer.assign(List.of(topicPartition));
+ consumer.seek(topicPartition, 0);
+
+ TestUtils.waitForCondition(() -> clusterInstance.brokers().values().stream().allMatch(broker ->
+ broker.replicaManager().localLog(topicPartition).get().highWatermark() == numRecords
+ ), 30000, "Failed to update high watermark for followers after timeout.");
+
+ BounceBrokerScheduler scheduler = new BounceBrokerScheduler(numIters, clusterInstance);
+ try {
+ scheduler.start();
+
+ while (scheduler.isRunning()) {
+ int coin = SEEDED_RANDOM.nextInt(0, 3);
+
+ if (coin == 0) {
+ logger.info("Seeking to end of log.");
+ consumer.seekToEnd(List.of());
+ assertEquals(numRecords, consumer.position(topicPartition));
+ } else if (coin == 1) {
+ int pos = SEEDED_RANDOM.nextInt(numRecords);
+ logger.info("Seeking to {}", pos);
+ consumer.seek(topicPartition, pos);
+ assertEquals(pos, consumer.position(topicPartition));
+ } else {
+ logger.info("Committing offset.");
+ consumer.commitSync();
+ assertEquals(consumer.position(topicPartition), consumer.committed(Set.of(topicPartition)).get(topicPartition).offset());
+ }
+ }
+ } finally {
+ scheduler.shutdown();
+ }
+ }
+ }
+
+ @ClusterTest
+ public void testClassicSubscribeWhenTopicUnavailable() throws InterruptedException {
+ testSubscribeWhenTopicUnavailable(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncSubscribeWhenTopicUnavailable() throws InterruptedException {
+ testSubscribeWhenTopicUnavailable(GroupProtocol.CONSUMER);
+ }
+
+ private void testSubscribeWhenTopicUnavailable(GroupProtocol groupProtocol) throws InterruptedException {
+ String newTopic = "new-topic";
+ TopicPartition newTopicPartition = new TopicPartition(newTopic, 0);
+ int numRecords = 1000;
+
+ Consumer consumer = clusterInstance.consumer(
+ Map.of(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name, ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 6000,
+ ConsumerConfig.METADATA_MAX_AGE_CONFIG, 100));
+ consumers.add(consumer);
+ consumer.subscribe(List.of(newTopic));
+ consumer.poll(Duration.ZERO);
+ // Schedule topic creation after 2 seconds
+ executor.schedule(() -> assertDoesNotThrow(() -> clusterInstance.createTopic(newTopic, numPartitions, numReplica)),
+ 2, TimeUnit.SECONDS);
+
+ // Start first poller
+ ConsumerAssignmentPoller poller = new ConsumerAssignmentPoller(consumer, List.of(newTopic));
+ consumerPollers.add(poller);
+ poller.start();
+ ClientsTestUtils.sendRecords(clusterInstance, newTopicPartition, numRecords);
+ receiveExactRecords(poller, numRecords, 60000L);
+ poller.shutdown();
+
+ // Simulate broker failure and recovery
+ clusterInstance.brokers().keySet().forEach(clusterInstance::shutdownBroker);
+ Thread.sleep(500);
+ clusterInstance.brokers().keySet().forEach(clusterInstance::startBroker);
+
+ // Start second poller after recovery
+ ConsumerAssignmentPoller poller2 = new ConsumerAssignmentPoller(consumer, List.of(newTopic));
+ consumerPollers.add(poller2);
+ poller2.start();
+
+ ClientsTestUtils.sendRecords(clusterInstance, newTopicPartition, numRecords);
+ receiveExactRecords(poller2, numRecords, 60000L);
+ }
+
+
+ @ClusterTest
+ public void testClassicClose() throws Exception {
+ testClose(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncClose() throws Exception {
+ testClose(GroupProtocol.CONSUMER);
+ }
+
+ private void testClose(GroupProtocol groupProtocol) throws Exception {
+ int numRecords = 10;
+ ClientsTestUtils.sendRecords(clusterInstance, topicPartition, numRecords);
+
+ checkCloseGoodPath(groupProtocol, numRecords, "group1");
+ checkCloseWithCoordinatorFailure(groupProtocol, numRecords, "group2", "group3");
+ checkCloseWithClusterFailure(groupProtocol, numRecords, "group4", "group5");
+ }
+
+ /**
+ * Consumer is closed while cluster is healthy. Consumer should complete pending offset commits
+ * and leave group. New consumer instance should be able to join group and start consuming from
+ * last committed offset.
+ */
+ private void checkCloseGoodPath(GroupProtocol groupProtocol, int numRecords, String groupId) throws InterruptedException {
+ Consumer consumer = createConsumerAndReceive(groupId, false, numRecords, Map.of(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name));
+ assertDoesNotThrow(() -> submitCloseAndValidate(consumer, Long.MAX_VALUE, Optional.empty(), gracefulCloseTimeMs).get());
+ checkClosedState(groupId, numRecords);
+ }
+
+ /**
+ * Consumer closed while coordinator is unavailable. Close of consumers using group
+ * management should complete after commit attempt even though commits fail due to rebalance.
+ * Close of consumers using manual assignment should complete with successful commits since a
+ * broker is available.
+ */
+ private void checkCloseWithCoordinatorFailure(GroupProtocol groupProtocol, int numRecords, String dynamicGroup, String manualGroup) throws Exception {
+ Consumer dynamicConsumer = createConsumerAndReceive(dynamicGroup, false, numRecords, Map.of(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name));
+ Consumer manualConsumer = createConsumerAndReceive(manualGroup, true, numRecords, Map.of(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name));
+
+ findCoordinators(List.of(dynamicGroup, manualGroup)).forEach(clusterInstance::shutdownBroker);
+
+ submitCloseAndValidate(dynamicConsumer, Long.MAX_VALUE, Optional.empty(), gracefulCloseTimeMs).get();
+ submitCloseAndValidate(manualConsumer, Long.MAX_VALUE, Optional.empty(), gracefulCloseTimeMs).get();
+
+ restartDeadBrokers();
+ checkClosedState(dynamicGroup, 0);
+ checkClosedState(manualGroup, numRecords);
+ }
+
+ /**
+ * Consumer is closed while all brokers are unavailable. Cannot rebalance or commit offsets since
+ * there is no coordinator, but close should timeout and return. If close is invoked with a very
+ * large timeout, close should timeout after request timeout.
+ */
+ private void checkCloseWithClusterFailure(GroupProtocol groupProtocol, int numRecords, String group1, String group2) throws Exception {
+ Consumer consumer1 = createConsumerAndReceive(group1, false, numRecords, Map.of(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name));
+ Map consumerConfig = new HashMap<>();
+
+ long requestTimeout = 6000;
+ if (groupProtocol.equals(GroupProtocol.CLASSIC)) {
+ consumerConfig.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "5000");
+ consumerConfig.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "1000");
+ }
+ consumerConfig.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, Long.toString(requestTimeout));
+ consumerConfig.put(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name);
+
+ Consumer consumer2 = createConsumerAndReceive(group2, true, numRecords, consumerConfig);
+
+ clusterInstance.brokers().keySet().forEach(clusterInstance::shutdownBroker);
+
+ long closeTimeout = 2000;
+ submitCloseAndValidate(consumer1, closeTimeout, Optional.empty(), Optional.of(closeTimeout)).get();
+ submitCloseAndValidate(consumer2, Long.MAX_VALUE, Optional.empty(), Optional.of(requestTimeout)).get();
+ }
+
+ private Set findCoordinators(List groups) throws Exception {
+ FindCoordinatorRequest request = new FindCoordinatorRequest.Builder(new FindCoordinatorRequestData()
+ .setKeyType(FindCoordinatorRequest.CoordinatorType.GROUP.id())
+ .setCoordinatorKeys(groups)).build();
+ Set nodes = new HashSet<>();
+ TestUtils.waitForCondition(() -> {
+ FindCoordinatorResponse response = null;
+ try {
+ response = IntegrationTestUtils.connectAndReceive(request, clusterInstance.brokerBoundPorts().get(0));
+ } catch (IOException e) {
+ return false;
+ }
+
+ if (response.hasError())
+ return false;
+ for (String group : groups)
+ if (response.coordinatorByKey(group).isEmpty())
+ return false;
+ else
+ nodes.add(response.coordinatorByKey(group).get().nodeId());
+ return true;
+ }, "Failed to find coordinator for group " + groups);
+ return nodes;
+ }
+
+ @ClusterTest
+ public void testClassicConsumerReceivesFatalExceptionWhenGroupPassesMaxSize() throws Exception {
+ testConsumerReceivesFatalExceptionWhenGroupPassesMaxSize(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerReceivesFatalExceptionWhenGroupPassesMaxSize() throws Exception {
+ testConsumerReceivesFatalExceptionWhenGroupPassesMaxSize(GroupProtocol.CONSUMER);
+ }
+
+ private void testConsumerReceivesFatalExceptionWhenGroupPassesMaxSize(GroupProtocol groupProtocol) throws Exception {
+ String group = "fatal-exception-test";
+ String topic = "fatal-exception-test";
+
+ Map consumerConfig = new HashMap<>();
+ int numPartition = Integer.parseInt(MAX_GROUP_SIZE);
+
+ consumerConfig.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, "60000");
+ if (groupProtocol.equals(GroupProtocol.CLASSIC)) {
+ consumerConfig.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "1000");
+ }
+ consumerConfig.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
+
+ clusterInstance.createTopic(topic, Integer.parseInt(MAX_GROUP_SIZE), (short) BROKER_COUNT);
+ Set partitions = new HashSet<>();
+ for (int i = 0; i < Integer.parseInt(MAX_GROUP_SIZE); ++i)
+ partitions.add(new TopicPartition(topic, i));
+
+ addConsumersToGroupAndWaitForGroupAssignment(
+ Integer.parseInt(MAX_GROUP_SIZE),
+ List.of(topic),
+ partitions,
+ group,
+ consumerConfig
+ );
+
+ addConsumersToGroup(
+ 1,
+ List.of(topic),
+ group,
+ consumerConfig
+ );
+
+ ConsumerAssignmentPoller rejectedConsumer = consumerPollers.get(consumerPollers.size() - 1);
+ consumerPollers.remove(consumerPollers.size() - 1);
+
+ TestUtils.waitForCondition(
+ () -> rejectedConsumer.getThrownException().isPresent(),
+ "Extra consumer did not throw an exception"
+ );
+
+ assertInstanceOf(GroupMaxSizeReachedException.class, rejectedConsumer.getThrownException().get());
+
+ // assert group continues to live and the records to be distributed across all partitions.
+ var data = "data".getBytes(StandardCharsets.UTF_8);
+ try (Producer producer = clusterInstance.producer()) {
+ IntStream.range(0, numPartition * 100).forEach(index ->
+ producer.send(new ProducerRecord<>(topic, index % numPartition, data, data)));
+ }
+
+ TestUtils.waitForCondition(
+ () -> consumerPollers.stream().allMatch(p -> p.receivedMessages() >= 100),
+ 10000L, "The consumers in the group could not fetch the expected records"
+ );
+ }
+
+ /**
+ * Create 'numOfConsumersToAdd' consumers, add them to the consumer group, and create corresponding
+ * pollers. Wait for partition re-assignment and validate.
+ *
+ * Assignment validation requires that total number of partitions is greater than or equal to
+ * the resulting number of consumers in the group.
+ *
+ * @param numOfConsumersToAdd number of consumers to create and add to the consumer group
+ * @param topicsToSubscribe topics to subscribe
+ * @param subscriptions set of all topic partitions
+ * @param group consumer group ID
+ */
+ private void addConsumersToGroupAndWaitForGroupAssignment(
+ int numOfConsumersToAdd,
+ List topicsToSubscribe,
+ Set subscriptions,
+ String group,
+ Map consumerConfig
+ ) throws InterruptedException {
+ // Validation: number of consumers should not exceed number of partitions
+ assertTrue(consumers.size() + numOfConsumersToAdd <= subscriptions.size(),
+ "Total consumers exceed number of partitions");
+
+ // Add consumers and pollers
+ addConsumersToGroup(numOfConsumersToAdd, topicsToSubscribe, group, consumerConfig);
+
+ // Validate that all pollers have assigned partitions
+ validateGroupAssignment(consumerPollers, subscriptions);
+ }
+
+ /**
+ * Check whether partition assignment is valid.
+ * Assumes partition assignment is valid iff:
+ * 1. Every consumer got assigned at least one partition
+ * 2. Each partition is assigned to only one consumer
+ * 3. Every partition is assigned to one of the consumers
+ * 4. The assignment is the same as expected assignment (if provided)
+ *
+ * @param assignments List of assignments, one set per consumer
+ * @param partitions All expected partitions
+ * @param expectedAssignment Optional expected assignment
+ * @return true if assignment is valid
+ */
+ private boolean isPartitionAssignmentValid(
+ List> assignments,
+ Set partitions,
+ List> expectedAssignment
+ ) {
+ // 1. Check that every consumer has non-empty assignment
+ boolean allNonEmpty = assignments.stream().noneMatch(Set::isEmpty);
+ if (!allNonEmpty) return false;
+
+ // 2. Check that total assigned partitions equals number of unique partitions
+ Set allAssignedPartitions = new HashSet<>();
+ for (Set assignment : assignments) {
+ allAssignedPartitions.addAll(assignment);
+ }
+
+ if (allAssignedPartitions.size() != partitions.size()) {
+ // Either some partitions were assigned multiple times or some were not assigned
+ return false;
+ }
+
+ // 3. Check that assigned partitions exactly match the expected set
+ if (!allAssignedPartitions.equals(partitions)) {
+ return false;
+ }
+
+ // 4. If expected assignment is given, check for exact match
+ if (expectedAssignment != null && !expectedAssignment.isEmpty()) {
+ if (assignments.size() != expectedAssignment.size()) return false;
+ for (int i = 0; i < assignments.size(); i++) {
+ if (!assignments.get(i).equals(expectedAssignment.get(i))) return false;
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Wait for consumers to get partition assignment and validate it.
+ *
+ * @param consumerPollers Consumer pollers corresponding to the consumer group being tested
+ * @param subscriptions Set of all topic partitions
+ * @param msg Optional message to print if validation fails
+ * @param waitTimeMs Wait timeout in milliseconds
+ * @param expectedAssignments Expected assignments (optional)
+ */
+ private void validateGroupAssignment(
+ List consumerPollers,
+ Set subscriptions,
+ Optional msg,
+ long waitTimeMs,
+ List> expectedAssignments
+ ) throws InterruptedException {
+ List> assignments = new ArrayList<>();
+
+ TestUtils.waitForCondition(() -> {
+ assignments.clear();
+ consumerPollers.forEach(poller -> assignments.add(poller.consumerAssignment()));
+ return isPartitionAssignmentValid(assignments, subscriptions, expectedAssignments);
+ }, waitTimeMs, msg.orElse("Did not get valid assignment for partitions " + subscriptions + ". Instead got: " + assignments));
+ }
+
+ // Overload for convenience (optional msg and expectedAssignments)
+ private void validateGroupAssignment(
+ List consumerPollers,
+ Set subscriptions
+ ) throws InterruptedException {
+ validateGroupAssignment(consumerPollers, subscriptions, Optional.empty(), 10000L, new ArrayList<>());
+ }
+
+ /**
+ * Create 'numOfConsumersToAdd' consumers, add them to the consumer group, and create corresponding pollers.
+ *
+ * @param numOfConsumersToAdd number of consumers to create and add to the consumer group
+ * @param topicsToSubscribe topics to which new consumers will subscribe
+ * @param group consumer group ID
+ */
+ private void addConsumersToGroup(
+ int numOfConsumersToAdd,
+ List topicsToSubscribe,
+ String group,
+ Map consumerConfigs) {
+
+ Map configs = new HashMap<>(consumerConfigs);
+ configs.put(ConsumerConfig.GROUP_ID_CONFIG, group);
+
+ for (int i = 0; i < numOfConsumersToAdd; i++) {
+ Consumer consumer = clusterInstance.consumer(configs);
+ consumers.add(consumer);
+
+ ConsumerAssignmentPoller poller = new ConsumerAssignmentPoller(consumer, topicsToSubscribe);
+ poller.start();
+ consumerPollers.add(poller);
+ }
+ }
+
+ @ClusterTest
+ public void testClassicCloseDuringRebalance() throws Exception {
+ testCloseDuringRebalance(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncCloseDuringRebalance() throws Exception {
+ testCloseDuringRebalance(GroupProtocol.CONSUMER);
+ }
+
+ public void testCloseDuringRebalance(GroupProtocol groupProtocol) throws Exception {
+ Map consumerConfig = new HashMap<>();
+ consumerConfig.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, "60000");
+ if (groupProtocol.equals(GroupProtocol.CLASSIC)) {
+ consumerConfig.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "1000");
+ }
+ consumerConfig.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
+
+ checkCloseDuringRebalance(consumerConfig);
+ }
+
+ private void checkCloseDuringRebalance(Map consumerConfig) throws Exception {
+ Map configs = new HashMap<>(consumerConfig);
+ String groupId = "group";
+ configs.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
+
+ Consumer consumer1 = clusterInstance.consumer(configs);
+ Future f1 = subscribeAndPoll(consumer1, Optional.empty());
+ waitForRebalance(2000, f1, null);
+
+ Consumer consumer2 = clusterInstance.consumer(configs);
+ Future f2 = subscribeAndPoll(consumer2, Optional.empty());
+ waitForRebalance(2000, f2, consumer1);
+
+ Future rebalanceFuture = createConsumerToRebalance(groupId);
+
+ Future closeFuture1 = submitCloseAndValidate(consumer1, Long.MAX_VALUE, Optional.empty(), gracefulCloseTimeMs);
+
+ waitForRebalance(2000, rebalanceFuture, consumer2);
+
+ createConsumerToRebalance(groupId); // one more time
+ clusterInstance.brokers().values().forEach(KafkaBroker::shutdown);
+
+ Future closeFuture2 = submitCloseAndValidate(consumer2, Long.MAX_VALUE, Optional.empty(), Optional.of(0L));
+
+ closeFuture1.get(2000, TimeUnit.MILLISECONDS);
+ closeFuture2.get(2000, TimeUnit.MILLISECONDS);
+ }
+
+ Future subscribeAndPoll(Consumer consumer, Optional revokeSemaphore) {
+ return executor.submit(() -> {
+ consumer.subscribe(List.of(topic));
+ revokeSemaphore.ifPresent(Semaphore::release);
+ consumer.poll(Duration.ofMillis(500));
+ return null;
+ });
+ }
+
+ void waitForRebalance(long timeoutMs, Future future, Consumer otherConsumers) {
+ long startMs = System.currentTimeMillis();
+ while (System.currentTimeMillis() < startMs + timeoutMs && !future.isDone()) {
+ if (otherConsumers != null) {
+ otherConsumers.poll(Duration.ofMillis(100));
+ }
+ }
+ assertTrue(future.isDone(), "Rebalance did not complete in time");
+ }
+
+ Future createConsumerToRebalance(String groupId) throws Exception {
+ Consumer consumer = clusterInstance.consumer(Map.of(ConsumerConfig.GROUP_ID_CONFIG, groupId));
+ consumers.add(consumer);
+ Semaphore rebalanceSemaphore = new Semaphore(0);
+ Future future = subscribeAndPoll(consumer, Optional.of(rebalanceSemaphore));
+ assertTrue(rebalanceSemaphore.tryAcquire(2000, TimeUnit.MILLISECONDS), "Rebalance not triggered");
+ assertFalse(future.isDone(), "Rebalance completed too early");
+ return future;
+ }
+
+
+
+ private Consumer createConsumerAndReceive(String groupId, boolean manualAssign, int numRecords,
+ Map consumerConfig) throws InterruptedException {
+ Map configs = new HashMap<>(consumerConfig);
+ configs.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
+ Consumer consumer = clusterInstance.consumer(configs);
+ ConsumerAssignmentPoller poller;
+
+ if (manualAssign) {
+ poller = new ConsumerAssignmentPoller(consumer, Set.of(topicPartition));
+ } else {
+ poller = new ConsumerAssignmentPoller(consumer, List.of(topic));
+ }
+ poller.start();
+ consumers.add(consumer);
+ consumerPollers.add(poller);
+ receiveExactRecords(poller, numRecords, 60000L);
+ poller.shutdown();
+
+ return consumer;
+ }
+
+ private void restartDeadBrokers() {
+ clusterInstance.brokers().forEach((id, broker) -> {
+ if (broker.isShutdown()) {
+ broker.startup();
+ }
+ });
+ }
+
+ private void checkClosedState(String groupId, int committedRecords) throws InterruptedException {
+ // Check that close was graceful with offsets committed and leave group sent.
+ // New instance of consumer should be assigned partitions immediately and should see committed offsets. Semaphore assignSemaphore = new Semaphore(0);
+
+ Semaphore assignSemaphore = new Semaphore(0);
+ try (Consumer consumer = clusterInstance.consumer(Map.of(ConsumerConfig.GROUP_ID_CONFIG, groupId))) {
+ consumer.subscribe(List.of(topic), new ConsumerRebalanceListener() {
+ @Override
+ public void onPartitionsAssigned(Collection partitions) {
+ assignSemaphore.release();
+ }
+
+ @Override
+ public void onPartitionsRevoked(Collection partitions) {
+ // Do nothing
+ }
+ });
+
+ TestUtils.waitForCondition(() -> {
+ consumer.poll(Duration.ofMillis(100));
+ return assignSemaphore.tryAcquire();
+ }, "Assignment did not complete on time");
+
+ if (committedRecords > 0) {
+ Map committed = consumer.committed(Set.of(topicPartition));
+ long offset = committed.get(topicPartition).offset();
+ assertEquals(committedRecords, offset, "Committed offset does not match expected value.");
+ }
+ }
+ }
+
+ private Future submitCloseAndValidate(
+ Consumer consumer,
+ long closeTimeoutMs,
+ Optional minCloseTimeMs,
+ Optional maxCloseTimeMs) {
+
+ return executor.submit(() -> {
+ final long closeGraceTimeMs = 2000;
+ long startMs = System.currentTimeMillis();
+ logger.info("Closing consumer with timeout {} ms.", closeTimeoutMs);
+
+ consumer.close(CloseOptions.timeout(Duration.ofMillis(closeTimeoutMs)));
+ long timeTakenMs = System.currentTimeMillis() - startMs;
+
+ maxCloseTimeMs.ifPresent(ms ->
+ assertTrue(timeTakenMs < ms + closeGraceTimeMs, "Close took too long " + timeTakenMs)
+ );
+
+ minCloseTimeMs.ifPresent(ms ->
+ assertTrue(timeTakenMs >= ms, "Close finished too quickly " + timeTakenMs)
+ );
+
+ logger.info("consumer.close() completed in {} ms.", timeTakenMs);
+ }, 0);
+ }
+
+ private void receiveExactRecords(ConsumerAssignmentPoller consumer, int numRecords, long timeoutMs) throws InterruptedException {
+ TestUtils.waitForCondition(() -> consumer.receivedMessages() == numRecords, timeoutMs,
+ String.format("Consumer did not receive expected %d. It received %d", numRecords, consumer.receivedMessages()));
+ }
+
+ // A mock class to represent broker bouncing (simulate broker restart behavior)
+ private static class BounceBrokerScheduler extends ShutdownableThread {
+ private final int numIters;
+ private int iter = 0;
+
+ final ClusterInstance clusterInstance;
+
+ public BounceBrokerScheduler(int numIters, ClusterInstance clusterInstance) {
+ super("daemon-bounce-broker", false);
+ this.numIters = numIters;
+ this.clusterInstance = clusterInstance;
+ }
+
+ private void killRandomBroker() {
+ this.clusterInstance.shutdownBroker(TestUtils.randomSelect(clusterInstance.brokerIds()));
+ }
+
+ private void restartDeadBrokers() {
+ clusterInstance.brokers().forEach((id, broker) -> {
+ if (broker.isShutdown()) {
+ broker.startup();
+ }
+ });
+ }
+
+ @Override
+ public void doWork() {
+ killRandomBroker();
+ assertDoesNotThrow(() -> Thread.sleep(500));
+ restartDeadBrokers();
+
+ iter++;
+ if (iter == numIters) {
+ initiateShutdown();
+ } else {
+ assertDoesNotThrow(() -> Thread.sleep(500));
+ }
+ }
+ }
+}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerIntegrationTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerIntegrationTest.java
index a56de229318dd..220866c240f4a 100644
--- a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerIntegrationTest.java
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerIntegrationTest.java
@@ -16,7 +16,12 @@
*/
package org.apache.kafka.clients.consumer;
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.NewPartitionReassignment;
+import org.apache.kafka.clients.admin.NewPartitions;
+import org.apache.kafka.clients.admin.NewTopic;
import org.apache.kafka.clients.consumer.internals.AbstractHeartbeatRequestManager;
+import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.KafkaException;
@@ -26,17 +31,21 @@
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.apache.kafka.common.test.ClusterInstance;
-import org.apache.kafka.common.test.TestUtils;
import org.apache.kafka.common.test.api.ClusterConfigProperty;
import org.apache.kafka.common.test.api.ClusterTest;
import org.apache.kafka.common.test.api.ClusterTests;
import org.apache.kafka.common.test.api.Type;
+import org.apache.kafka.coordinator.group.GroupCoordinatorConfig;
+import org.apache.kafka.test.TestUtils;
import java.time.Duration;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -216,6 +225,116 @@ public void testLeaderEpoch(ClusterInstance clusterInstance) throws Exception {
}
}
+ @ClusterTest(
+ types = {Type.KRAFT},
+ brokers = 3,
+ serverProperties = {
+ @ClusterConfigProperty(id = 0, key = "broker.rack", value = "rack0"),
+ @ClusterConfigProperty(id = 1, key = "broker.rack", value = "rack1"),
+ @ClusterConfigProperty(id = 2, key = "broker.rack", value = "rack2"),
+ @ClusterConfigProperty(key = GroupCoordinatorConfig.CONSUMER_GROUP_ASSIGNORS_CONFIG, value = "org.apache.kafka.clients.consumer.RackAwareAssignor")
+ }
+ )
+ public void testRackAwareAssignment(ClusterInstance clusterInstance) throws ExecutionException, InterruptedException {
+ String topic = "test-topic";
+ try (Admin admin = clusterInstance.admin();
+ Producer producer = clusterInstance.producer();
+ Consumer consumer0 = clusterInstance.consumer(Map.of(
+ ConsumerConfig.GROUP_ID_CONFIG, "group0",
+ ConsumerConfig.CLIENT_RACK_CONFIG, "rack0",
+ ConsumerConfig.GROUP_PROTOCOL_CONFIG, GroupProtocol.CONSUMER.name()
+ ));
+ Consumer consumer1 = clusterInstance.consumer(Map.of(
+ ConsumerConfig.GROUP_ID_CONFIG, "group0",
+ ConsumerConfig.CLIENT_RACK_CONFIG, "rack1",
+ ConsumerConfig.GROUP_PROTOCOL_CONFIG, GroupProtocol.CONSUMER.name()
+ ));
+ Consumer consumer2 = clusterInstance.consumer(Map.of(
+ ConsumerConfig.GROUP_ID_CONFIG, "group0",
+ ConsumerConfig.CLIENT_RACK_CONFIG, "rack2",
+ ConsumerConfig.GROUP_PROTOCOL_CONFIG, GroupProtocol.CONSUMER.name()
+ ))
+ ) {
+ // Create a new topic with 1 partition on broker 0.
+ admin.createTopics(List.of(new NewTopic(topic, Map.of(0, List.of(0)))));
+ clusterInstance.waitTopicCreation(topic, 1);
+
+ producer.send(new ProducerRecord<>(topic, "key".getBytes(), "value".getBytes()));
+ producer.flush();
+
+ consumer0.subscribe(List.of(topic));
+ consumer1.subscribe(List.of(topic));
+ consumer2.subscribe(List.of(topic));
+
+ TestUtils.waitForCondition(() -> {
+ consumer0.poll(Duration.ofMillis(1000));
+ consumer1.poll(Duration.ofMillis(1000));
+ consumer2.poll(Duration.ofMillis(1000));
+ return consumer0.assignment().equals(Set.of(new TopicPartition(topic, 0))) &&
+ consumer1.assignment().isEmpty() &&
+ consumer2.assignment().isEmpty();
+ }, "Consumer 0 should be assigned to topic partition 0");
+
+ // Add a new partition 1 and 2 to broker 1.
+ admin.createPartitions(
+ Map.of(
+ topic,
+ NewPartitions.increaseTo(3, List.of(List.of(1), List.of(1)))
+ )
+ );
+ clusterInstance.waitTopicCreation(topic, 3);
+ TestUtils.waitForCondition(() -> {
+ consumer0.poll(Duration.ofMillis(1000));
+ consumer1.poll(Duration.ofMillis(1000));
+ consumer2.poll(Duration.ofMillis(1000));
+ return consumer0.assignment().equals(Set.of(new TopicPartition(topic, 0))) &&
+ consumer1.assignment().equals(Set.of(new TopicPartition(topic, 1), new TopicPartition(topic, 2))) &&
+ consumer2.assignment().isEmpty();
+ }, "Consumer 1 should be assigned to topic partition 1 and 2");
+
+ // Add a new partition 3, 4, and 5 to broker 2.
+ admin.createPartitions(
+ Map.of(
+ topic,
+ NewPartitions.increaseTo(6, List.of(List.of(2), List.of(2), List.of(2)))
+ )
+ );
+ clusterInstance.waitTopicCreation(topic, 6);
+ TestUtils.waitForCondition(() -> {
+ consumer0.poll(Duration.ofMillis(1000));
+ consumer1.poll(Duration.ofMillis(1000));
+ consumer2.poll(Duration.ofMillis(1000));
+ return consumer0.assignment().equals(Set.of(new TopicPartition(topic, 0))) &&
+ consumer1.assignment().equals(Set.of(new TopicPartition(topic, 1), new TopicPartition(topic, 2))) &&
+ consumer2.assignment().equals(Set.of(new TopicPartition(topic, 3), new TopicPartition(topic, 4), new TopicPartition(topic, 5)));
+ }, "Consumer 2 should be assigned to topic partition 3, 4, and 5");
+
+ // Change partitions to different brokers.
+ // partition 0 -> broker 2
+ // partition 1 -> broker 2
+ // partition 2 -> broker 2
+ // partition 3 -> broker 1
+ // partition 4 -> broker 1
+ // partition 5 -> broker 0
+ admin.alterPartitionReassignments(Map.of(
+ new TopicPartition(topic, 0), Optional.of(new NewPartitionReassignment(List.of(2))),
+ new TopicPartition(topic, 1), Optional.of(new NewPartitionReassignment(List.of(2))),
+ new TopicPartition(topic, 2), Optional.of(new NewPartitionReassignment(List.of(2))),
+ new TopicPartition(topic, 3), Optional.of(new NewPartitionReassignment(List.of(1))),
+ new TopicPartition(topic, 4), Optional.of(new NewPartitionReassignment(List.of(1))),
+ new TopicPartition(topic, 5), Optional.of(new NewPartitionReassignment(List.of(0)))
+ )).all().get();
+ TestUtils.waitForCondition(() -> {
+ consumer0.poll(Duration.ofMillis(1000));
+ consumer1.poll(Duration.ofMillis(1000));
+ consumer2.poll(Duration.ofMillis(1000));
+ return consumer0.assignment().equals(Set.of(new TopicPartition(topic, 5))) &&
+ consumer1.assignment().equals(Set.of(new TopicPartition(topic, 3), new TopicPartition(topic, 4))) &&
+ consumer2.assignment().equals(Set.of(new TopicPartition(topic, 0), new TopicPartition(topic, 1), new TopicPartition(topic, 2)));
+ }, "Consumer with topic partition mapping should be 0 -> 5 | 1 -> 3, 4 | 2 -> 0, 1, 2");
+ }
+ }
+
private void sendMsg(ClusterInstance clusterInstance, String topic, int sendMsgNum) {
try (var producer = clusterInstance.producer(Map.of(
ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class,
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerWithLegacyMessageFormatIntegrationTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerWithLegacyMessageFormatIntegrationTest.java
new file mode 100644
index 0000000000000..755d9c89b7c52
--- /dev/null
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerWithLegacyMessageFormatIntegrationTest.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer;
+
+import org.apache.kafka.clients.ClientsTestUtils;
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.compress.Compression;
+import org.apache.kafka.common.record.AbstractRecords;
+import org.apache.kafka.common.record.CompressionType;
+import org.apache.kafka.common.record.MemoryRecords;
+import org.apache.kafka.common.record.MemoryRecordsBuilder;
+import org.apache.kafka.common.record.RecordBatch;
+import org.apache.kafka.common.record.RecordVersion;
+import org.apache.kafka.common.record.SimpleRecord;
+import org.apache.kafka.common.record.TimestampType;
+import org.apache.kafka.common.test.ClusterInstance;
+import org.apache.kafka.common.test.api.ClusterTest;
+import org.apache.kafka.common.test.api.ClusterTestDefaults;
+import org.apache.kafka.storage.internals.log.UnifiedLog;
+
+import org.junit.jupiter.api.BeforeEach;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_PROTOCOL_CONFIG;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+
+@ClusterTestDefaults(
+ brokers = 3
+)
+public class ConsumerWithLegacyMessageFormatIntegrationTest {
+
+ private final ClusterInstance cluster;
+
+ private final String topic1 = "part-test-topic-1";
+ private final String topic2 = "part-test-topic-2";
+ private final String topic3 = "part-test-topic-3";
+
+ private final TopicPartition t1p0 = new TopicPartition(topic1, 0);
+ private final TopicPartition t1p1 = new TopicPartition(topic1, 1);
+ private final TopicPartition t2p0 = new TopicPartition(topic2, 0);
+ private final TopicPartition t2p1 = new TopicPartition(topic2, 1);
+ private final TopicPartition t3p0 = new TopicPartition(topic3, 0);
+ private final TopicPartition t3p1 = new TopicPartition(topic3, 1);
+
+ public ConsumerWithLegacyMessageFormatIntegrationTest(ClusterInstance cluster) {
+ this.cluster = cluster;
+ }
+
+ private void appendLegacyRecords(int numRecords, TopicPartition tp, int brokerId, byte magicValue) {
+ List records = new ArrayList<>();
+ for (int i = 0; i < numRecords; i++) {
+ records.add(new SimpleRecord(i, ("key " + i).getBytes(), ("value " + i).getBytes()));
+ }
+
+ ByteBuffer buffer = ByteBuffer.allocate(AbstractRecords.estimateSizeInBytes(magicValue,
+ CompressionType.NONE, records));
+ MemoryRecordsBuilder builder = MemoryRecords.builder(
+ buffer,
+ magicValue,
+ Compression.of(CompressionType.NONE).build(),
+ TimestampType.CREATE_TIME,
+ 0L,
+ RecordBatch.NO_TIMESTAMP,
+ RecordBatch.NO_PRODUCER_ID,
+ RecordBatch.NO_PRODUCER_EPOCH,
+ 0,
+ false,
+ RecordBatch.NO_PARTITION_LEADER_EPOCH
+ );
+
+ records.forEach(builder::append);
+
+ cluster.brokers().values().stream()
+ .filter(b -> b.config().brokerId() == brokerId)
+ .forEach(b -> {
+ UnifiedLog unifiedLog = b.replicaManager().logManager().getLog(tp, false).get();
+ unifiedLog.appendAsLeaderWithRecordVersion(builder.build(), 0, RecordVersion.lookup(magicValue));
+ // Default isolation.level is read_uncommitted. It makes Partition#fetchOffsetForTimestamp to return UnifiedLog#highWatermark,
+ // so increasing high watermark to make it return the correct offset.
+ assertDoesNotThrow(() -> unifiedLog.maybeIncrementHighWatermark(unifiedLog.logEndOffsetMetadata()));
+ });
+ }
+
+ private void createTopicWithAssignment(String topic, Map> assignment) throws InterruptedException {
+ try (Admin admin = cluster.admin()) {
+ NewTopic newTopic = new NewTopic(topic, assignment);
+ admin.createTopics(List.of(newTopic));
+ cluster.waitTopicCreation(topic, assignment.size());
+ }
+ }
+
+ @BeforeEach
+ public void setupTopics() throws InterruptedException {
+ cluster.createTopic(topic1, 2, (short) 1);
+ createTopicWithAssignment(topic2, Map.of(0, List.of(0), 1, List.of(1)));
+ createTopicWithAssignment(topic3, Map.of(0, List.of(0), 1, List.of(1)));
+
+ // v2 message format for topic1
+ ClientsTestUtils.sendRecords(cluster, t1p0, 100, 0);
+ ClientsTestUtils.sendRecords(cluster, t1p1, 100, 0);
+ // v0 message format for topic2
+ appendLegacyRecords(100, t2p0, 0, RecordBatch.MAGIC_VALUE_V0);
+ appendLegacyRecords(100, t2p1, 1, RecordBatch.MAGIC_VALUE_V0);
+ // v1 message format for topic3
+ appendLegacyRecords(100, t3p0, 0, RecordBatch.MAGIC_VALUE_V1);
+ appendLegacyRecords(100, t3p1, 1, RecordBatch.MAGIC_VALUE_V1);
+ }
+
+ @ClusterTest
+ public void testOffsetsForTimesWithClassicConsumer() {
+ testOffsetsForTimes(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testOffsetsForTimesWithAsyncConsumer() {
+ testOffsetsForTimes(GroupProtocol.CONSUMER);
+ }
+
+ public void testOffsetsForTimes(GroupProtocol groupProtocol) {
+ try (Consumer consumer = cluster.consumer(Map.of(
+ GROUP_PROTOCOL_CONFIG, groupProtocol.name.toLowerCase(Locale.ROOT)))
+ ) {
+ // Test negative target time
+ assertThrows(IllegalArgumentException.class, () ->
+ consumer.offsetsForTimes(Map.of(t1p0, -1L)));
+
+ Map timestampsToSearch = Map.of(
+ t1p0, 0L,
+ t1p1, 20L,
+ t2p0, 40L,
+ t2p1, 60L,
+ t3p0, 80L,
+ t3p1, 100L
+ );
+
+ Map timestampOffsets = consumer.offsetsForTimes(timestampsToSearch);
+
+ OffsetAndTimestamp timestampTopic1P0 = timestampOffsets.get(t1p0);
+ assertEquals(0, timestampTopic1P0.offset());
+ assertEquals(0, timestampTopic1P0.timestamp());
+ assertEquals(Optional.of(0), timestampTopic1P0.leaderEpoch());
+
+ OffsetAndTimestamp timestampTopic1P1 = timestampOffsets.get(t1p1);
+ assertEquals(20, timestampTopic1P1.offset());
+ assertEquals(20, timestampTopic1P1.timestamp());
+ assertEquals(Optional.of(0), timestampTopic1P1.leaderEpoch());
+
+ OffsetAndTimestamp timestampTopic2P0 = timestampOffsets.get(t2p0);
+ assertNull(timestampTopic2P0, "v0 message format shouldn't have timestamp");
+
+ OffsetAndTimestamp timestampTopic2P1 = timestampOffsets.get(t2p1);
+ assertNull(timestampTopic2P1);
+
+ OffsetAndTimestamp timestampTopic3P0 = timestampOffsets.get(t3p0);
+ assertEquals(80, timestampTopic3P0.offset());
+ assertEquals(80, timestampTopic3P0.timestamp());
+ assertEquals(Optional.empty(), timestampTopic3P0.leaderEpoch());
+
+ assertNull(timestampOffsets.get(t3p1), "v1 message format doesn't have leader epoch");
+ }
+
+ }
+
+ @ClusterTest
+ public void testEarliestOrLatestOffsetsWithClassicConsumer() {
+ testEarliestOrLatestOffsets(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testEarliestOrLatestOffsetsWithAsyncConsumer() {
+ testEarliestOrLatestOffsets(GroupProtocol.CONSUMER);
+ }
+
+ public void testEarliestOrLatestOffsets(GroupProtocol groupProtocol) {
+ Set partitions = Set.of(t1p0, t1p1, t2p0, t2p1, t3p0, t3p1);
+
+ try (Consumer consumer = cluster.consumer(Map.of(
+ GROUP_PROTOCOL_CONFIG, groupProtocol.name.toLowerCase(Locale.ROOT)))
+ ) {
+ Map earliests = consumer.beginningOffsets(partitions);
+ assertEquals(0L, earliests.get(t1p0));
+ assertEquals(0L, earliests.get(t1p1));
+ assertEquals(0L, earliests.get(t2p0));
+ assertEquals(0L, earliests.get(t2p1));
+ assertEquals(0L, earliests.get(t3p0));
+ assertEquals(0L, earliests.get(t3p1));
+
+ Map latests = consumer.endOffsets(partitions);
+ assertEquals(100L, latests.get(t1p0));
+ assertEquals(100L, latests.get(t1p1));
+ assertEquals(100L, latests.get(t2p0));
+ assertEquals(100L, latests.get(t2p1));
+ assertEquals(100L, latests.get(t3p0));
+ assertEquals(100L, latests.get(t3p1));
+ }
+ }
+}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerAssignTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerAssignTest.java
new file mode 100644
index 0000000000000..b12b6dacb4d8b
--- /dev/null
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerAssignTest.java
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.clients.consumer;
+
+import org.apache.kafka.clients.ClientsTestUtils;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.test.ClusterInstance;
+import org.apache.kafka.common.test.api.ClusterConfigProperty;
+import org.apache.kafka.common.test.api.ClusterTest;
+import org.apache.kafka.common.test.api.ClusterTestDefaults;
+import org.apache.kafka.common.test.api.Type;
+
+import org.junit.jupiter.api.BeforeEach;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_PROTOCOL_CONFIG;
+import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG;
+import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.GROUP_MIN_SESSION_TIMEOUT_MS_CONFIG;
+import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.OFFSETS_TOPIC_PARTITIONS_CONFIG;
+import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.OFFSETS_TOPIC_REPLICATION_FACTOR_CONFIG;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Integration tests for the consumer that covers logic related to manual assignment.
+ */
+@ClusterTestDefaults(
+ types = {Type.KRAFT},
+ brokers = PlaintextConsumerAssignTest.BROKER_COUNT,
+ serverProperties = {
+ @ClusterConfigProperty(key = OFFSETS_TOPIC_REPLICATION_FACTOR_CONFIG, value = "3"),
+ @ClusterConfigProperty(key = OFFSETS_TOPIC_PARTITIONS_CONFIG, value = "1"),
+ @ClusterConfigProperty(key = GROUP_MIN_SESSION_TIMEOUT_MS_CONFIG, value = "100"),
+ @ClusterConfigProperty(key = GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, value = "10"),
+ }
+)
+public class PlaintextConsumerAssignTest {
+
+ public static final int BROKER_COUNT = 3;
+
+ private final ClusterInstance clusterInstance;
+ private final String topic = "topic";
+ private final int partition = 0;
+ TopicPartition tp = new TopicPartition(topic, partition);
+
+ PlaintextConsumerAssignTest(ClusterInstance clusterInstance) {
+ this.clusterInstance = clusterInstance;
+ }
+
+ @BeforeEach
+ public void setup() throws InterruptedException {
+ clusterInstance.createTopic(topic, 2, (short) BROKER_COUNT);
+ }
+
+ @ClusterTest
+ public void testClassicAssignAndCommitAsyncNotCommitted() throws Exception {
+ testAssignAndCommitAsyncNotCommitted(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncAssignAndCommitAsyncNotCommitted() throws Exception {
+ testAssignAndCommitAsyncNotCommitted(GroupProtocol.CONSUMER);
+ }
+
+ private void testAssignAndCommitAsyncNotCommitted(GroupProtocol groupProtocol) throws InterruptedException {
+ int numRecords = 10000;
+ long startingTimestamp = System.currentTimeMillis();
+ CountConsumerCommitCallback cb = new CountConsumerCommitCallback();
+
+ try (Consumer consumer = clusterInstance.consumer(Map.of(GROUP_PROTOCOL_CONFIG, groupProtocol.name))) {
+ ClientsTestUtils.sendRecords(clusterInstance, tp, numRecords, startingTimestamp);
+ consumer.assign(List.of(tp));
+ consumer.commitAsync(cb);
+ ClientsTestUtils.pollUntilTrue(consumer, () -> cb.successCount >= 1 || cb.lastError.isPresent(),
+ 10000, "Failed to observe commit callback before timeout");
+ Map committedOffset = consumer.committed(Set.of(tp));
+ assertNotNull(committedOffset);
+ // No valid fetch position due to the absence of consumer.poll; and therefore no offset was committed to
+ // tp. The committed offset should be null. This is intentional.
+ assertNull(committedOffset.get(tp));
+ assertTrue(consumer.assignment().contains(tp));
+ }
+ }
+
+ @ClusterTest
+ public void testClassicAssignAndCommitSyncNotCommitted() throws Exception {
+ testAssignAndCommitSyncNotCommitted(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncAssignAndCommitSyncNotCommitted() {
+ testAssignAndCommitSyncNotCommitted(GroupProtocol.CONSUMER);
+ }
+
+ private void testAssignAndCommitSyncNotCommitted(GroupProtocol groupProtocol) {
+ int numRecords = 10000;
+ long startingTimestamp = System.currentTimeMillis();
+
+ try (Consumer consumer = clusterInstance.consumer(Map.of(GROUP_PROTOCOL_CONFIG, groupProtocol.name))) {
+ ClientsTestUtils.sendRecords(clusterInstance, tp, numRecords, startingTimestamp);
+ consumer.assign(List.of(tp));
+ consumer.commitSync();
+ Map committedOffset = consumer.committed(Set.of(tp));
+ assertNotNull(committedOffset);
+ // No valid fetch position due to the absence of consumer.poll; and therefore no offset was committed to
+ // tp. The committed offset should be null. This is intentional.
+ assertNull(committedOffset.get(tp));
+ assertTrue(consumer.assignment().contains(tp));
+ }
+ }
+
+ @ClusterTest
+ public void testClassicAssignAndCommitSyncAllConsumed() throws Exception {
+ testAssignAndCommitSyncAllConsumed(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncAssignAndCommitSyncAllConsumed() throws Exception {
+ testAssignAndCommitSyncAllConsumed(GroupProtocol.CONSUMER);
+ }
+
+ private void testAssignAndCommitSyncAllConsumed(GroupProtocol groupProtocol) throws InterruptedException {
+ int numRecords = 10000;
+
+ try (Consumer consumer = clusterInstance.consumer(Map.of(GROUP_PROTOCOL_CONFIG, groupProtocol.name))) {
+ long startingTimestamp = System.currentTimeMillis();
+ ClientsTestUtils.sendRecords(clusterInstance, tp, numRecords, startingTimestamp);
+ consumer.assign(List.of(tp));
+ consumer.seek(tp, 0);
+ ClientsTestUtils.consumeAndVerifyRecords(consumer, tp, numRecords, 0, 0, startingTimestamp);
+
+ consumer.commitSync();
+ Map committedOffset = consumer.committed(Set.of(tp));
+ assertNotNull(committedOffset);
+ assertNotNull(committedOffset.get(tp));
+ assertEquals(numRecords, committedOffset.get(tp).offset());
+ }
+ }
+
+ @ClusterTest
+ public void testClassicAssignAndConsume() throws InterruptedException {
+ testAssignAndConsume(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncAssignAndConsume() throws InterruptedException {
+ testAssignAndConsume(GroupProtocol.CONSUMER);
+ }
+
+ private void testAssignAndConsume(GroupProtocol groupProtocol) throws InterruptedException {
+ int numRecords = 10;
+
+ try (Consumer consumer = clusterInstance.consumer(Map.of(GROUP_PROTOCOL_CONFIG, groupProtocol.name))) {
+ long startingTimestamp = System.currentTimeMillis();
+ ClientsTestUtils.sendRecords(clusterInstance, tp, numRecords, startingTimestamp);
+ consumer.assign(List.of(tp));
+ ClientsTestUtils.consumeAndVerifyRecords(consumer, tp, numRecords, 0, 0, startingTimestamp);
+
+ assertEquals(numRecords, consumer.position(tp));
+ }
+ }
+
+ @ClusterTest
+ public void testClassicAssignAndConsumeSkippingPosition() throws InterruptedException {
+ testAssignAndConsumeSkippingPosition(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncAssignAndConsumeSkippingPosition() throws InterruptedException {
+ testAssignAndConsumeSkippingPosition(GroupProtocol.CONSUMER);
+ }
+
+ private void testAssignAndConsumeSkippingPosition(GroupProtocol groupProtocol) throws InterruptedException {
+ int numRecords = 10;
+
+ try (Consumer consumer = clusterInstance.consumer(Map.of(GROUP_PROTOCOL_CONFIG, groupProtocol.name))) {
+ long startingTimestamp = System.currentTimeMillis();
+ ClientsTestUtils.sendRecords(clusterInstance, tp, numRecords, startingTimestamp);
+ consumer.assign(List.of(tp));
+ int offset = 1;
+ consumer.seek(tp, offset);
+ ClientsTestUtils.consumeAndVerifyRecords(consumer, tp, numRecords - offset, offset, offset, startingTimestamp + offset);
+
+ assertEquals(numRecords, consumer.position(tp));
+ }
+ }
+
+ @ClusterTest
+ public void testClassicAssignAndFetchCommittedOffsets() throws InterruptedException {
+ testAssignAndFetchCommittedOffsets(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncAssignAndFetchCommittedOffsets() throws InterruptedException {
+ testAssignAndFetchCommittedOffsets(GroupProtocol.CONSUMER);
+ }
+
+ private void testAssignAndFetchCommittedOffsets(GroupProtocol groupProtocol) throws InterruptedException {
+ int numRecords = 100;
+ long startingTimestamp = System.currentTimeMillis();
+
+ try (Consumer consumer = clusterInstance.consumer(Map.of(GROUP_PROTOCOL_CONFIG, groupProtocol.name, GROUP_ID_CONFIG, "group1"))) {
+ ClientsTestUtils.sendRecords(clusterInstance, tp, numRecords, startingTimestamp);
+ consumer.assign(List.of(tp));
+ // First consumer consumes and commits offsets
+ consumer.seek(tp, 0);
+ ClientsTestUtils.consumeAndVerifyRecords(consumer, tp, numRecords, 0, 0, startingTimestamp);
+ consumer.commitSync();
+ assertEquals(numRecords, consumer.committed(Set.of(tp)).get(tp).offset());
+ }
+
+ // We should see the committed offsets from another consumer
+ try (Consumer anotherConsumer = clusterInstance.consumer(Map.of(GROUP_PROTOCOL_CONFIG, groupProtocol.name, GROUP_ID_CONFIG, "group1"))) {
+ anotherConsumer.assign(List.of(tp));
+ assertEquals(numRecords, anotherConsumer.committed(Set.of(tp)).get(tp).offset());
+ }
+ }
+
+ @ClusterTest
+ public void testClassicAssignAndConsumeFromCommittedOffsets() throws InterruptedException {
+ testAssignAndConsumeFromCommittedOffsets(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncAssignAndConsumeFromCommittedOffsets() throws InterruptedException {
+ testAssignAndConsumeFromCommittedOffsets(GroupProtocol.CONSUMER);
+ }
+
+ private void testAssignAndConsumeFromCommittedOffsets(GroupProtocol groupProtocol) throws InterruptedException {
+ int numRecords = 100;
+ int offset = 10;
+ long startingTimestamp = System.currentTimeMillis();
+ Map config = Map.of(GROUP_PROTOCOL_CONFIG, groupProtocol.name, GROUP_ID_CONFIG, "group1");
+
+ try (Consumer consumer = clusterInstance.consumer(config)) {
+ ClientsTestUtils.sendRecords(clusterInstance, tp, numRecords, startingTimestamp);
+ consumer.assign(List.of(tp));
+ consumer.commitSync(Map.of(tp, new OffsetAndMetadata(offset)));
+ assertEquals(offset, consumer.committed(Set.of(tp)).get(tp).offset());
+ }
+
+ // We should see the committed offsets from another consumer
+ try (Consumer anotherConsumer = clusterInstance.consumer(config)) {
+ assertEquals(offset, anotherConsumer.committed(Set.of(tp)).get(tp).offset());
+ anotherConsumer.assign(List.of(tp));
+ ClientsTestUtils.consumeAndVerifyRecords(anotherConsumer, tp, numRecords - offset, offset, offset, startingTimestamp + offset);
+ }
+ }
+
+ @ClusterTest
+ public void testClassicAssignAndRetrievingCommittedOffsetsMultipleTimes() throws InterruptedException {
+ testAssignAndRetrievingCommittedOffsetsMultipleTimes(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncAssignAndRetrievingCommittedOffsetsMultipleTimes() throws InterruptedException {
+ testAssignAndRetrievingCommittedOffsetsMultipleTimes(GroupProtocol.CONSUMER);
+ }
+
+ private void testAssignAndRetrievingCommittedOffsetsMultipleTimes(GroupProtocol groupProtocol) throws InterruptedException {
+ int numRecords = 100;
+ long startingTimestamp = System.currentTimeMillis();
+
+ try (Consumer consumer = clusterInstance.consumer(Map.of(GROUP_PROTOCOL_CONFIG, groupProtocol.name))) {
+ ClientsTestUtils.sendRecords(clusterInstance, tp, numRecords, startingTimestamp);
+ consumer.assign(List.of(tp));
+
+ // Consume and commit offsets
+ consumer.seek(tp, 0);
+ ClientsTestUtils.consumeAndVerifyRecords(consumer, tp, numRecords, 0, 0, startingTimestamp);
+ consumer.commitSync();
+
+ // Check committed offsets twice with same consumer
+ assertEquals(numRecords, consumer.committed(Set.of(tp)).get(tp).offset());
+ assertEquals(numRecords, consumer.committed(Set.of(tp)).get(tp).offset());
+ }
+ }
+
+ private static class CountConsumerCommitCallback implements OffsetCommitCallback {
+ int successCount = 0;
+ int failCount = 0;
+ Optional lastError = Optional.empty();
+
+ public void onComplete(Map offsets, Exception exception) {
+ if (exception == null) {
+ successCount += 1;
+ } else {
+ failCount += 1;
+ lastError = Optional.of(exception);
+ }
+ }
+ }
+}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerCallbackTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerCallbackTest.java
index 800ca3b2c78fc..0e09d62033b3b 100644
--- a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerCallbackTest.java
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerCallbackTest.java
@@ -16,25 +16,24 @@
*/
package org.apache.kafka.clients.consumer;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.ClientsTestUtils;
import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.record.TimestampType;
import org.apache.kafka.common.test.ClusterInstance;
-import org.apache.kafka.common.test.TestUtils;
import org.apache.kafka.common.test.api.ClusterTest;
import org.apache.kafka.common.test.api.ClusterTestDefaults;
import org.apache.kafka.common.test.api.Type;
-import java.time.Duration;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
+import static org.apache.kafka.clients.ClientsTestUtils.consumeAndVerifyRecords;
+import static org.apache.kafka.clients.ClientsTestUtils.sendRecords;
import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG;
import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_PROTOCOL_CONFIG;
import static org.apache.kafka.clients.consumer.GroupProtocol.CLASSIC;
@@ -169,6 +168,80 @@ public void testClassicConsumerGetPositionOfNewlyAssignedPartitionOnPartitionsAs
testGetPositionOfNewlyAssignedPartitionOnPartitionsAssignedCallback(CLASSIC);
}
+ @ClusterTest
+ public void testOnPartitionsAssignedCalledWithNewPartitionsOnlyForClassicCooperative() throws InterruptedException {
+ try (var consumer = createClassicConsumerCooperativeProtocol()) {
+ testOnPartitionsAssignedCalledWithExpectedPartitions(consumer, true);
+ }
+ }
+
+ @ClusterTest
+ public void testOnPartitionsAssignedCalledWithNewPartitionsOnlyForAsyncConsumer() throws InterruptedException {
+ try (var consumer = createConsumer(CONSUMER)) {
+ testOnPartitionsAssignedCalledWithExpectedPartitions(consumer, true);
+ }
+ }
+
+ @ClusterTest
+ public void testOnPartitionsAssignedCalledWithNewPartitionsOnlyForClassicEager() throws InterruptedException {
+ try (var consumer = createConsumer(CLASSIC)) {
+ testOnPartitionsAssignedCalledWithExpectedPartitions(consumer, false);
+ }
+ }
+
+ private void testOnPartitionsAssignedCalledWithExpectedPartitions(
+ Consumer consumer,
+ boolean expectNewPartitionsOnlyInCallback) throws InterruptedException {
+ subscribeAndExpectOnPartitionsAssigned(consumer, List.of(topic), List.of(tp));
+ assertEquals(Set.of(tp), consumer.assignment());
+
+ // Add a new partition assignment while keeping the previous one
+ String newTopic = "newTopic";
+ TopicPartition addedPartition = new TopicPartition(newTopic, 0);
+ List expectedPartitionsInCallback;
+ if (expectNewPartitionsOnlyInCallback) {
+ expectedPartitionsInCallback = List.of(addedPartition);
+ } else {
+ expectedPartitionsInCallback = List.of(tp, addedPartition);
+ }
+
+ // Change subscription to keep the previous one and add a new topic. Assignment should be updated
+ // to contain partitions from both topics, but the onPartitionsAssigned parameters may containing
+ // the full new assignment or just the newly added partitions depending on the case.
+ subscribeAndExpectOnPartitionsAssigned(
+ consumer,
+ List.of(topic, newTopic),
+ expectedPartitionsInCallback);
+ assertEquals(Set.of(tp, addedPartition), consumer.assignment());
+ }
+
+ private void subscribeAndExpectOnPartitionsAssigned(Consumer consumer, List topics, Collection expectedPartitionsInCallback) throws InterruptedException {
+ var partitionsAssigned = new AtomicBoolean(false);
+ AtomicReference> partitionsFromCallback = new AtomicReference<>();
+ consumer.subscribe(topics, new ConsumerRebalanceListener() {
+ @Override
+ public void onPartitionsAssigned(Collection partitions) {
+ if (partitions.containsAll(expectedPartitionsInCallback)) {
+ partitionsFromCallback.set(partitions);
+ partitionsAssigned.set(true);
+ }
+ }
+
+ @Override
+ public void onPartitionsRevoked(Collection partitions) {
+ // noop
+ }
+ });
+ ClientsTestUtils.pollUntilTrue(
+ consumer,
+ partitionsAssigned::get,
+ "Timed out before expected rebalance completed"
+ );
+ // These are different types, so comparing values instead
+ assertTrue(expectedPartitionsInCallback.containsAll(partitionsFromCallback.get()) && partitionsFromCallback.get().containsAll(expectedPartitionsInCallback),
+ "Expected partitions " + expectedPartitionsInCallback + " as parameter for onPartitionsAssigned, but got " + partitionsFromCallback.get());
+ }
+
@ClusterTest
public void testAsyncConsumerGetPositionOfNewlyAssignedPartitionOnPartitionsAssignedCallback() throws InterruptedException {
testGetPositionOfNewlyAssignedPartitionOnPartitionsAssignedCallback(CONSUMER);
@@ -198,7 +271,7 @@ private void testSeekPositionAndPauseNewlyAssignedPartitionOnPartitionsAssignedC
var totalRecords = 120;
var startingTimestamp = 0L;
- sendRecords(totalRecords, startingTimestamp);
+ sendRecords(cluster, tp, totalRecords, startingTimestamp);
triggerOnPartitionsAssigned(tp, consumer, (executeConsumer, partitions) -> {
executeConsumer.seek(tp, startingOffset);
@@ -209,6 +282,7 @@ private void testSeekPositionAndPauseNewlyAssignedPartitionOnPartitionsAssignedC
consumer.resume(List.of(tp));
consumeAndVerifyRecords(
consumer,
+ tp,
(int) (totalRecords - startingOffset),
(int) startingOffset,
(int) startingOffset,
@@ -238,11 +312,9 @@ public void onPartitionsRevoked(Collection partitions) {
// noop
}
});
- TestUtils.waitForCondition(
- () -> {
- consumer.poll(Duration.ofMillis(100));
- return partitionsAssigned.get();
- },
+ ClientsTestUtils.pollUntilTrue(
+ consumer,
+ partitionsAssigned::get,
"Timed out before expected rebalance completed"
);
}
@@ -273,11 +345,9 @@ public void onPartitionsRevoked(Collection partitions) {
}
}
});
- TestUtils.waitForCondition(
- () -> {
- consumer.poll(Duration.ofMillis(100));
- return partitionsAssigned.get();
- },
+ ClientsTestUtils.pollUntilTrue(
+ consumer,
+ partitionsAssigned::get,
"Timed out before expected rebalance completed"
);
}
@@ -291,62 +361,11 @@ private Consumer createConsumer(GroupProtocol protocol) {
));
}
- private void sendRecords(int numRecords, long startingTimestamp) {
- try (Producer producer = cluster.producer()) {
- for (var i = 0; i < numRecords; i++) {
- var timestamp = startingTimestamp + i;
- var record = new ProducerRecord<>(
- tp.topic(),
- tp.partition(),
- timestamp,
- ("key " + i).getBytes(),
- ("value " + i).getBytes()
- );
- producer.send(record);
- }
- producer.flush();
- }
- }
-
- protected void consumeAndVerifyRecords(
- Consumer consumer,
- int numRecords,
- int startingOffset,
- int startingKeyAndValueIndex,
- long startingTimestamp
- ) throws InterruptedException {
- var records = consumeRecords(consumer, numRecords);
- for (var i = 0; i < numRecords; i++) {
- var record = records.get(i);
- var offset = startingOffset + i;
-
- assertEquals(tp.topic(), record.topic());
- assertEquals(tp.partition(), record.partition());
-
- assertEquals(TimestampType.CREATE_TIME, record.timestampType());
- var timestamp = startingTimestamp + i;
- assertEquals(timestamp, record.timestamp());
-
- assertEquals(offset, record.offset());
- var keyAndValueIndex = startingKeyAndValueIndex + i;
- assertEquals("key " + keyAndValueIndex, new String(record.key()));
- assertEquals("value " + keyAndValueIndex, new String(record.value()));
- // this is true only because K and V are byte arrays
- assertEquals(("key " + keyAndValueIndex).length(), record.serializedKeySize());
- assertEquals(("value " + keyAndValueIndex).length(), record.serializedValueSize());
- }
- }
-
- protected List> consumeRecords(
- Consumer consumer,
- int numRecords
- ) throws InterruptedException {
- List> records = new ArrayList<>();
- TestUtils.waitForCondition(() -> {
- consumer.poll(Duration.ofMillis(100)).forEach(records::add);
- return records.size() >= numRecords;
- }, 60000, "Timed out before consuming expected " + numRecords + " records.");
-
- return records;
+ private Consumer createClassicConsumerCooperativeProtocol() {
+ return cluster.consumer(Map.of(
+ GROUP_PROTOCOL_CONFIG, CLASSIC.name.toLowerCase(Locale.ROOT),
+ ENABLE_AUTO_COMMIT_CONFIG, "false",
+ ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, "org.apache.kafka.clients.consumer.CooperativeStickyAssignor"
+ ));
}
}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerCommitTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerCommitTest.java
new file mode 100644
index 0000000000000..c00d1ddab90a0
--- /dev/null
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerCommitTest.java
@@ -0,0 +1,636 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer;
+
+import org.apache.kafka.clients.ClientsTestUtils;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.internals.Topic;
+import org.apache.kafka.common.test.ClusterInstance;
+import org.apache.kafka.common.test.api.ClusterConfigProperty;
+import org.apache.kafka.common.test.api.ClusterTest;
+import org.apache.kafka.common.test.api.ClusterTestDefaults;
+import org.apache.kafka.common.test.api.Type;
+import org.apache.kafka.test.MockConsumerInterceptor;
+import org.apache.kafka.test.TestUtils;
+
+import org.junit.jupiter.api.BeforeEach;
+
+import java.time.Duration;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.apache.kafka.clients.ClientsTestUtils.awaitAssignment;
+import static org.apache.kafka.clients.ClientsTestUtils.consumeAndVerifyRecords;
+import static org.apache.kafka.clients.ClientsTestUtils.sendRecords;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_PROTOCOL_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG;
+import static org.apache.kafka.clients.consumer.PlaintextConsumerCommitTest.BROKER_COUNT;
+import static org.apache.kafka.clients.consumer.PlaintextConsumerCommitTest.OFFSETS_TOPIC_PARTITIONS;
+import static org.apache.kafka.clients.consumer.PlaintextConsumerCommitTest.OFFSETS_TOPIC_REPLICATION;
+import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.GROUP_MIN_SESSION_TIMEOUT_MS_CONFIG;
+import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.OFFSETS_TOPIC_PARTITIONS_CONFIG;
+import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.OFFSETS_TOPIC_REPLICATION_FACTOR_CONFIG;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@ClusterTestDefaults(
+ types = {Type.KRAFT},
+ brokers = BROKER_COUNT,
+ serverProperties = {
+ @ClusterConfigProperty(key = OFFSETS_TOPIC_PARTITIONS_CONFIG, value = OFFSETS_TOPIC_PARTITIONS),
+ @ClusterConfigProperty(key = OFFSETS_TOPIC_REPLICATION_FACTOR_CONFIG, value = OFFSETS_TOPIC_REPLICATION),
+ @ClusterConfigProperty(key = GROUP_MIN_SESSION_TIMEOUT_MS_CONFIG, value = "100"),
+ }
+)
+public class PlaintextConsumerCommitTest {
+
+ public static final int BROKER_COUNT = 3;
+ public static final String OFFSETS_TOPIC_PARTITIONS = "1";
+ public static final String OFFSETS_TOPIC_REPLICATION = "3";
+ private final ClusterInstance cluster;
+ private final String topic = "topic";
+ private final TopicPartition tp = new TopicPartition(topic, 0);
+ private final TopicPartition tp1 = new TopicPartition(topic, 1);
+
+ public PlaintextConsumerCommitTest(ClusterInstance clusterInstance) {
+ this.cluster = clusterInstance;
+ }
+
+ @BeforeEach
+ public void setup() throws InterruptedException {
+ cluster.createTopic(topic, 2, (short) BROKER_COUNT);
+ }
+
+ @ClusterTest
+ public void testClassicConsumerAutoCommitOnClose() throws InterruptedException {
+ testAutoCommitOnClose(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerAutoCommitOnClose() throws InterruptedException {
+ testAutoCommitOnClose(GroupProtocol.CONSUMER);
+ }
+
+ private void testAutoCommitOnClose(GroupProtocol groupProtocol) throws InterruptedException {
+ try (var consumer = createConsumer(groupProtocol, true)) {
+ sendRecords(cluster, tp, 1000);
+
+ consumer.subscribe(List.of(topic));
+ awaitAssignment(consumer, Set.of(tp, tp1));
+ // should auto-commit sought positions before closing
+ consumer.seek(tp, 300);
+ consumer.seek(tp1, 500);
+ }
+
+ // now we should see the committed positions from another consumer
+ try (var anotherConsumer = createConsumer(groupProtocol, true)) {
+ assertEquals(300, anotherConsumer.committed(Set.of(tp)).get(tp).offset());
+ assertEquals(500, anotherConsumer.committed(Set.of(tp1)).get(tp1).offset());
+ }
+ }
+
+ @ClusterTest
+ public void testClassicConsumerAutoCommitOnCloseAfterWakeup() throws InterruptedException {
+ testAutoCommitOnCloseAfterWakeup(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerAutoCommitOnCloseAfterWakeup() throws InterruptedException {
+ testAutoCommitOnCloseAfterWakeup(GroupProtocol.CONSUMER);
+ }
+
+ private void testAutoCommitOnCloseAfterWakeup(GroupProtocol groupProtocol) throws InterruptedException {
+ try (var consumer = createConsumer(groupProtocol, true)) {
+ sendRecords(cluster, tp, 1000);
+
+ consumer.subscribe(List.of(topic));
+ awaitAssignment(consumer, Set.of(tp, tp1));
+
+ // should auto-commit sought positions before closing
+ consumer.seek(tp, 300);
+ consumer.seek(tp1, 500);
+
+ // wakeup the consumer before closing to simulate trying to break a poll
+ // loop from another thread
+ consumer.wakeup();
+ }
+
+ // now we should see the committed positions from another consumer
+ try (var anotherConsumer = createConsumer(groupProtocol, true)) {
+ assertEquals(300, anotherConsumer.committed(Set.of(tp)).get(tp).offset());
+ assertEquals(500, anotherConsumer.committed(Set.of(tp1)).get(tp1).offset());
+ }
+ }
+
+ @ClusterTest
+ public void testClassicConsumerCommitMetadata() throws InterruptedException {
+ testCommitMetadata(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerCommitMetadata() throws InterruptedException {
+ testCommitMetadata(GroupProtocol.CONSUMER);
+ }
+
+ private void testCommitMetadata(GroupProtocol groupProtocol) throws InterruptedException {
+ try (var consumer = createConsumer(groupProtocol, true)) {
+ consumer.assign(List.of(tp));
+ // sync commit
+ var syncMetadata = new OffsetAndMetadata(5, Optional.of(15), "foo");
+ consumer.commitSync(Map.of(tp, syncMetadata));
+ assertEquals(syncMetadata, consumer.committed(Set.of(tp)).get(tp));
+
+ // async commit
+ var asyncMetadata = new OffsetAndMetadata(10, "bar");
+ sendAndAwaitAsyncCommit(consumer, Map.of(tp, asyncMetadata));
+ assertEquals(asyncMetadata, consumer.committed(Set.of(tp)).get(tp));
+
+ // handle null metadata
+ var nullMetadata = new OffsetAndMetadata(5, null);
+ consumer.commitSync(Map.of(tp, nullMetadata));
+ assertEquals(nullMetadata, consumer.committed(Set.of(tp)).get(tp));
+ }
+ }
+
+ @ClusterTest
+ public void testClassicConsumerAsyncCommit() throws InterruptedException {
+ testAsyncCommit(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerAsyncCommit() throws InterruptedException {
+ testAsyncCommit(GroupProtocol.CONSUMER);
+ }
+
+ private void testAsyncCommit(GroupProtocol groupProtocol) throws InterruptedException {
+ // Ensure the __consumer_offsets topic is created to prevent transient issues,
+ // such as RetriableCommitFailedException during async offset commits.
+ cluster.createTopic(
+ Topic.GROUP_METADATA_TOPIC_NAME,
+ Integer.parseInt(OFFSETS_TOPIC_PARTITIONS),
+ Short.parseShort(OFFSETS_TOPIC_REPLICATION)
+ );
+ try (var consumer = createConsumer(groupProtocol, false)) {
+ consumer.assign(List.of(tp));
+
+ var callback = new CountConsumerCommitCallback();
+ var count = 5;
+ for (var i = 1; i <= count; i++)
+ consumer.commitAsync(Map.of(tp, new OffsetAndMetadata(i)), callback);
+
+ ClientsTestUtils.pollUntilTrue(
+ consumer,
+ () -> callback.successCount >= count || callback.lastError.isPresent(),
+ "Failed to observe commit callback before timeout"
+ );
+
+ assertEquals(Optional.empty(), callback.lastError);
+ assertEquals(count, callback.successCount);
+ assertEquals(new OffsetAndMetadata(count), consumer.committed(Set.of(tp)).get(tp));
+ }
+ }
+
+ @ClusterTest
+ public void testClassicConsumerAutoCommitIntercept() throws InterruptedException {
+ testAutoCommitIntercept(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerAutoCommitIntercept() throws InterruptedException {
+ testAutoCommitIntercept(GroupProtocol.CONSUMER);
+ }
+
+ private void testAutoCommitIntercept(GroupProtocol groupProtocol) throws InterruptedException {
+ var topic2 = "topic2";
+ cluster.createTopic(topic2, 2, (short) 3);
+ var numRecords = 100;
+ try (var producer = cluster.producer();
+ // create consumer with interceptor
+ Consumer consumer = cluster.consumer(Map.of(
+ GROUP_PROTOCOL_CONFIG, groupProtocol.name().toLowerCase(Locale.ROOT),
+ ENABLE_AUTO_COMMIT_CONFIG, "true",
+ INTERCEPTOR_CLASSES_CONFIG, "org.apache.kafka.test.MockConsumerInterceptor"
+ ))
+ ) {
+ // produce records
+ for (var i = 0; i < numRecords; i++) {
+ producer.send(new ProducerRecord<>(tp.topic(), tp.partition(), ("key " + i).getBytes(), ("value " + i).getBytes()));
+ }
+
+ var rebalanceListener = new ConsumerRebalanceListener() {
+ @Override
+ public void onPartitionsAssigned(Collection partitions) {
+ // keep partitions paused in this test so that we can verify the commits based on specific seeks
+ consumer.pause(partitions);
+ }
+ @Override
+ public void onPartitionsRevoked(Collection partitions) {
+ // No-op
+ }
+ };
+
+ changeConsumerSubscriptionAndValidateAssignment(
+ consumer,
+ List.of(topic),
+ Set.of(tp, tp1),
+ rebalanceListener
+ );
+ consumer.seek(tp, 10);
+ consumer.seek(tp1, 20);
+
+ // change subscription to trigger rebalance
+ var commitCountBeforeRebalance = MockConsumerInterceptor.ON_COMMIT_COUNT.intValue();
+ var expectedAssignment = Set.of(tp, tp1, new TopicPartition(topic2, 0), new TopicPartition(topic2, 1));
+ changeConsumerSubscriptionAndValidateAssignment(
+ consumer,
+ List.of(topic, topic2),
+ expectedAssignment,
+ rebalanceListener
+ );
+
+ // after rebalancing, we should have reset to the committed positions
+ var committed1 = consumer.committed(Set.of(tp));
+ assertEquals(10, committed1.get(tp).offset());
+ var committed2 = consumer.committed(Set.of(tp1));
+ assertEquals(20, committed2.get(tp1).offset());
+
+ // In both CLASSIC and CONSUMER protocols, interceptors are executed in poll and close.
+ // However, in the CONSUMER protocol, the assignment may be changed outside a poll, so
+ // we need to poll once to ensure the interceptor is called.
+ if (groupProtocol == GroupProtocol.CONSUMER) {
+ consumer.poll(Duration.ZERO);
+ }
+
+ assertTrue(MockConsumerInterceptor.ON_COMMIT_COUNT.intValue() > commitCountBeforeRebalance);
+
+ // verify commits are intercepted on close
+ var commitCountBeforeClose = MockConsumerInterceptor.ON_COMMIT_COUNT.intValue();
+ consumer.close();
+ assertTrue(MockConsumerInterceptor.ON_COMMIT_COUNT.intValue() > commitCountBeforeClose);
+ producer.close();
+ // cleanup
+ MockConsumerInterceptor.resetCounters();
+ }
+ }
+
+ @ClusterTest
+ public void testClassicConsumerCommitSpecifiedOffsets() throws InterruptedException {
+ testCommitSpecifiedOffsets(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerCommitSpecifiedOffsets() throws InterruptedException {
+ testCommitSpecifiedOffsets(GroupProtocol.CONSUMER);
+ }
+
+ private void testCommitSpecifiedOffsets(GroupProtocol groupProtocol) throws InterruptedException {
+ try (Producer producer = cluster.producer();
+ var consumer = createConsumer(groupProtocol, false)
+ ) {
+ sendRecords(producer, tp, 5, System.currentTimeMillis());
+ sendRecords(producer, tp1, 7, System.currentTimeMillis());
+
+ consumer.assign(List.of(tp, tp1));
+
+ var pos1 = consumer.position(tp);
+ var pos2 = consumer.position(tp1);
+
+ consumer.commitSync(Map.of(tp, new OffsetAndMetadata(3L)));
+
+ assertEquals(3, consumer.committed(Set.of(tp)).get(tp).offset());
+ assertNull(consumer.committed(Collections.singleton(tp1)).get(tp1));
+
+ // Positions should not change
+ assertEquals(pos1, consumer.position(tp));
+ assertEquals(pos2, consumer.position(tp1));
+
+ consumer.commitSync(Map.of(tp1, new OffsetAndMetadata(5L)));
+
+ assertEquals(3, consumer.committed(Set.of(tp)).get(tp).offset());
+ assertEquals(5, consumer.committed(Set.of(tp1)).get(tp1).offset());
+
+ // Using async should pick up the committed changes after commit completes
+ sendAndAwaitAsyncCommit(consumer, Map.of(tp1, new OffsetAndMetadata(7L)));
+ assertEquals(7, consumer.committed(Collections.singleton(tp1)).get(tp1).offset());
+ }
+ }
+
+ @ClusterTest
+ public void testClassicConsumerAutoCommitOnRebalance() throws InterruptedException {
+ testAutoCommitOnRebalance(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerAutoCommitOnRebalance() throws InterruptedException {
+ testAutoCommitOnRebalance(GroupProtocol.CONSUMER);
+ }
+
+ private void testAutoCommitOnRebalance(GroupProtocol groupProtocol) throws InterruptedException {
+ var topic2 = "topic2";
+ cluster.createTopic(topic2, 2, (short) BROKER_COUNT);
+ try (var consumer = createConsumer(groupProtocol, true)) {
+ sendRecords(cluster, tp, 1000);
+
+ var rebalanceListener = new ConsumerRebalanceListener() {
+ @Override
+ public void onPartitionsAssigned(Collection partitions) {
+ // keep partitions paused in this test so that we can verify the commits based on specific seeks
+ consumer.pause(partitions);
+ }
+
+ @Override
+ public void onPartitionsRevoked(Collection partitions) {
+
+ }
+ };
+
+ consumer.subscribe(List.of(topic), rebalanceListener);
+ awaitAssignment(consumer, Set.of(tp, tp1));
+
+ consumer.seek(tp, 300);
+ consumer.seek(tp1, 500);
+ // change subscription to trigger rebalance
+ consumer.subscribe(List.of(topic, topic2), rebalanceListener);
+
+ var newAssignment = Set.of(tp, tp1, new TopicPartition(topic2, 0), new TopicPartition(topic2, 1));
+ awaitAssignment(consumer, newAssignment);
+
+ // after rebalancing, we should have reset to the committed positions
+ assertEquals(300, consumer.committed(Set.of(tp)).get(tp).offset());
+ assertEquals(500, consumer.committed(Set.of(tp1)).get(tp1).offset());
+ }
+ }
+
+ @ClusterTest
+ public void testClassicConsumerSubscribeAndCommitSync() throws InterruptedException {
+ testSubscribeAndCommitSync(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerSubscribeAndCommitSync() throws InterruptedException {
+ testSubscribeAndCommitSync(GroupProtocol.CONSUMER);
+ }
+
+ private void testSubscribeAndCommitSync(GroupProtocol groupProtocol) throws InterruptedException {
+ // This test ensure that the member ID is propagated from the group coordinator when the
+ // assignment is received into a subsequent offset commit
+ try (var consumer = createConsumer(groupProtocol, false)) {
+ assertEquals(0, consumer.assignment().size());
+ consumer.subscribe(List.of(topic));
+ awaitAssignment(consumer, Set.of(tp, tp1));
+
+ consumer.seek(tp, 0);
+ consumer.commitSync();
+ }
+ }
+
+ @ClusterTest
+ public void testClassicConsumerPositionAndCommit() throws InterruptedException {
+ testPositionAndCommit(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerPositionAndCommit() throws InterruptedException {
+ testPositionAndCommit(GroupProtocol.CONSUMER);
+ }
+
+ private void testPositionAndCommit(GroupProtocol groupProtocol) throws InterruptedException {
+ try (Producer producer = cluster.producer();
+ var consumer = createConsumer(groupProtocol, false);
+ var otherConsumer = createConsumer(groupProtocol, false)
+ ) {
+ var startingTimestamp = System.currentTimeMillis();
+ sendRecords(producer, tp, 5, startingTimestamp);
+
+ var topicPartition = new TopicPartition(topic, 15);
+ assertNull(consumer.committed(Collections.singleton(topicPartition)).get(topicPartition));
+
+ // position() on a partition that we aren't subscribed to throws an exception
+ assertThrows(IllegalStateException.class, () -> consumer.position(topicPartition));
+
+ consumer.assign(List.of(tp));
+
+ assertEquals(0L, consumer.position(tp), "position() on a partition that we are subscribed to should reset the offset");
+ consumer.commitSync();
+ assertEquals(0L, consumer.committed(Set.of(tp)).get(tp).offset());
+ consumeAndVerifyRecords(consumer, tp, 5, 0, 0, startingTimestamp);
+ assertEquals(5L, consumer.position(tp), "After consuming 5 records, position should be 5");
+ consumer.commitSync();
+ assertEquals(5L, consumer.committed(Set.of(tp)).get(tp).offset(), "Committed offset should be returned");
+
+ startingTimestamp = System.currentTimeMillis();
+ sendRecords(producer, tp, 1, startingTimestamp);
+
+ // another consumer in the same group should get the same position
+ otherConsumer.assign(List.of(tp));
+ consumeAndVerifyRecords(otherConsumer, tp, 1, 5, 0, startingTimestamp);
+ }
+ }
+
+ /**
+ * This is testing when closing the consumer but commit request has already been sent.
+ * During the closing, the consumer won't find the coordinator anymore.
+ */
+ @ClusterTest
+ public void testCommitAsyncFailsWhenCoordinatorUnavailableDuringClose() throws InterruptedException {
+ try (Producer producer = cluster.producer();
+ var consumer = createConsumer(GroupProtocol.CONSUMER, false)
+ ) {
+ sendRecords(producer, tp, 3, System.currentTimeMillis());
+ consumer.assign(List.of(tp));
+
+ var callback = new CountConsumerCommitCallback();
+
+ // Close the coordinator before committing because otherwise the commit will fail to find the coordinator.
+ cluster.brokerIds().forEach(cluster::shutdownBroker);
+
+ TestUtils.waitForCondition(() -> cluster.aliveBrokers().isEmpty(), "All brokers should be shut down");
+
+ consumer.poll(Duration.ofMillis(500));
+ consumer.commitAsync(Map.of(tp, new OffsetAndMetadata(1L)), callback);
+
+ long startTime = System.currentTimeMillis();
+ consumer.close(CloseOptions.timeout(Duration.ofMillis(500)));
+ long closeDuration = System.currentTimeMillis() - startTime;
+
+ assertTrue(closeDuration < 1000, "The closing process for the consumer was too long: " + closeDuration + " ms");
+ assertTrue(callback.lastError.isPresent());
+ assertEquals(CommitFailedException.class, callback.lastError.get().getClass());
+ assertEquals("Failed to commit offsets: Coordinator unknown and consumer is closing", callback.lastError.get().getMessage());
+ assertEquals(1, callback.exceptionCount);
+ }
+ }
+
+ // TODO: This only works in the new consumer, but should be fixed for the old consumer as well
+ @ClusterTest
+ public void testCommitAsyncCompletedBeforeConsumerCloses() throws InterruptedException {
+ // This is testing the contract that asynchronous offset commit are completed before the consumer
+ // is closed, even when no commit sync is performed as part of the close (due to auto-commit
+ // disabled, or simply because there are no consumed offsets).
+
+ // Create offsets topic to ensure coordinator is available during close
+ cluster.createTopic(Topic.GROUP_METADATA_TOPIC_NAME, Integer.parseInt(OFFSETS_TOPIC_PARTITIONS), Short.parseShort(OFFSETS_TOPIC_REPLICATION));
+
+ try (Producer producer = cluster.producer(Map.of(ProducerConfig.ACKS_CONFIG, "all"));
+ var consumer = createConsumer(GroupProtocol.CONSUMER, false)
+ ) {
+ sendRecords(producer, tp, 3, System.currentTimeMillis());
+ sendRecords(producer, tp1, 3, System.currentTimeMillis());
+ consumer.assign(List.of(tp, tp1));
+
+ // Try without looking up the coordinator first
+ var cb = new CountConsumerCommitCallback();
+ consumer.commitAsync(Map.of(tp, new OffsetAndMetadata(1L)), cb);
+ consumer.commitAsync(Map.of(tp1, new OffsetAndMetadata(1L)), cb);
+
+ consumer.close();
+ assertEquals(2, cb.successCount);
+ }
+ }
+
+ // TODO: This only works in the new consumer, but should be fixed for the old consumer as well
+ @ClusterTest
+ public void testCommitAsyncCompletedBeforeCommitSyncReturns() {
+ // This is testing the contract that asynchronous offset commits sent previously with the
+ // `commitAsync` are guaranteed to have their callbacks invoked prior to completion of
+ // `commitSync` (given that it does not time out).
+ try (Producer producer = cluster.producer();
+ var consumer = createConsumer(GroupProtocol.CONSUMER, false)
+ ) {
+ sendRecords(producer, tp, 3, System.currentTimeMillis());
+ sendRecords(producer, tp1, 3, System.currentTimeMillis());
+
+ consumer.assign(List.of(tp, tp1));
+
+ // Try without looking up the coordinator first
+ var cb = new CountConsumerCommitCallback();
+ consumer.commitAsync(Map.of(tp, new OffsetAndMetadata(1L)), cb);
+ consumer.commitSync(Map.of());
+
+ assertEquals(1, consumer.committed(Set.of(tp)).get(tp).offset());
+ assertEquals(1, cb.successCount);
+
+ // Try with coordinator known
+ consumer.commitAsync(Map.of(tp, new OffsetAndMetadata(2L)), cb);
+ consumer.commitSync(Map.of(tp1, new OffsetAndMetadata(2L)));
+
+ assertEquals(2, consumer.committed(Set.of(tp)).get(tp).offset());
+ assertEquals(2, consumer.committed(Set.of(tp1)).get(tp1).offset());
+ assertEquals(2, cb.successCount);
+
+ // Try with empty sync commit
+ consumer.commitAsync(Map.of(tp, new OffsetAndMetadata(3L)), cb);
+ consumer.commitSync(Map.of());
+
+ assertEquals(3, consumer.committed(Set.of(tp)).get(tp).offset());
+ assertEquals(2, consumer.committed(Set.of(tp1)).get(tp1).offset());
+ assertEquals(3, cb.successCount);
+ }
+ }
+
+ private Consumer createConsumer(GroupProtocol protocol, boolean enableAutoCommit) {
+ return cluster.consumer(Map.of(
+ GROUP_ID_CONFIG, "test-group",
+ GROUP_PROTOCOL_CONFIG, protocol.name().toLowerCase(Locale.ROOT),
+ ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit
+ ));
+ }
+
+ private void sendAndAwaitAsyncCommit(
+ Consumer consumer,
+ Map offsetsOpt
+ ) throws InterruptedException {
+ var commitCallback = new RetryCommitCallback(consumer, offsetsOpt);
+
+ commitCallback.sendAsyncCommit();
+ ClientsTestUtils.pollUntilTrue(
+ consumer,
+ () -> commitCallback.isComplete,
+ "Failed to observe commit callback before timeout"
+ );
+
+ assertEquals(Optional.empty(), commitCallback.error);
+ }
+
+ private static class RetryCommitCallback implements OffsetCommitCallback {
+ private boolean isComplete = false;
+ private Optional error = Optional.empty();
+
+ private final Consumer consumer;
+ private final Map offsetsOpt;
+
+ public RetryCommitCallback(
+ Consumer consumer,
+ Map offsetsOpt
+ ) {
+ this.consumer = consumer;
+ this.offsetsOpt = offsetsOpt;
+ }
+
+ @Override
+ public void onComplete(Map offsets, Exception exception) {
+ if (exception instanceof RetriableCommitFailedException) {
+ sendAsyncCommit();
+ } else {
+ isComplete = true;
+ error = Optional.ofNullable(exception);
+ }
+ }
+
+ void sendAsyncCommit() {
+ consumer.commitAsync(offsetsOpt, this);
+ }
+ }
+
+ private static class CountConsumerCommitCallback implements OffsetCommitCallback {
+ private int successCount = 0;
+ private int exceptionCount = 0;
+ private Optional lastError = Optional.empty();
+
+ @Override
+ public void onComplete(Map offsets, Exception exception) {
+ if (exception == null) {
+ successCount += 1;
+ } else {
+ exceptionCount += 1;
+ lastError = Optional.of(exception);
+ }
+ }
+ }
+
+ private void changeConsumerSubscriptionAndValidateAssignment(
+ Consumer consumer,
+ List topicsToSubscribe,
+ Set expectedAssignment,
+ ConsumerRebalanceListener rebalanceListener
+ ) throws InterruptedException {
+ consumer.subscribe(topicsToSubscribe, rebalanceListener);
+ awaitAssignment(consumer, expectedAssignment);
+ }
+}
diff --git a/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerFetchTest.java b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerFetchTest.java
new file mode 100644
index 0000000000000..e65dd0b7ed98f
--- /dev/null
+++ b/clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerFetchTest.java
@@ -0,0 +1,489 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer;
+
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.test.ClusterInstance;
+import org.apache.kafka.common.test.api.ClusterConfigProperty;
+import org.apache.kafka.common.test.api.ClusterTest;
+import org.apache.kafka.common.test.api.ClusterTestDefaults;
+import org.apache.kafka.common.test.api.Type;
+
+import org.junit.jupiter.api.BeforeEach;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+import static org.apache.kafka.clients.ClientsTestUtils.awaitAssignment;
+import static org.apache.kafka.clients.ClientsTestUtils.consumeAndVerifyRecords;
+import static org.apache.kafka.clients.ClientsTestUtils.consumeRecords;
+import static org.apache.kafka.clients.ClientsTestUtils.sendRecords;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_OFFSET_RESET_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.FETCH_MAX_BYTES_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_PROTOCOL_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG;
+import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.GROUP_MIN_SESSION_TIMEOUT_MS_CONFIG;
+import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.OFFSETS_TOPIC_PARTITIONS_CONFIG;
+import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.OFFSETS_TOPIC_REPLICATION_FACTOR_CONFIG;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@ClusterTestDefaults(
+ types = {Type.KRAFT},
+ brokers = PlaintextConsumerFetchTest.BROKER_COUNT,
+ serverProperties = {
+ @ClusterConfigProperty(key = OFFSETS_TOPIC_PARTITIONS_CONFIG, value = "1"),
+ @ClusterConfigProperty(key = OFFSETS_TOPIC_REPLICATION_FACTOR_CONFIG, value = "3"),
+ @ClusterConfigProperty(key = GROUP_MIN_SESSION_TIMEOUT_MS_CONFIG, value = "100"),
+ }
+)
+public class PlaintextConsumerFetchTest {
+
+ public static final int BROKER_COUNT = 3;
+ private final ClusterInstance cluster;
+ private final String topic = "topic";
+ private final TopicPartition tp = new TopicPartition(topic, 0);
+ private final TopicPartition tp2 = new TopicPartition(topic, 1);
+
+ public PlaintextConsumerFetchTest(ClusterInstance cluster) {
+ this.cluster = cluster;
+ }
+
+ @BeforeEach
+ public void setup() throws InterruptedException {
+ cluster.createTopic(topic, 2, (short) BROKER_COUNT);
+ }
+
+ @ClusterTest
+ public void testClassicConsumerFetchInvalidOffset() {
+ testFetchInvalidOffset(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerFetchInvalidOffset() {
+ testFetchInvalidOffset(GroupProtocol.CONSUMER);
+ }
+
+ private void testFetchInvalidOffset(GroupProtocol groupProtocol) {
+ Map config = Map.of(
+ GROUP_PROTOCOL_CONFIG, groupProtocol.name().toLowerCase(Locale.ROOT),
+ AUTO_OFFSET_RESET_CONFIG, "none"
+ );
+ try (var consumer = cluster.consumer(config)) {
+ // produce one record
+ var totalRecords = 2;
+ sendRecords(cluster, tp, totalRecords);
+ consumer.assign(List.of(tp));
+
+ // poll should fail because there is no offset reset strategy set.
+ // we fail only when resetting positions after coordinator is known, so using a long timeout.
+ assertThrows(NoOffsetForPartitionException.class, () -> consumer.poll(Duration.ofMillis(15000)));
+
+ // seek to out of range position
+ var outOfRangePos = totalRecords + 1;
+ consumer.seek(tp, outOfRangePos);
+ var e = assertThrows(OffsetOutOfRangeException.class, () -> consumer.poll(Duration.ofMillis(20000)));
+ var outOfRangePartitions = e.offsetOutOfRangePartitions();
+ assertNotNull(outOfRangePartitions);
+ assertEquals(1, outOfRangePartitions.size());
+ assertEquals(outOfRangePos, outOfRangePartitions.get(tp).longValue());
+ }
+ }
+
+ @ClusterTest
+ public void testClassicConsumerFetchOutOfRangeOffsetResetConfigEarliest() throws InterruptedException {
+ testFetchOutOfRangeOffsetResetConfigEarliest(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerFetchOutOfRangeOffsetResetConfigEarliest() throws InterruptedException {
+ testFetchOutOfRangeOffsetResetConfigEarliest(GroupProtocol.CONSUMER);
+ }
+
+ private void testFetchOutOfRangeOffsetResetConfigEarliest(GroupProtocol groupProtocol) throws InterruptedException {
+ Map config = Map.of(
+ GROUP_PROTOCOL_CONFIG, groupProtocol.name().toLowerCase(Locale.ROOT),
+ // ensure no in-flight fetch request so that the offset can be reset immediately
+ FETCH_MAX_WAIT_MS_CONFIG, 0
+ );
+ try (Consumer consumer = cluster.consumer(config)) {
+ var totalRecords = 10;
+ var startingTimestamp = 0;
+ sendRecords(cluster, tp, totalRecords, startingTimestamp);
+ consumer.assign(List.of(tp));
+ consumeAndVerifyRecords(consumer, tp, totalRecords, 0);
+ // seek to out of range position
+ var outOfRangePos = totalRecords + 1;
+ consumer.seek(tp, outOfRangePos);
+ // assert that poll resets to the beginning position
+ consumeAndVerifyRecords(consumer, tp, 1, 0);
+ }
+ }
+
+ @ClusterTest
+ public void testClassicConsumerFetchOutOfRangeOffsetResetConfigLatest() throws InterruptedException {
+ testFetchOutOfRangeOffsetResetConfigLatest(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerFetchOutOfRangeOffsetResetConfigLatest() throws InterruptedException {
+ testFetchOutOfRangeOffsetResetConfigLatest(GroupProtocol.CONSUMER);
+ }
+
+ private void testFetchOutOfRangeOffsetResetConfigLatest(GroupProtocol groupProtocol) throws InterruptedException {
+ Map config = Map.of(
+ GROUP_PROTOCOL_CONFIG, groupProtocol.name().toLowerCase(Locale.ROOT),
+ AUTO_OFFSET_RESET_CONFIG, "latest",
+ // ensure no in-flight fetch request so that the offset can be reset immediately
+ FETCH_MAX_WAIT_MS_CONFIG, 0
+ );
+ try (Consumer consumer = cluster.consumer(config);
+ Producer producer = cluster.producer()
+ ) {
+ var totalRecords = 10;
+ var startingTimestamp = 0;
+ sendRecords(producer, tp, totalRecords, startingTimestamp);
+ consumer.assign(List.of(tp));
+ consumer.seek(tp, 0);
+
+ // consume some, but not all the records
+ consumeAndVerifyRecords(consumer, tp, totalRecords / 2, 0);
+ // seek to out of range position
+ var outOfRangePos = totalRecords + 17; // arbitrary, much higher offset
+ consumer.seek(tp, outOfRangePos);
+ // assert that poll resets to the ending position
+ assertTrue(consumer.poll(Duration.ofMillis(50)).isEmpty());
+ sendRecords(producer, tp, totalRecords, totalRecords);
+ var nextRecord = consumer.poll(Duration.ofMillis(50)).iterator().next();
+ // ensure the seek went to the last known record at the time of the previous poll
+ assertEquals(totalRecords, nextRecord.offset());
+ }
+ }
+
+ @ClusterTest
+ public void testClassicConsumerFetchOutOfRangeOffsetResetConfigByDuration() throws InterruptedException {
+ testFetchOutOfRangeOffsetResetConfigByDuration(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerFetchOutOfRangeOffsetResetConfigByDuration() throws InterruptedException {
+ testFetchOutOfRangeOffsetResetConfigByDuration(GroupProtocol.CONSUMER);
+ }
+
+ private void testFetchOutOfRangeOffsetResetConfigByDuration(GroupProtocol groupProtocol) throws InterruptedException {
+ Map config = Map.of(
+ GROUP_PROTOCOL_CONFIG, groupProtocol.name().toLowerCase(Locale.ROOT),
+ AUTO_OFFSET_RESET_CONFIG, "by_duration:PT1H",
+ // ensure no in-flight fetch request so that the offset can be reset immediately
+ FETCH_MAX_WAIT_MS_CONFIG, 0
+ );
+ try (Consumer consumer1 = cluster.consumer(config);
+ Consumer consumer2 = cluster.consumer(config)
+ ) {
+ var totalRecords = 10;
+ var startingTimestamp = System.currentTimeMillis();
+ sendRecords(cluster, tp, totalRecords, startingTimestamp);
+ consumer1.assign(List.of(tp));
+ consumeAndVerifyRecords(
+ consumer1,
+ tp,
+ totalRecords,
+ 0,
+ 0,
+ startingTimestamp
+ );
+
+ // seek to out of range position
+ var outOfRangePos = totalRecords + 1;
+ consumer1.seek(tp, outOfRangePos);
+ // assert that poll resets to the beginning position
+ consumeAndVerifyRecords(
+ consumer1,
+ tp,
+ 1,
+ 0,
+ 0,
+ startingTimestamp
+ );
+
+ // Test the scenario where starting offset is earlier than the requested duration
+ var totalRecords2 = 25;
+ startingTimestamp = Instant.now().minus(Duration.ofHours(24)).toEpochMilli();
+
+ // generate records with 1 hour interval for 1 day
+ var hourMillis = Duration.ofHours(1).toMillis();
+ sendRecords(cluster, tp2, totalRecords2, startingTimestamp, hourMillis);
+ consumer2.assign(List.of(tp2));
+ // consumer should read one record from last one hour
+ consumeAndVerifyRecords(
+ consumer2,
+ tp2,
+ 1,
+ 24,
+ 24,
+ startingTimestamp + 24 * hourMillis,
+ hourMillis
+ );
+
+ // seek to out of range position
+ outOfRangePos = totalRecords2 + 1;
+ consumer2.seek(tp2, outOfRangePos);
+ // assert that poll resets to the duration offset. consumer should read one record from last one hour
+ consumeAndVerifyRecords(
+ consumer2,
+ tp2,
+ 1,
+ 24,
+ 24,
+ startingTimestamp + 24 * hourMillis,
+ hourMillis
+ );
+ }
+ }
+
+ @ClusterTest
+ public void testClassicConsumerFetchRecordLargerThanFetchMaxBytes() throws InterruptedException {
+ testFetchRecordLargerThanFetchMaxBytes(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerFetchRecordLargerThanFetchMaxBytes() throws InterruptedException {
+ testFetchRecordLargerThanFetchMaxBytes(GroupProtocol.CONSUMER);
+ }
+
+ private void testFetchRecordLargerThanFetchMaxBytes(GroupProtocol groupProtocol) throws InterruptedException {
+ int maxFetchBytes = 10 * 1024;
+ checkLargeRecord(Map.of(
+ GROUP_PROTOCOL_CONFIG, groupProtocol.name().toLowerCase(Locale.ROOT),
+ FETCH_MAX_BYTES_CONFIG, maxFetchBytes
+ ), maxFetchBytes + 1);
+ }
+
+ @ClusterTest
+ public void testClassicConsumerFetchRecordLargerThanMaxPartitionFetchBytes() throws InterruptedException {
+ testFetchRecordLargerThanMaxPartitionFetchBytes(GroupProtocol.CLASSIC);
+ }
+
+ @ClusterTest
+ public void testAsyncConsumerFetchRecordLargerThanMaxPartitionFetchBytes() throws InterruptedException {
+ testFetchRecordLargerThanMaxPartitionFetchBytes(GroupProtocol.CONSUMER);
+ }
+
+ private void testFetchRecordLargerThanMaxPartitionFetchBytes(GroupProtocol groupProtocol) throws InterruptedException {
+ int maxFetchBytes = 10 * 1024;
+ checkLargeRecord(Map.of(
+ GROUP_PROTOCOL_CONFIG, groupProtocol.name().toLowerCase(Locale.ROOT),
+ MAX_PARTITION_FETCH_BYTES_CONFIG, maxFetchBytes
+ ), maxFetchBytes + 1);
+ }
+
+ private void checkLargeRecord(Map