Skip to content

KAFKA-19719: --no-initial-controllers should not assume kraft.version=1 #20604

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Sep 30, 2025
Merged

KAFKA-19719: --no-initial-controllers should not assume kraft.version=1 #20604

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ec37eb5
KAFKA-19719: --no-initial-controllers should not assume kraft.version…
kevin-wu24 Sep 25, 2025
2dacec3
fixing test
kevin-wu24 Sep 27, 2025
c31566d
Merge branch '4.1' into KAFKA-19719-cherry-pick-41
kevin-wu24 Sep 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 25 additions & 13 deletions core/src/main/scala/kafka/tools/StorageTool.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,21 +135,30 @@ object StorageTool extends Logging {
featureNamesAndLevels(_).foreachEntry {
(k, v) => formatter.setFeatureLevel(k, v)
})
Option(namespace.getString("initial_controllers")).
val initialControllers = namespace.getString("initial_controllers")
val isStandalone = namespace.getBoolean("standalone")
val staticVotersEmpty = config.quorumConfig.voters().isEmpty
formatter.setHasDynamicQuorum(staticVotersEmpty)
if (!staticVotersEmpty && (Option(initialControllers).isDefined || isStandalone)) {
throw new TerseFailure("You cannot specify " +
QuorumConfig.QUORUM_VOTERS_CONFIG + " and format the node " +
"with --initial-controllers or --standalone. " +
"If you want to use dynamic quorum, please remove " +
QuorumConfig.QUORUM_VOTERS_CONFIG + " and specify " +
QuorumConfig.QUORUM_BOOTSTRAP_SERVERS_CONFIG + " instead.")
}
Option(initialControllers).
foreach(v => formatter.setInitialControllers(DynamicVoters.parse(v)))
if (namespace.getBoolean("standalone")) {
if (isStandalone) {
formatter.setInitialControllers(createStandaloneDynamicVoters(config))
}
if (namespace.getBoolean("no_initial_controllers")) {
formatter.setNoInitialControllersFlag(true)
} else {
if (config.processRoles.contains(ProcessRole.ControllerRole)) {
if (config.quorumConfig.voters().isEmpty && formatter.initialVoters().isEmpty) {
if (!namespace.getBoolean("no_initial_controllers") &&
config.processRoles.contains(ProcessRole.ControllerRole) &&
staticVotersEmpty &&
formatter.initialVoters().isEmpty) {
throw new TerseFailure("Because " + QuorumConfig.QUORUM_VOTERS_CONFIG +
" is not set on this controller, you must specify one of the following: " +
"--standalone, --initial-controllers, or --no-initial-controllers.");
}
}
}
Option(namespace.getList("add_scram")).
foreach(scramArgs => formatter.setScramArguments(scramArgs.asInstanceOf[util.List[String]]))
Expand Down Expand Up @@ -319,18 +328,21 @@ object StorageTool extends Logging {

val reconfigurableQuorumOptions = formatParser.addMutuallyExclusiveGroup()
reconfigurableQuorumOptions.addArgument("--standalone", "-s")
.help("Used to initialize a controller as a single-node dynamic quorum.")
.help("Used to initialize a controller as a single-node dynamic quorum. When setting this flag, " +
"the controller.quorum.voters config must not be set, and controller.quorum.bootstrap.servers is set instead.")
.action(storeTrue())

reconfigurableQuorumOptions.addArgument("--no-initial-controllers", "-N")
.help("Used to initialize a server without a dynamic quorum topology.")
.help("Used to initialize a server without specifying a dynamic quorum. When setting this flag, " +
"the controller.quorum.voters config should not be set, and controller.quorum.bootstrap.servers is set instead.")
.action(storeTrue())

reconfigurableQuorumOptions.addArgument("--initial-controllers", "-I")
.help("Used to initialize a server with a specific dynamic quorum topology. The argument " +
.help("Used to initialize a server with the specified dynamic quorum. The argument " +
"is a comma-separated list of id@hostname:port:directory. The same values must be used to " +
"format all nodes. For example:\[email protected]:8082:JEXY6aqzQY-32P5TStzaFg,[email protected]:8083:" +
"MvDxzVmcRsaTz33bUuRU6A,[email protected]:8084:07R5amHmR32VDA6jHkGbTA\n")
"MvDxzVmcRsaTz33bUuRU6A,[email protected]:8084:07R5amHmR32VDA6jHkGbTA\n. When setting this flag, " +
"the controller.quorum.voters config must not be set, and controller.quorum.bootstrap.servers is set instead.")
.action(store())
}

Expand Down
50 changes: 35 additions & 15 deletions core/src/test/java/kafka/server/ReconfigurableQuorumIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

import org.junit.jupiter.api.Test;

import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -83,9 +84,8 @@ public void testCreateAndDestroyReconfigurableCluster() throws Exception {
new TestKitNodes.Builder().
setNumBrokerNodes(1).
setNumControllerNodes(1).
setFeature(KRaftVersion.FEATURE_NAME, KRaftVersion.KRAFT_VERSION_1.featureLevel()).
build()
).build()) {
).setStandalone(true).build()) {
cluster.format();
cluster.startup();
try (Admin admin = Admin.create(cluster.clientProperties())) {
Expand All @@ -107,13 +107,23 @@ static Map<Integer, Uuid> findVoterDirs(Admin admin) throws Exception {

@Test
public void testRemoveController() throws Exception {
try (KafkaClusterTestKit cluster = new KafkaClusterTestKit.Builder(
new TestKitNodes.Builder().
setNumBrokerNodes(1).
setNumControllerNodes(3).
setFeature(KRaftVersion.FEATURE_NAME, KRaftVersion.KRAFT_VERSION_1.featureLevel()).
build()
).build()) {
final var nodes = new TestKitNodes.Builder().
setNumBrokerNodes(1).
setNumControllerNodes(3).
build();

final Map<Integer, Uuid> initialVoters = new HashMap<>();
for (final var controllerNode : nodes.controllerNodes().values()) {
initialVoters.put(
controllerNode.id(),
controllerNode.metadataDirectoryId()
);
}

try (KafkaClusterTestKit cluster = new KafkaClusterTestKit.Builder(nodes).
setInitialVoterSet(initialVoters).
build()
) {
cluster.format();
cluster.startup();
try (Admin admin = Admin.create(cluster.clientProperties())) {
Expand All @@ -132,12 +142,22 @@ public void testRemoveController() throws Exception {

@Test
public void testRemoveAndAddSameController() throws Exception {
try (KafkaClusterTestKit cluster = new KafkaClusterTestKit.Builder(
new TestKitNodes.Builder().
setNumBrokerNodes(1).
setNumControllerNodes(4).
setFeature(KRaftVersion.FEATURE_NAME, KRaftVersion.KRAFT_VERSION_1.featureLevel()).
build()).build()
final var nodes = new TestKitNodes.Builder().
setNumBrokerNodes(1).
setNumControllerNodes(4).
build();

final Map<Integer, Uuid> initialVoters = new HashMap<>();
for (final var controllerNode : nodes.controllerNodes().values()) {
initialVoters.put(
controllerNode.id(),
controllerNode.metadataDirectoryId()
);
}

try (KafkaClusterTestKit cluster = new KafkaClusterTestKit.Builder(nodes).
setInitialVoterSet(initialVoters).
build()
) {
cluster.format();
cluster.startup();
Expand Down
3 changes: 1 addition & 2 deletions core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1011,8 +1011,7 @@ class KRaftClusterTest {
val cluster = new KafkaClusterTestKit.Builder(
new TestKitNodes.Builder().
setNumBrokerNodes(1).
setNumControllerNodes(1).
setFeature(KRaftVersion.FEATURE_NAME, 1.toShort).build()).build()
setNumControllerNodes(1).build()).setStandalone(true).build()
try {
cluster.format()
cluster.startup()
Expand Down
14 changes: 6 additions & 8 deletions core/src/test/scala/unit/kafka/tools/StorageToolTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,10 @@ Found problem:
def testFormatWithStandaloneFlagOnBrokerFails(): Unit = {
val availableDirs = Seq(TestUtils.tempDir())
val properties = new Properties()
properties.putAll(defaultStaticQuorumProperties)
properties.setProperty("process.roles", "broker")
properties.setProperty("node.id", "0")
properties.setProperty("controller.listener.names", "CONTROLLER")
properties.setProperty("controller.quorum.bootstrap.servers", "localhost:9093")
properties.setProperty("log.dirs", availableDirs.mkString(","))
val stream = new ByteArrayOutputStream()
val arguments = ListBuffer[String]("--release-version", "3.9-IV0", "--standalone")
Expand Down Expand Up @@ -458,19 +461,14 @@ Found problem:
Seq("--release-version", "3.9-IV0"))).getMessage)
}

@ParameterizedTest
@ValueSource(booleans = Array(false, true))
def testFormatWithNoInitialControllersSucceedsOnController(setKraftVersionFeature: Boolean): Unit = {
@Test
def testFormatWithNoInitialControllersSucceedsOnController(): Unit = {
val availableDirs = Seq(TestUtils.tempDir())
val properties = new Properties()
properties.putAll(defaultDynamicQuorumProperties)
properties.setProperty("log.dirs", availableDirs.mkString(","))
val stream = new ByteArrayOutputStream()
val arguments = ListBuffer[String]("--release-version", "3.9-IV0", "--no-initial-controllers")
if (setKraftVersionFeature) {
arguments += "--feature"
arguments += "kraft.version=1"
}
assertEquals(0, runFormatCommand(stream, properties, arguments.toSeq))
assertTrue(stream.toString().
contains("Formatting metadata directory %s".format(availableDirs.head)),
Expand Down
60 changes: 21 additions & 39 deletions docs/ops.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4072,45 +4072,27 @@ <h5 class="anchor-heading"><a id="static_versus_dynamic_kraft_quorums" class="an
If you are not sure whether you are using static or dynamic quorums, you can determine this by
running something like the following:<p>

<pre><code class="language-bash">
$ bin/kafka-features.sh --bootstrap-controller localhost:9093 describe
</code></pre><p>

If the <code>kraft.version</code> field is level 0 or absent, you are using a static quorum. If
it is 1 or above, you are using a dynamic quorum. For example, here is an example of a static
quorum:<p/>
<pre><code class="language-bash">
Feature: kraft.version SupportedMinVersion: 0 SupportedMaxVersion: 1 FinalizedVersionLevel: 0 Epoch: 5
Feature: metadata.version SupportedMinVersion: 3.3-IV3 SupportedMaxVersion: 3.9-IV0 FinalizedVersionLevel: 3.9-IV0 Epoch: 5
</code></pre><p/>

Here is another example of a static quorum:<p/>
<pre><code class="language-bash">
Feature: metadata.version SupportedMinVersion: 3.3-IV3 SupportedMaxVersion: 3.8-IV0 FinalizedVersionLevel: 3.8-IV0 Epoch: 5
</code></pre><p/>

Here is an example of a dynamic quorum:<p/>
<pre><code class="language-bash">
Feature: kraft.version SupportedMinVersion: 0 SupportedMaxVersion: 1 FinalizedVersionLevel: 1 Epoch: 5
Feature: metadata.version SupportedMinVersion: 3.3-IV3 SupportedMaxVersion: 3.9-IV0 FinalizedVersionLevel: 3.9-IV0 Epoch: 5
</code></pre><p/>

The static versus dynamic nature of the quorum is determined at the time of formatting.
Specifically, the quorum will be formatted as dynamic if <code>controller.quorum.voters</code> is
<b>not</b> present, and if the software version is Apache Kafka 3.9 or newer. If you have
followed the instructions earlier in this document, you will get a dynamic quorum.<p>

If you would like the formatting process to fail if a dynamic quorum cannot be achieved, format your
controllers using the <code>--feature kraft.version=1</code>. (Note that you should not supply
this flag when formatting brokers -- only when formatting controllers.)<p>

<pre><code class="language-bash">
$ bin/kafka-storage.sh format -t KAFKA_CLUSTER_ID --feature kraft.version=1 -c controller.properties
Cannot set kraft.version to 1 unless KIP-853 configuration is present. Try removing the --feature flag for kraft.version.
</code></pre><p>

Note: Currently it is <b>not</b> possible to convert clusters using a static controller quorum to
use a dynamic controller quorum. This function will be supported in the future release.
<pre><code class="language-bash">$ bin/kafka-features.sh --bootstrap-controller localhost:9093 describe</code></pre>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I’ve cherry-picked #20532 and #20401 to 4.1, so backporting should be easier now

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, I fixed the conflict in this file.

<p>
If the <code>kraft.version</code> field is level 0 or absent, you are using a static quorum. If
it is 1 or above, you are using a dynamic quorum. For example, here is an example of a static
quorum:<p>
<pre><code class="language-bash">Feature: kraft.version SupportedMinVersion: 0 SupportedMaxVersion: 1 FinalizedVersionLevel: 0 Epoch: 5
Feature: metadata.version SupportedMinVersion: 3.3-IV3 SupportedMaxVersion: 3.9-IV0 FinalizedVersionLevel: 3.9-IV0 Epoch: 5</code></pre>
<p>
Here is another example of a static quorum:<p>
<pre><code class="language-bash">Feature: metadata.version SupportedMinVersion: 3.3-IV3 SupportedMaxVersion: 3.8-IV0 FinalizedVersionLevel: 3.8-IV0 Epoch: 5</code></pre>
<p>
Here is an example of a dynamic quorum:<p>
<pre><code class="language-bash">Feature: kraft.version SupportedMinVersion: 0 SupportedMaxVersion: 1 FinalizedVersionLevel: 1 Epoch: 5
Feature: metadata.version SupportedMinVersion: 3.3-IV3 SupportedMaxVersion: 3.9-IV0 FinalizedVersionLevel: 3.9-IV0 Epoch: 5</code></pre>
<p>
The static versus dynamic nature of the quorum is determined at the time of formatting.
Specifically, the quorum will be formatted as dynamic if <code>controller.quorum.voters</code> is
<b>not</b> present, and one of --standalone, --initial-controllers, or --no-initial-controllers is set.
If you have followed the instructions earlier in this document, you will get a dynamic quorum.
<p>
Note: To migrate from static voter set to dynamic voter set, please refer to the <a href="#kraft_upgrade">Upgrade</a> section.

<h5 class="anchor-heading"><a id="kraft_reconfig_add" class="anchor-link"></a><a href="#kraft_reconfig_add">Add New Controller</a></h5>
If a dynamic controller cluster already exists, it can be expanded by first provisioning a new controller using the <a href="#kraft_nodes_observers">kafka-storage.sh tool</a> and starting the controller.
Expand Down
29 changes: 14 additions & 15 deletions metadata/src/main/java/org/apache/kafka/metadata/storage/Formatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ public class Formatter {
* The initial KIP-853 voters.
*/
private Optional<DynamicVoters> initialControllers = Optional.empty();
private boolean noInitialControllersFlag = false;
private boolean hasDynamicQuorum = false;

public Formatter setPrintStream(PrintStream printStream) {
this.printStream = printStream;
Expand Down Expand Up @@ -217,8 +217,8 @@ public Formatter setInitialControllers(DynamicVoters initialControllers) {
return this;
}

public Formatter setNoInitialControllersFlag(boolean noInitialControllersFlag) {
this.noInitialControllersFlag = noInitialControllersFlag;
public Formatter setHasDynamicQuorum(boolean hasDynamicQuorum) {
this.hasDynamicQuorum = hasDynamicQuorum;
return this;
}

Expand All @@ -227,7 +227,7 @@ public Optional<DynamicVoters> initialVoters() {
}

boolean hasDynamicQuorum() {
return initialControllers.isPresent() || noInitialControllersFlag;
return hasDynamicQuorum;
}

public BootstrapMetadata bootstrapMetadata() {
Expand Down Expand Up @@ -337,8 +337,8 @@ Map<String, Short> calculateEffectiveFeatureLevels() {
/**
* Calculate the effective feature level for kraft.version. In order to keep existing
* command-line invocations of StorageTool working, we default this to 0 if no dynamic
* voter quorum arguments were provided. As a convenience, if dynamic voter quorum arguments
* were passed, we set the latest kraft.version. (Currently there is only 1 non-zero version).
* voter quorum arguments were provided. As a convenience, if the static voters config is
* empty, we set the latest kraft.version. (Currently there is only 1 non-zero version).
*
* @param configuredKRaftVersionLevel The configured level for kraft.version
* @return The effective feature level.
Expand All @@ -348,20 +348,19 @@ private short effectiveKRaftFeatureLevel(Optional<Short> configuredKRaftVersionL
if (configuredKRaftVersionLevel.get() == 0) {
if (hasDynamicQuorum()) {
throw new FormatterException(
"Cannot set kraft.version to " +
configuredKRaftVersionLevel.get() +
" if one of the flags --standalone, --initial-controllers, or --no-initial-controllers is used. " +
"For dynamic controllers support, try removing the --feature flag for kraft.version."
"Cannot set kraft.version to 0 if controller.quorum.voters is empty and one of the flags " +
"--standalone, --initial-controllers, or --no-initial-controllers is used. For dynamic " +
"controllers support, try removing the --feature flag for kraft.version."
);
}
} else {
if (!hasDynamicQuorum()) {
throw new FormatterException(
"Cannot set kraft.version to " +
configuredKRaftVersionLevel.get() +
" unless one of the flags --standalone, --initial-controllers, or --no-initial-controllers is used. " +
"For dynamic controllers support, try using one of --standalone, --initial-controllers, or " +
"--no-initial-controllers."
"Cannot set kraft.version to " + configuredKRaftVersionLevel.get() +
" unless controller.quorum.voters is empty and one of the flags --standalone, " +
"--initial-controllers, or --no-initial-controllers is used. " +
"For dynamic controllers support, try using one of --standalone, --initial-controllers, " +
"or --no-initial-controllers and removing controller.quorum.voters."
);
}
}
Expand Down
Loading
Loading