Skip to content

Commit 6460715

Browse files
kenneth-jiakenneth-jia
committed
Tests don't rely on auto.create.topics.enable=true from brokers
1 parent 0762ec2 commit 6460715

File tree

6 files changed

+68
-13
lines changed

6 files changed

+68
-13
lines changed

‎scripts/start-local-kafka-cluster.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ def GenerateBrokerConfig(brokerId, brokerPort, zookeeperPort, logDir):
9090
offsets.commit.timeout.ms=10000
9191
unclean.leader.election.enable=false
9292
min.insync.replicas=2
93+
auto.create.topics.enable=false
9394
''')
9495
properties = brokerTemplate.substitute(broker_id=brokerId, listener_port=brokerPort, zookeeper_port=zookeeperPort, log_dir=logDir)
9596
return properties

‎tests/integration/TestKafkaConsumer.cc‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,8 @@ TEST(KafkaManualCommitConsumer, NoOffsetCommitCallback)
379379

380380
std::cout << "[" << Utility::getCurrentTime() << "] Topic[" << topic << "] would be used" << std::endl;
381381

382+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
383+
382384
// Prepare some messages to send
383385
const std::vector<std::tuple<Headers, std::string, std::string>> messages = {
384386
{Headers{}, "key1", "value1"},
@@ -423,6 +425,8 @@ TEST(KafkaManualCommitConsumer, OffsetCommitCallback)
423425

424426
std::cout << "[" << Utility::getCurrentTime() << "] Topic[" << topic << "] would be used" << std::endl;
425427

428+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
429+
426430
// Prepare some messages to send
427431
const std::vector<std::tuple<Headers, std::string, std::string>> messages = {
428432
{Headers{}, "key1", "value1"},
@@ -485,6 +489,8 @@ TEST(KafkaManualCommitConsumer, OffsetCommitCallbackTriggeredBeforeClose)
485489

486490
std::cout << "[" << Utility::getCurrentTime() << "] Topic[" << topic << "] would be used" << std::endl;
487491

492+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
493+
488494
// Prepare some messages to send
489495
const std::vector<std::tuple<Headers, std::string, std::string>> messages = {
490496
{Headers{}, "key1", "value1"},
@@ -544,6 +550,8 @@ TEST(KafkaManualCommitConsumer, OffsetCommitCallback_ManuallyPollEvents)
544550

545551
std::cout << "[" << Utility::getCurrentTime() << "] Topic[" << topic << "] would be used" << std::endl;
546552

553+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
554+
547555
// Prepare some messages to send
548556
const std::vector<std::tuple<Headers, std::string, std::string>> messages = {
549557
{Headers{}, "key1", "value1"},
@@ -610,6 +618,8 @@ TEST(KafkaManualCommitConsumer, OffsetCommitAndPosition)
610618

611619
std::cout << "[" << Utility::getCurrentTime() << "] Topic[" << topic << "] would be used" << std::endl;
612620

621+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
622+
613623
// Prepare some messages to send
614624
const std::vector<std::tuple<Headers, std::string, std::string>> messages = {
615625
{Headers{}, "key1", "value1"},
@@ -854,6 +864,9 @@ TEST(KafkaAutoCommitConsumer, OffsetCommitAndPosition)
854864
const Partition partition = 0;
855865

856866
std::cout << "[" << Utility::getCurrentTime() << "] Topic[" << topic << "] would be used" << std::endl;
867+
868+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
869+
857870
// Prepare some messages to send
858871
std::vector<std::tuple<Headers, std::string, std::string>> messages = {
859872
{Headers{}, "key1", "value1"},
@@ -1234,7 +1247,6 @@ TEST(KafkaAutoCommitConsumer, WrongOperation_AssignThenSubscribe)
12341247
TEST(KafkaClient, GetBrokerMetadata)
12351248
{
12361249
const Topic topic = Utility::getRandomString();
1237-
12381250
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
12391251

12401252
// Start consumer
@@ -1269,7 +1281,6 @@ TEST(KafkaClient, GetBrokerMetadata)
12691281
TEST(KafkaAutoCommitConsumer, SubscribeAndPoll)
12701282
{
12711283
const Topic topic = Utility::getRandomString();
1272-
12731284
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
12741285

12751286
const auto props = KafkaTestUtility::GetKafkaClientCommonConfig().put(ConsumerConfig::ENABLE_PARTITION_EOF, "true");
@@ -1464,6 +1475,9 @@ TEST(KafkaManualCommitConsumer, OffsetsForTime)
14641475
const Topic topic2 = Utility::getRandomString();
14651476
const Partition partition2 = 1;
14661477

1478+
KafkaTestUtility::CreateKafkaTopic(topic1, 5, 3);
1479+
KafkaTestUtility::CreateKafkaTopic(topic2, 5, 3);
1480+
14671481
using namespace std::chrono;
14681482

14691483
constexpr int MESSAGES_NUM = 5;
@@ -1574,6 +1588,8 @@ TEST(KafkaManualCommitConsumer, RecoverByTime)
15741588
const Topic topic = Utility::getRandomString();
15751589
const Partition partition = 0;
15761590

1591+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
1592+
15771593
// Prepare some messages to send
15781594
const std::vector<std::pair<std::string, std::string>> messages = {
15791595
{"key1", "value1"},
@@ -1680,6 +1696,7 @@ TEST(KafkaManualCommitConsumer, RecoverByTime)
16801696
TEST(KafkaAutoCommitConsumer, AutoCreateTopics)
16811697
{
16821698
const Topic topic = Utility::getRandomString();
1699+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
16831700

16841701
KafkaAutoCommitConsumer consumer(KafkaTestUtility::GetKafkaClientCommonConfig()
16851702
.put("allow.auto.create.topics", "true"));

‎tests/integration/TestKafkaProducer.cc‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ TEST(KafkaSyncProducer, SendMessagesWithAcks1)
2323
const Topic topic = Utility::getRandomString();
2424
const Partition partition = 0;
2525

26+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
27+
2628
// Properties for the producer
2729
const auto props = KafkaTestUtility::GetKafkaClientCommonConfig().put(ProducerConfig::ACKS, "1");
2830

@@ -67,6 +69,8 @@ TEST(KafkaSyncProducer, SendMessagesWithAcksAll)
6769
const Topic topic = Utility::getRandomString();
6870
const Partition partition = 0;
6971

72+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
73+
7074
// Properties for the producer
7175
const auto props = KafkaTestUtility::GetKafkaClientCommonConfig().put(ProducerConfig::ACKS, "all");
7276

@@ -143,6 +147,8 @@ TEST(KafkaSyncProducer, InSyncBrokersAckTimeout)
143147
const Topic topic = Utility::getRandomString();
144148
const Partition partition = 0;
145149

150+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
151+
146152
const auto key = std::string(100000, 'a');
147153
const auto value = std::string(100000, 'a');
148154
const auto record = ProducerRecord(topic, partition, Key(key.c_str(), key.size()), Value(value.c_str(), value.size()));
@@ -178,6 +184,7 @@ TEST(KafkaSyncProducer, DefaultPartitioner)
178184
KafkaSyncProducer producer(KafkaTestUtility::GetKafkaClientCommonConfig());
179185

180186
const Topic topic = Utility::getRandomString();
187+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
181188

182189
std::map<Partition, int> partitionCounts;
183190
constexpr int MSG_NUM = 20;
@@ -199,6 +206,9 @@ TEST(KafkaSyncProducer, DefaultPartitioner)
199206

200207
TEST(KafkaSyncProducer, TryOtherPartitioners)
201208
{
209+
const Topic topic = Utility::getRandomString();
210+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
211+
202212
// Try another "partitioner" instead of the default one
203213
{
204214
auto props = KafkaTestUtility::GetKafkaClientCommonConfig();
@@ -214,7 +224,7 @@ TEST(KafkaSyncProducer, TryOtherPartitioners)
214224
std::string key;
215225
std::string value = "v" + std::to_string(i);
216226

217-
auto record = ProducerRecord(Utility::getRandomString(), Key(key.c_str(), key.size()), Value(value.c_str(), value.size()));
227+
auto record = ProducerRecord(topic, Key(key.c_str(), key.size()), Value(value.c_str(), value.size()));
218228

219229
auto metadata = producer.send(record);
220230
std::cout << metadata.toString() << std::endl;
@@ -265,6 +275,8 @@ TEST(KafkaAsyncProducer, MessageDeliveryCallback)
265275
const Topic topic = Utility::getRandomString();
266276
const Partition partition = 0;
267277

278+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
279+
268280
// Properties for the producer
269281
std::set<ProducerRecord::Id> msgIdsSent;
270282

@@ -313,6 +325,8 @@ TEST(KafkaAsyncProducer, DeliveryCallback_ManuallyPollEvents)
313325
const Topic topic = Utility::getRandomString();
314326
const Partition partition = 0;
315327

328+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
329+
316330
// Properties for the producer
317331
std::set<ProducerRecord::Id> msgIdsSent;
318332

@@ -364,6 +378,8 @@ TEST(KafkaAsyncProducer, NoBlockSendingWhileQueueIsFull_ManuallyPollEvents)
364378
const Topic topic = Utility::getRandomString();
365379
const auto appThreadId = std::this_thread::get_id();
366380

381+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
382+
367383
int msgSentCnt = 0;
368384

369385
Producer::Callback drCallback =
@@ -435,6 +451,8 @@ TEST(KafkaAsyncProducer, TooLargeMessageForBroker)
435451
const Topic topic = Utility::getRandomString();
436452
const Partition partition = 0;
437453

454+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
455+
438456
const auto value = std::string(2048, 'a');
439457
const auto record = ProducerRecord(topic, partition, Key(nullptr, 0), Value(value.c_str(), value.size()));
440458

@@ -466,6 +484,7 @@ TEST(KafkaAsyncProducer, TooLargeMessageForBroker)
466484
TEST(KafkaAsyncProducer, CopyRecordValueWithinSend)
467485
{
468486
const Topic topic = Utility::getRandomString();
487+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
469488

470489
const auto props = KafkaTestUtility::GetKafkaClientCommonConfig()
471490
.put(ProducerConfig::PARTITIONER, "murmur2"); // `ProducerRecord`s with empty key are mapped to a single partition

‎tests/integration/TestTransaction.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,8 @@ TEST(Transaction, ContinueTheTransaction)
244244
const std::string transactionId = Utility::getRandomString();
245245
const std::string messageToSent = "message to sent";
246246

247+
KafkaTestUtility::CreateKafkaTopic(topic, 1, 3);
248+
247249
// Start a producer to send the message, but fail to commit
248250
{
249251
KafkaAsyncProducer producer(KafkaTestUtility::GetKafkaClientCommonConfig()

‎tests/robustness/TestKafkaConsumer.cc‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ TEST(KafkaManualCommitConsumer, AlwaysFinishClosing_ManuallyPollEvents)
2222
Topic topic = Utility::getRandomString();
2323
Partition partition = 0;
2424

25+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
26+
2527
// Producer some messages
2628
std::vector<std::tuple<Headers, std::string, std::string>> messages = {
2729
{Headers{}, "key1", "value1"},
@@ -86,6 +88,8 @@ TEST(KafkaManualCommitConsumer, CommitOffsetWhileBrokersStop)
8688
const Topic topic = Utility::getRandomString();
8789
const Partition partition = 0;
8890

91+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
92+
8993
// Producer some messages
9094
std::vector<std::tuple<Headers, std::string, std::string>> messages = {
9195
{Headers{}, "key1", "value1"}
@@ -149,7 +153,6 @@ TEST(KafkaManualCommitConsumer, CommitOffsetWhileBrokersStop)
149153
TEST(KafkaAutoCommitConsumer, BrokerStopBeforeConsumerStart)
150154
{
151155
const Topic topic = Utility::getRandomString();
152-
153156
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
154157

155158
// Pause the brokers for a while
@@ -212,7 +215,6 @@ TEST(KafkaAutoCommitConsumer, BrokerStopBeforeSubscription)
212215
auto asyncTask = KafkaTestUtility::PauseBrokersForAWhile(std::chrono::seconds(5));
213216

214217
const Topic topic = Utility::getRandomString();
215-
216218
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
217219

218220
TopicPartitions assignment;
@@ -250,7 +252,6 @@ TEST(KafkaAutoCommitConsumer, BrokerStopBeforeSubscription)
250252
TEST(KafkaAutoCommitConsumer, BrokerStopBeforeSeek)
251253
{
252254
const Topic topic = Utility::getRandomString();
253-
254255
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
255256

256257
// Consumer properties
@@ -298,13 +299,14 @@ TEST(KafkaAutoCommitConsumer, BrokerStopBeforeSeek)
298299
auto records = KafkaTestUtility::ConsumeMessagesUntilTimeout(consumer, std::chrono::seconds(10));
299300
EXPECT_FALSE(records.empty());
300301
ASSERT_TRUE(std::all_of(records.cbegin(), records.cend(), [](const auto& record){ return record.error().value() == RD_KAFKA_RESP_ERR__PARTITION_EOF; }));
301-
302+
302303
std::cout << "[" << Utility::getCurrentTime() << "] " << consumer.name() << " polled " << records.size() << " EOFs" << std::endl;
303304
}
304305

305306
TEST(KafkaAutoCommitConsumer, BrokerStopDuringMsgPoll)
306307
{
307308
const Topic topic = Utility::getRandomString();
309+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
308310

309311
// Prepare messages to test
310312
const std::vector<std::tuple<Headers, std::string, std::string>> messages = {
@@ -347,3 +349,4 @@ TEST(KafkaAutoCommitConsumer, BrokerStopDuringMsgPoll)
347349

348350
std::cout << "[" << Utility::getCurrentTime() << "] " << consumer.name() << " polled " << records.size() << " messages" << std::endl;
349351
}
352+

‎tests/robustness/TestKafkaProducer.cc‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ TEST(KafkaSyncProducer, RecordTimestamp)
100100
TEST(KafkaAsyncProducer, NoMissedDeliveryCallback)
101101
{
102102
const Topic topic = Utility::getRandomString();
103+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
103104

104105
std::mutex inFlightMutex;
105106
std::set<ProducerRecord::Id> inFlightIds;
@@ -151,6 +152,7 @@ TEST(KafkaAsyncProducer, NoMissedDeliveryCallback)
151152
TEST(KafkaAsyncProducer, MightMissDeliveryCallbackIfCloseWithLimitedTimeout)
152153
{
153154
const Topic topic = Utility::getRandomString();
155+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
154156

155157
std::size_t deliveryCount = 0;
156158
{
@@ -195,6 +197,7 @@ TEST(KafkaAsyncProducer, BrokerStopWhileSendingMessages)
195197
};
196198

197199
const Topic topic = Utility::getRandomString();
200+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
198201

199202
std::size_t deliveryCount = 0;
200203
{
@@ -241,6 +244,8 @@ TEST(KafkaAsyncProducer, Send_AckTimeout)
241244
};
242245

243246
const Topic topic = Utility::getRandomString();
247+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
248+
244249
{
245250
KafkaAsyncProducer producer(KafkaTestUtility::GetKafkaClientCommonConfig()
246251
.put(ProducerConfig::MESSAGE_TIMEOUT_MS, "3000")); // If with no response, the delivery would fail in a short time
@@ -252,7 +257,7 @@ TEST(KafkaAsyncProducer, Send_AckTimeout)
252257
for (const auto& msg: messages)
253258
{
254259
auto record = ProducerRecord(topic, Key(msg.first.c_str(), msg.first.size()), Value(msg.second.c_str(), msg.second.size()));
255-
260+
256261
producer.send(record, [&failureCount](const Producer::RecordMetadata& metadata, std::error_code ec) {
257262
std::cout << "[" << Utility::getCurrentTime() << "] delivery callback: result[" << ec.message() << "], metadata[" << metadata.toString() << "]" << std::endl;
258263
EXPECT_EQ(RD_KAFKA_RESP_ERR__MSG_TIMED_OUT, ec.value());
@@ -276,6 +281,8 @@ TEST(KafkaAsyncProducer, ManuallyPollEvents_AckTimeout)
276281
};
277282

278283
const Topic topic = Utility::getRandomString();
284+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
285+
279286
{
280287
KafkaAsyncProducer producer(KafkaTestUtility::GetKafkaClientCommonConfig()
281288
.put(ProducerConfig::MESSAGE_TIMEOUT_MS, "3000"), // If with no response, the delivery would fail in a short time
@@ -288,7 +295,7 @@ TEST(KafkaAsyncProducer, ManuallyPollEvents_AckTimeout)
288295
for (const auto& msg: messages)
289296
{
290297
auto record = ProducerRecord(topic, Key(msg.first.c_str(), msg.first.size()), Value(msg.second.c_str(), msg.second.size()));
291-
298+
292299
producer.send(record, [&failureCount](const Producer::RecordMetadata& metadata, std::error_code ec) {
293300
std::cout << "[" << Utility::getCurrentTime() << "] delivery callback: result[" << ec.message() << "], metadata[" << metadata.toString() << "]" << std::endl;
294301
EXPECT_EQ(RD_KAFKA_RESP_ERR__MSG_TIMED_OUT, ec.value());
@@ -299,7 +306,7 @@ TEST(KafkaAsyncProducer, ManuallyPollEvents_AckTimeout)
299306

300307
const auto timeout = std::chrono::seconds(10);
301308
const auto interval = std::chrono::milliseconds(100);
302-
309+
303310
for (const auto end = std::chrono::steady_clock::now() + timeout; std::chrono::steady_clock::now() < end;)
304311
{
305312
// Keep polling for the delivery-callbacks
@@ -318,6 +325,9 @@ TEST(KafkaAsyncProducer, ManuallyPollEvents_AlwaysFinishClosing)
318325
{"3", "value3"},
319326
};
320327

328+
const Topic topic = Utility::getRandomString();
329+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
330+
321331
std::size_t failureCount = 0;
322332
{
323333
KafkaAsyncProducer producer(KafkaTestUtility::GetKafkaClientCommonConfig()
@@ -330,8 +340,8 @@ TEST(KafkaAsyncProducer, ManuallyPollEvents_AlwaysFinishClosing)
330340
const auto appThreadId = std::this_thread::get_id();
331341
for (const auto& msg: messages)
332342
{
333-
auto record = ProducerRecord(Utility::getRandomString(), Key(msg.first.c_str(), msg.first.size()), Value(msg.second.c_str(), msg.second.size()));
334-
343+
auto record = ProducerRecord(topic, Key(msg.first.c_str(), msg.first.size()), Value(msg.second.c_str(), msg.second.size()));
344+
335345
producer.send(record, [&failureCount, appThreadId](const Producer::RecordMetadata& metadata, std::error_code ec) {
336346
std::cout << "[" << Utility::getCurrentTime() << "] delivery callback: result[" << ec.message() << "], metadata[" << metadata.toString() << "]" << std::endl;
337347
EXPECT_EQ(RD_KAFKA_RESP_ERR__MSG_TIMED_OUT, ec.value());
@@ -348,12 +358,15 @@ TEST(KafkaAsyncProducer, ManuallyPollEvents_AlwaysFinishClosing)
348358

349359
TEST(KafkaSyncProducer, Send_AckTimeout)
350360
{
361+
const Topic topic = Utility::getRandomString();
362+
KafkaTestUtility::CreateKafkaTopic(topic, 5, 3);
363+
351364
KafkaSyncProducer producer(KafkaTestUtility::GetKafkaClientCommonConfig().put(ProducerConfig::MESSAGE_TIMEOUT_MS, "3000"));
352365

353366
// Pause the brokers for a while
354367
auto asyncTask = KafkaTestUtility::PauseBrokersForAWhile(std::chrono::seconds(5));
355368

356-
auto record = ProducerRecord(Utility::getRandomString(), NullKey, NullValue);
369+
auto record = ProducerRecord(topic, NullKey, NullValue);
357370
std::cout << "[" << Utility::getCurrentTime() << "] About to send record: " << record.toString() << std::endl;
358371

359372
EXPECT_KAFKA_THROW(producer.send(record), RD_KAFKA_RESP_ERR__MSG_TIMED_OUT);

0 commit comments

Comments
 (0)