A couple of consumer integration tests (#1881)

* A couple of consumer integration tests

* Changes following review feedback

Author: Matt Howlett

test/Confluent.Kafka.IntegrationTests/TemporaryTopic.cs

@@ -28,9 +28,12 @@ public class TemporaryTopic : IDisposable
         public string Name { get; set; }
 
         public TemporaryTopic(string bootstrapServers, int numPartitions)
+            : this("dotnet_test_", bootstrapServers, numPartitions) {}
+
+        public TemporaryTopic(string prefix, string bootstrapServers, int numPartitions)
         {
             this.bootstrapServers = bootstrapServers;
-            this.Name = "dotnet_test_" + Guid.NewGuid().ToString();
+            this.Name = prefix + Guid.NewGuid().ToString();
 
             var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers }).Build();
             adminClient.CreateTopicsAsync(new List<TopicSpecification> {

test/Confluent.Kafka.IntegrationTests/Tests/Consumer_Subscription_DisjointTopics.cs

@@ -0,0 +1,147 @@
+// Copyright 2022 Confluent Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Refer to LICENSE for more information.
+
+#pragma warning disable xUnit1026
+
+using System;
+using System.Collections.Generic;
+using Xunit;
+
+
+namespace Confluent.Kafka.IntegrationTests
+{
+    public partial class Tests
+    {
+        private void DisjointTopicsSubscribeTest(String bootstrapServers, PartitionAssignmentStrategy assignmentStrategy)
+        {
+            var consumerConfig = new ConsumerConfig
+            {
+                GroupId = Guid.NewGuid().ToString(),
+                BootstrapServers = bootstrapServers,
+                SessionTimeoutMs = 6000,
+                AutoOffsetReset = AutoOffsetReset.Earliest,
+                PartitionAssignmentStrategy = assignmentStrategy
+            };
+
+            using (var topic1 = new TemporaryTopic(bootstrapServers, 4))
+            using (var topic2 = new TemporaryTopic(bootstrapServers, 4))
+            using (var topic3 = new TemporaryTopic(bootstrapServers, 4))
+            using (var topic4 = new TemporaryTopic(bootstrapServers, 4))
+            using (var consumer1 = new ConsumerBuilder<byte[], byte[]>(consumerConfig).Build())
+            using (var consumer2 = new ConsumerBuilder<byte[], byte[]>(consumerConfig).Build())
+            using (var consumer3 = new ConsumerBuilder<byte[], byte[]>(consumerConfig).Build())
+            using (var consumer4 = new ConsumerBuilder<byte[], byte[]>(consumerConfig).Build())
+            using (var consumer5 = new ConsumerBuilder<byte[], byte[]>(consumerConfig).Build())
+            using (var consumer6 = new ConsumerBuilder<byte[], byte[]>(consumerConfig).Build())
+            {
+                Util.ProduceNullStringMessages(bootstrapServers, topic1.Name, 100, 1000);
+                Util.ProduceNullStringMessages(bootstrapServers, topic2.Name, 100, 1000);
+                Util.ProduceNullStringMessages(bootstrapServers, topic3.Name, 100, 1000);
+                Util.ProduceNullStringMessages(bootstrapServers, topic4.Name, 100, 1000);
+
+                consumer1.Subscribe(topic1.Name);
+                // Consume -> wait for assignment
+                consumer1.Consume(TimeSpan.FromSeconds(10));
+                Assert.Equal(4, consumer1.Assignment.Count);
+                Assert.Equal(topic1.Name, consumer1.Assignment[0].Topic);
+
+                consumer2.Subscribe(topic2.Name);
+                consumer1.Consume(TimeSpan.FromSeconds(10));
+                Assert.Equal(4, consumer1.Assignment.Count);
+                Assert.Equal(topic1.Name, consumer1.Assignment[0].Topic);
+                consumer2.Consume(TimeSpan.FromSeconds(10));
+                Assert.Equal(4, consumer2.Assignment.Count);
+                Assert.Equal(topic2.Name, consumer2.Assignment[0].Topic);
+
+                consumer3.Subscribe(topic3.Name);
+                consumer1.Consume(TimeSpan.FromSeconds(10));
+                Assert.Equal(4, consumer1.Assignment.Count);
+                Assert.Equal(topic1.Name, consumer1.Assignment[0].Topic);
+                consumer2.Consume(TimeSpan.FromSeconds(10));
+                Assert.Equal(4, consumer2.Assignment.Count);
+                Assert.Equal(topic2.Name, consumer2.Assignment[0].Topic);
+                consumer3.Consume(TimeSpan.FromSeconds(10));
+                Assert.Equal(4, consumer3.Assignment.Count);
+                Assert.Equal(topic3.Name, consumer3.Assignment[0].Topic);
+
+                consumer4.Subscribe(topic4.Name);
+                consumer1.Consume(TimeSpan.FromSeconds(10));
+                Assert.Equal(4, consumer1.Assignment.Count);
+                Assert.Equal(topic1.Name, consumer1.Assignment[0].Topic);
+                consumer2.Consume(TimeSpan.FromSeconds(10));
+                Assert.Equal(4, consumer2.Assignment.Count);
+                Assert.Equal(topic2.Name, consumer2.Assignment[0].Topic);
+                consumer3.Consume(TimeSpan.FromSeconds(10));
+                Assert.Equal(4, consumer3.Assignment.Count);
+                Assert.Equal(topic3.Name, consumer3.Assignment[0].Topic);
+                consumer4.Consume(TimeSpan.FromSeconds(10));
+                Assert.Equal(4, consumer4.Assignment.Count);
+                Assert.Equal(topic4.Name, consumer4.Assignment[0].Topic);
+
+                consumer5.Subscribe(topic2.Name);
+                consumer5.Consume(TimeSpan.FromSeconds(10));
+                Assert.Equal(2, consumer2.Assignment.Count);
+                Assert.Equal(2, consumer5.Assignment.Count);
+                Assert.Equal(topic2.Name, consumer2.Assignment[0].Topic);
+                Assert.Equal(topic2.Name, consumer5.Assignment[0].Topic);
+
+                consumer6.Subscribe(new List<string> { topic3.Name, topic4.Name });
+                consumer6.Consume(TimeSpan.FromSeconds(10));
+                Assert.True(consumer3.Assignment.Count > 0);
+                Assert.True(consumer4.Assignment.Count > 0);
+                Assert.True(consumer6.Assignment.Count > 0);
+                Assert.Equal(8, consumer3.Assignment.Count + consumer4.Assignment.Count + consumer6.Assignment.Count);
+                Assert.Equal(topic3.Name, consumer3.Assignment[0].Topic);
+                Assert.Equal(topic4.Name, consumer4.Assignment[0].Topic);
+                Assert.True(consumer6.Assignment[0].Topic == topic3.Name || consumer6.Assignment[0].Topic == topic4.Name);
+
+                consumer1.Unsubscribe();
+                consumer2.Consume(TimeSpan.FromSeconds(10)); // wait for rebalance.
+                Assert.Equal(0, consumer1.Assignment.Count);
+
+                consumer1.Subscribe(topic1.Name);
+                consumer1.Consume(TimeSpan.FromSeconds(10));
+                Assert.Equal(4, consumer1.Assignment.Count);
+                Assert.Equal(topic1.Name, consumer1.Assignment[0].Topic);
+
+                consumer1.Close();
+                consumer2.Close();
+                consumer3.Close();
+                consumer4.Close();
+                consumer5.Close();
+                consumer6.Close();
+            }
+        }
+
+        /// <summary>
+        ///     Check various scenarios where the same consumer group subscribes to
+        ///     different topics in a disjoint fashion.
+        /// </summary>
+        [Theory, MemberData(nameof(KafkaParameters))]
+        public void Consumer_Subscription_DisjointTopics(string bootstrapServers)
+        {
+            LogToFile("start Consumer_Subscription_DisjointTopics");
+
+            DisjointTopicsSubscribeTest(bootstrapServers, PartitionAssignmentStrategy.Range);
+            DisjointTopicsSubscribeTest(bootstrapServers, PartitionAssignmentStrategy.RoundRobin);
+            DisjointTopicsSubscribeTest(bootstrapServers, PartitionAssignmentStrategy.CooperativeSticky);
+
+            Assert.Equal(0, Library.HandleCount);
+            LogToFile("end   Consumer_Subscription_DisjointTopics");
+        }
+
+    }
+}

test/Confluent.Kafka.IntegrationTests/Tests/Consumer_Subscription_Regex.cs

@@ -0,0 +1,86 @@
+// Copyright 2022 Confluent Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Refer to LICENSE for more information.
+
+#pragma warning disable xUnit1026
+
+using System;
+using System.Threading;
+using Xunit;
+
+
+namespace Confluent.Kafka.IntegrationTests
+{
+    public partial class Tests
+    {
+        /// <summary>
+        ///     Test regex subscription in scenario where new topics are added.
+        /// </summary>
+        [Theory, MemberData(nameof(KafkaParameters))]
+        public void Consumer_Subscription_Regex(string bootstrapServers)
+        {
+            LogToFile("start Consumer_Subscription_Regex");
+
+            var topicMetadataRefreshPeriodMs = 1000;
+            var rebalanceWaitMs = 2000;
+
+            var consumerConfig = new ConsumerConfig
+            {
+                GroupId = Guid.NewGuid().ToString(),
+                BootstrapServers = bootstrapServers,
+                SessionTimeoutMs = 6000,
+                AutoOffsetReset = AutoOffsetReset.Earliest,
+                TopicMetadataRefreshIntervalMs = topicMetadataRefreshPeriodMs
+            };
+
+            string prefix = "dotnet_test_" + Guid.NewGuid().ToString() + "_";
+
+            using (var topic1 = new TemporaryTopic(prefix, bootstrapServers, 1))
+            using (var topic2 = new TemporaryTopic(prefix, bootstrapServers, 1))
+            using (var consumer = new ConsumerBuilder<byte[], byte[]>(consumerConfig).Build())
+            {
+                Util.ProduceNullStringMessages(bootstrapServers, topic1.Name, 100, 100);
+                Util.ProduceNullStringMessages(bootstrapServers, topic2.Name, 100, 100);
+
+                consumer.Subscribe("^" + prefix + ".*$");
+
+                // Wait for assignment
+                consumer.Consume(TimeSpan.FromSeconds(10));
+                Assert.Equal(2, consumer.Assignment.Count);
+
+                // Create new matching topic, wait long enough for metadata to be
+                // discovered and corresponding rebalance.
+                using var topic3 = new TemporaryTopic(prefix, bootstrapServers, 1);
+                Thread.Sleep(topicMetadataRefreshPeriodMs + rebalanceWaitMs);
+                Assert.Equal(3, consumer.Assignment.Count);
+
+                // Repeat a couple more times... 
+                using var topic4 = new TemporaryTopic(prefix, bootstrapServers, 1);
+                Thread.Sleep(topicMetadataRefreshPeriodMs + rebalanceWaitMs);
+                Assert.Equal(4, consumer.Assignment.Count);
+
+                using var topic5 = new TemporaryTopic(prefix, bootstrapServers, 1);
+                Thread.Sleep(topicMetadataRefreshPeriodMs + rebalanceWaitMs);
+                Assert.Equal(5, consumer.Assignment.Count);
+
+                consumer.Close();
+            }
+
+            Assert.Equal(0, Library.HandleCount);
+            LogToFile("end   Consumer_Subscription_Regex");
+        }
+
+    }
+}