Skip to content

Commit 315c3f5

Browse files
milindlemasab
milindl
and
emasab
authored
[admin] Add consumer group bindings for KIP-88, 222, 518, 396 (partial) (#1981)
* [admin] Add ListConsumerGroupOffsets and AlterConsumerGroupOffsets bindings * [admin] Add ListConsumerGroups and DescribeConsumerGroups bindings * Also deprecates the older ListGroups code * [admin] Add KIP-518 changes to List/Describe Consumer Groups Co-authored-by: Emanuele Sabellico <[email protected]>
1 parent e402072 commit 315c3f5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+3241
-44
lines changed

‎CHANGELOG.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,19 @@
66
- Added `LatestCompatibilityStrict` configuration property to JsonSerializerConfig to check the compatibility with latest schema
77
when `UseLatestVersion` is set to true.
88
- Added DeleteConsumerGroupOffset to AdminClient.
9+
- [KIP-222](https://cwiki.apache.org/confluence/display/KAFKA/KIP-222+-+Add+Consumer+Group+operations+to+Admin+API)
10+
Finish remaining implementation: Add Consumer Group operations to Admin API (`DeleteGroups` is already present).
11+
- [KIP-518](https://cwiki.apache.org/confluence/display/KAFKA/KIP-518%3A+Allow+listing+consumer+groups+per+state)
12+
Allow listing consumer groups per state.
13+
- [KIP-396](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=97551484)
14+
Partially implemented: support for AlterConsumerGroupOffsets.
15+
- As result of the above KIPs, added (#1981)
16+
- `ListConsumerGroups` Admin operation. Supports listing by state.
17+
- `DescribeConsumerGroups` Admin operation. Supports multiple groups.
18+
- `ListConsumerGroupOffsets` Admin operation. Currently, only supports
19+
1 group with multiple partitions. Supports the `requireStable` option.
20+
- `AlterConsumerGroupOffsets` Admin operation. Currently, only supports
21+
1 group with multiple offsets.
922

1023
## Fixes
1124

‎examples/AdminClient/Program.cs‎

Lines changed: 227 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -295,11 +295,225 @@ static async Task DeleteAclsAsync(string bootstrapServers, string[] commandArgs)
295295
}
296296
}
297297

298+
static async Task AlterConsumerGroupOffsetsAsync(string bootstrapServers, string[] commandArgs)
299+
{
300+
if (commandArgs.Length < 4)
301+
{
302+
Console.WriteLine("usage: .. <bootstrapServers> alter-consumer-group-offsets <group_id> <topic1> <partition1> <offset1> ... <topicN> <partitionN> <offsetN>");
303+
Environment.ExitCode = 1;
304+
return;
305+
}
306+
307+
var group = commandArgs[0];
308+
var tpoes = new List<TopicPartitionOffset>();
309+
for (int i = 1; i + 2 < commandArgs.Length; i += 3) {
310+
try
311+
{
312+
var topic = commandArgs[i];
313+
var partition = Int32.Parse(commandArgs[i + 1]);
314+
var offset = Int64.Parse(commandArgs[i + 2]);
315+
tpoes.Add(new TopicPartitionOffset(topic, partition, offset));
316+
}
317+
catch (Exception e)
318+
{
319+
Console.Error.WriteLine($"An error occurred while parsing arguments: {e}");
320+
Environment.ExitCode = 1;
321+
return;
322+
}
323+
}
324+
325+
var input = new List<ConsumerGroupTopicPartitionOffsets>() { new ConsumerGroupTopicPartitionOffsets(group, tpoes) };
326+
327+
using (var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers }).Build())
328+
{
329+
try
330+
{
331+
var results = await adminClient.AlterConsumerGroupOffsetsAsync(input);
332+
Console.WriteLine("Successfully altered offsets:");
333+
foreach(var groupResult in results) {
334+
Console.WriteLine(groupResult);
335+
}
336+
337+
}
338+
catch (AlterConsumerGroupOffsetsException e)
339+
{
340+
Console.WriteLine($"An error occurred altering offsets: {(e.Results.Any() ? e.Results[0] : null)}");
341+
Environment.ExitCode = 1;
342+
return;
343+
}
344+
catch (KafkaException e)
345+
{
346+
Console.WriteLine("An error occurred altering consumer group offsets." +
347+
$" Code: {e.Error.Code}" +
348+
$", Reason: {e.Error.Reason}");
349+
Environment.ExitCode = 1;
350+
return;
351+
}
352+
}
353+
}
354+
355+
static async Task ListConsumerGroupOffsetsAsync(string bootstrapServers, string[] commandArgs)
356+
{
357+
if (commandArgs.Length < 1)
358+
{
359+
Console.WriteLine("usage: .. <bootstrapServers> list-consumer-group-offsets <group_id> [<topic1> <partition1> ... <topicN> <partitionN>]");
360+
Environment.ExitCode = 1;
361+
return;
362+
}
363+
364+
var group = commandArgs[0];
365+
var tpes = new List<TopicPartition>();
366+
for (int i = 1; i + 1 < commandArgs.Length; i += 2) {
367+
try
368+
{
369+
var topic = commandArgs[i];
370+
var partition = Int32.Parse(commandArgs[i + 1]);
371+
tpes.Add(new TopicPartition(topic, partition));
372+
}
373+
catch (Exception e)
374+
{
375+
Console.Error.WriteLine($"An error occurred while parsing arguments: {e}");
376+
Environment.ExitCode = 1;
377+
return;
378+
}
379+
}
380+
if(!tpes.Any())
381+
{
382+
// In case the list is empty, request offsets for all the partitions.
383+
tpes = null;
384+
}
385+
386+
var input = new List<ConsumerGroupTopicPartitions>() { new ConsumerGroupTopicPartitions(group, tpes) };
387+
388+
using (var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers }).Build())
389+
{
390+
try
391+
{
392+
var result = await adminClient.ListConsumerGroupOffsetsAsync(input);
393+
Console.WriteLine("Successfully listed offsets:");
394+
foreach(var groupResult in result) {
395+
Console.WriteLine(groupResult);
396+
}
397+
}
398+
catch (ListConsumerGroupOffsetsException e)
399+
{
400+
Console.WriteLine($"An error occurred listing offsets: {(e.Results.Any() ? e.Results[0] : null)}");
401+
Environment.ExitCode = 1;
402+
return;
403+
}
404+
catch (KafkaException e)
405+
{
406+
Console.WriteLine("An error occurred listing consumer group offsets." +
407+
$" Code: {e.Error.Code}" +
408+
$", Reason: {e.Error.Reason}");
409+
Environment.ExitCode = 1;
410+
return;
411+
}
412+
}
413+
}
414+
415+
static async Task ListConsumerGroupsAsync(string bootstrapServers, string[] commandArgs) {
416+
var timeout = TimeSpan.FromSeconds(30);
417+
var statesList = new List<ConsumerGroupState>();
418+
try
419+
{
420+
if (commandArgs.Length > 0)
421+
{
422+
timeout = TimeSpan.FromSeconds(Int32.Parse(commandArgs[0]));
423+
}
424+
if (commandArgs.Length > 1)
425+
{
426+
for (int i = 1; i < commandArgs.Length; i++) {
427+
statesList.Add(Enum.Parse<ConsumerGroupState>(commandArgs[i]));
428+
}
429+
}
430+
}
431+
catch (SystemException)
432+
{
433+
Console.WriteLine("usage: .. <bootstrapServers> list-consumer-groups [<timeout_seconds> <match_state_1> <match_state_2> ... <match_state_N>]");
434+
Environment.ExitCode = 1;
435+
return;
436+
}
437+
438+
using (var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers }).Build())
439+
{
440+
try
441+
{
442+
var result = await adminClient.ListConsumerGroupsAsync(new ListConsumerGroupsOptions() {
443+
RequestTimeout = timeout,
444+
MatchStates = statesList,
445+
});
446+
Console.WriteLine(result);
447+
}
448+
catch (KafkaException e)
449+
{
450+
Console.WriteLine("An error occurred listing consumer groups." +
451+
$" Code: {e.Error.Code}" +
452+
$", Reason: {e.Error.Reason}");
453+
Environment.ExitCode = 1;
454+
return;
455+
}
456+
457+
}
458+
}
459+
460+
461+
static async Task DescribeConsumerGroupsAsync(string bootstrapServers, string[] commandArgs) {
462+
if (commandArgs.Length < 1)
463+
{
464+
Console.WriteLine("usage: .. <bootstrapServers> describe-consumer-groups <group1> [<group2 ... <groupN>]");
465+
Environment.ExitCode = 1;
466+
return;
467+
}
468+
469+
var groupNames = commandArgs.ToList();
470+
var timeout = TimeSpan.FromSeconds(30);
471+
using (var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers }).Build())
472+
{
473+
try
474+
{
475+
var descResult = await adminClient.DescribeConsumerGroupsAsync(groupNames, new DescribeConsumerGroupsOptions() { RequestTimeout = timeout });
476+
foreach (var group in descResult.ConsumerGroupDescriptions)
477+
{
478+
Console.WriteLine($" Group: {group.GroupId} {group.Error}");
479+
Console.WriteLine($" Broker: {group.Coordinator}");
480+
Console.WriteLine($" IsSimpleConsumerGroup: {group.IsSimpleConsumerGroup}");
481+
Console.WriteLine($" PartitionAssignor: {group.PartitionAssignor}");
482+
Console.WriteLine($" State: {group.State}");
483+
Console.WriteLine($" Members:");
484+
foreach (var m in group.Members)
485+
{
486+
Console.WriteLine($" {m.ClientId} {m.ConsumerId} {m.Host}");
487+
Console.WriteLine($" Assignment:");
488+
var topicPartitions = "";
489+
if (m.Assignment.TopicPartitions != null)
490+
{
491+
topicPartitions = String.Join(", ", m.Assignment.TopicPartitions.Select(tp => tp.ToString()));
492+
}
493+
Console.WriteLine($" TopicPartitions: [{topicPartitions}]");
494+
}
495+
}
496+
}
497+
catch (KafkaException e)
498+
{
499+
Console.WriteLine($"An error occurred describing consumer groups: {e}");
500+
Environment.ExitCode = 1;
501+
}
502+
}
503+
}
504+
298505
public static async Task Main(string[] args)
299506
{
300507
if (args.Length < 2)
301508
{
302-
Console.WriteLine("usage: .. <bootstrapServers> <list-groups|metadata|library-version|create-topic|create-acls|describe-acls|delete-acls> ..");
509+
Console.WriteLine(
510+
"usage: .. <bootstrapServers> " + String.Join("|", new string[] {
511+
"list-groups", "metadata", "library-version", "create-topic", "create-acls",
512+
"describe-acls", "delete-acls",
513+
"list-consumer-groups", "describe-consumer-groups",
514+
"list-consumer-group-offsets", "alter-consumer-group-offsets"
515+
}) +
516+
" ..");
303517
Environment.ExitCode = 1;
304518
return;
305519
}
@@ -331,6 +545,18 @@ public static async Task Main(string[] args)
331545
case "delete-acls":
332546
await DeleteAclsAsync(bootstrapServers, commandArgs);
333547
break;
548+
case "alter-consumer-group-offsets":
549+
await AlterConsumerGroupOffsetsAsync(bootstrapServers, commandArgs);
550+
break;
551+
case "list-consumer-group-offsets":
552+
await ListConsumerGroupOffsetsAsync(bootstrapServers, commandArgs);
553+
break;
554+
case "list-consumer-groups":
555+
await ListConsumerGroupsAsync(bootstrapServers, commandArgs);
556+
break;
557+
case "describe-consumer-groups":
558+
await DescribeConsumerGroupsAsync(bootstrapServers, commandArgs);
559+
break;
334560
default:
335561
Console.WriteLine($"unknown command: {command}");
336562
break;

‎src/Confluent.Kafka/Admin/AlterConsumerGroupOffsetsException.cs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright 2022 Confluent Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
//
15+
// Refer to LICENSE for more information.
16+
17+
using System.Collections.Generic;
18+
19+
namespace Confluent.Kafka.Admin
20+
{
21+
/// <summary>
22+
/// Represents an error that occured during altering consumer group offsets.
23+
/// </summary>
24+
public class AlterConsumerGroupOffsetsException : KafkaException
25+
{
26+
/// <summary>
27+
/// Initializes a new instance of AlterConsumerGroupOffsetsException.
28+
/// </summary>
29+
/// <param name="results">
30+
/// The result corresponding to all groups/partitions in the request
31+
/// (whether or not they were in error). At least one of these
32+
/// results will be in error.
33+
/// </param>
34+
public AlterConsumerGroupOffsetsException(List<AlterConsumerGroupOffsetsReport> results)
35+
: base(new Error(ErrorCode.Local_Partial,
36+
"An error occurred altering consumer group offsets, check individual result elements"))
37+
{
38+
Results = results;
39+
}
40+
41+
/// <summary>
42+
/// The result corresponding to all groups/partitions in the request
43+
/// (whether or not they were in error). At least one of these
44+
/// results will be in error.
45+
/// </summary>
46+
public List<AlterConsumerGroupOffsetsReport> Results { get; }
47+
}
48+
}

‎src/Confluent.Kafka/Admin/AlterConsumerGroupOffsetsOptions.cs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Copyright 2022 Confluent Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
//
15+
// Refer to LICENSE for more information.
16+
17+
using System;
18+
19+
namespace Confluent.Kafka.Admin
20+
{
21+
/// <summary>
22+
/// Options for the "AdminClient.AlterConsumerGroupOffsetsAsync" method.
23+
/// </summary>
24+
public class AlterConsumerGroupOffsetsOptions
25+
{
26+
/// <summary>
27+
/// The overall request timeout, including broker lookup, request
28+
/// transmission, operation time on broker, and response. If set
29+
/// to null, the default request timeout for the AdminClient will
30+
/// be used.
31+
///
32+
/// Default: null
33+
/// </summary>
34+
public TimeSpan? RequestTimeout { get; set; }
35+
}
36+
}

0 commit comments

Comments
 (0)