From a3955500c78a600550ef602c74ff3ebfec70f757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Zborek?= Date: Tue, 12 May 2026 23:08:24 +0200 Subject: [PATCH 1/4] feat(csharp): implement rented message polling and deserialization in Iggy consumer --- foreign/csharp/Directory.Packages.props | 53 +- .../FetchMessagesTests.cs | 398 +-- .../IggyTypedConsumerTests.cs | 272 ++ .../Iggy_SDK.Tests.Integration.csproj | 89 +- .../Iggy_SDK/Consumers/DeserializedMessage.cs | 65 + .../Iggy_SDK/Consumers/IDeserializer.cs | 104 +- .../Iggy_SDK/Consumers/IggyConsumer.Rented.cs | 250 ++ .../csharp/Iggy_SDK/Consumers/IggyConsumer.cs | 1081 +++---- .../Iggy_SDK/Consumers/IggyConsumerOfT.cs | 300 +- .../Consumers/ReceivedRentedMessage.cs | 126 + .../Iggy_SDK/Contracts/MessageResponse.cs | 148 +- .../Contracts/PolledMessagesRental.cs | 71 + .../Contracts/RentedMessageResponse.cs | 71 + .../Encryption/AesMessageEncryptor.cs | 257 +- .../Iggy_SDK/Encryption/IMessageEncryptor.cs | 80 +- .../Extensions/IggyClientExtension.cs | 148 +- .../Iggy_SDK/IggyClient/IIggyConsumer.cs | 148 +- .../Implementations/HttpMessageStream.cs | 1712 +++++------ .../Implementations/TcpMessageStream.cs | 2556 +++++++++-------- .../MessageResponseConverter.cs | 204 +- .../csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 244 +- .../ConsumerTests/RentedConsumerTests.cs | 483 ++++ .../ConsumerTests/RentedTypedConsumerTests.cs | 183 ++ .../MapperTests/BinaryMapper.cs | 4 +- .../MessageResponseConverterTests.cs | 1 + 25 files changed, 5469 insertions(+), 3579 deletions(-) create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs create mode 100644 foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs create mode 100644 foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs create mode 100644 foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs create mode 100644 foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs create mode 100644 foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedTypedConsumerTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/MapperTests/MessageResponseConverterTests.cs diff --git a/foreign/csharp/Directory.Packages.props b/foreign/csharp/Directory.Packages.props index b2dee5c2a0..cd888e5b8f 100644 --- a/foreign/csharp/Directory.Packages.props +++ b/foreign/csharp/Directory.Packages.props @@ -1,27 +1,26 @@ - - - true - false - - - - - - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - + + + true + false + + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs index 4a32eff1f7..91768ada45 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs @@ -1,184 +1,214 @@ -// // Licensed to the Apache Software Foundation (ASF) under one -// // or more contributor license agreements. See the NOTICE file -// // distributed with this work for additional information -// // regarding copyright ownership. The ASF licenses this file -// // to you under the Apache License, Version 2.0 (the -// // "License"); you may not use this file except in compliance -// // with the License. You may obtain a copy of the License at -// // -// // http://www.apache.org/licenses/LICENSE-2.0 -// // -// // Unless required by applicable law or agreed to in writing, -// // software distributed under the License is distributed on an -// // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// // KIND, either express or implied. See the License for the -// // specific language governing permissions and limitations -// // under the License. - -using System.Text; -using Apache.Iggy.Contracts; -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Headers; -using Apache.Iggy.IggyClient; -using Apache.Iggy.Kinds; -using Apache.Iggy.Messages; -using Apache.Iggy.Tests.Integrations.Fixtures; -using Shouldly; -using Partitioning = Apache.Iggy.Kinds.Partitioning; - -namespace Apache.Iggy.Tests.Integrations; - -public class FetchMessagesTests -{ - private const int MessageCount = 20; - private const string TopicName = "topic"; - private const string HeadersTopicName = "headers-topic"; - - [ClassDataSource(Shared = SharedType.PerAssembly)] - public required IggyServerFixture Fixture { get; init; } - - private async Task<(IIggyClient client, string streamName)> CreateStreamWithMessages(Protocol protocol) - { - var client = await Fixture.CreateAuthenticatedClient(protocol); - - var streamName = $"fetch-msg-{Guid.NewGuid():N}"; - - await client.CreateStreamAsync(streamName); - await client.CreateTopicAsync(Identifier.String(streamName), TopicName, 1); - await client.CreateTopicAsync(Identifier.String(streamName), HeadersTopicName, 1); - - await client.SendMessagesAsync(Identifier.String(streamName), - Identifier.String(TopicName), Partitioning.None(), - CreateMessagesWithoutHeader(MessageCount)); - - await client.SendMessagesAsync(Identifier.String(streamName), - Identifier.String(HeadersTopicName), Partitioning.None(), - CreateMessagesWithHeader(MessageCount)); - - return (client, streamName); - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task PollMessages_WithNoHeaders_Should_PollMessages_Successfully(Protocol protocol) - { - var (client, streamName) = await CreateStreamWithMessages(protocol); - - var response = await client.PollMessagesAsync(new MessageFetchRequest - { - Count = 10, - AutoCommit = true, - Consumer = Consumer.New(1), - PartitionId = 0, - PollingStrategy = PollingStrategy.Next(), - StreamId = Identifier.String(streamName), - TopicId = Identifier.String(TopicName) - }); - - response.Messages.Count.ShouldBe(10); - response.PartitionId.ShouldBe(0); - response.CurrentOffset.ShouldBe(19u); - - foreach (var responseMessage in response.Messages) - { - responseMessage.UserHeaders.ShouldBeNull(); - responseMessage.Payload.ShouldNotBeNull(); - responseMessage.Payload.Length.ShouldBeGreaterThan(0); - } - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task PollMessages_InvalidTopic_Should_Throw_InvalidResponse(Protocol protocol) - { - var (client, streamName) = await CreateStreamWithMessages(protocol); - - var invalidFetchRequest = new MessageFetchRequest - { - Count = 10, - AutoCommit = true, - Consumer = Consumer.New(1), - PartitionId = 0, - PollingStrategy = PollingStrategy.Next(), - StreamId = Identifier.String(streamName), - TopicId = Identifier.Numeric(2137) - }; - - await Should.ThrowAsync(() => - client.PollMessagesAsync(invalidFetchRequest)); - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task PollMessages_WithHeaders_Should_PollMessages_Successfully(Protocol protocol) - { - var (client, streamName) = await CreateStreamWithMessages(protocol); - - var headersMessageFetchRequest = new MessageFetchRequest - { - Count = 10, - AutoCommit = true, - Consumer = Consumer.New(1), - PartitionId = 0, - PollingStrategy = PollingStrategy.Next(), - StreamId = Identifier.String(streamName), - TopicId = Identifier.String(HeadersTopicName) - }; - - var response = await client.PollMessagesAsync(headersMessageFetchRequest); - response.Messages.Count.ShouldBe(10); - response.PartitionId.ShouldBe(0); - response.CurrentOffset.ShouldBe(19u); - foreach (var responseMessage in response.Messages) - { - responseMessage.UserHeaders.ShouldNotBeNull(); - responseMessage.UserHeaders.Count.ShouldBe(2); - responseMessage.UserHeaders[HeaderKey.FromString("header1")].ToString().ShouldBe("value1"); - responseMessage.UserHeaders[HeaderKey.FromString("header2")].ToInt32().ShouldBeGreaterThan(0); - } - } - - private static Message[] CreateMessagesWithoutHeader(int count) - { - var messages = new List(); - for (var i = 0; i < count; i++) - { - var dummyJson = $$""" - { - "userId": {{i + 1}}, - "id": {{i + 1}}, - "title": "delete", - "completed": false - } - """; - messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson))); - } - - return messages.ToArray(); - } - - private static Message[] CreateMessagesWithHeader(int count) - { - var messages = new List(); - for (var i = 0; i < count; i++) - { - var dummyJson = $$""" - { - "userId": {{i + 1}}, - "id": {{i + 1}}, - "title": "delete", - "completed": false - } - """; - messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson), - new Dictionary - { - { HeaderKey.FromString("header1"), HeaderValue.FromString("value1") }, - { HeaderKey.FromString("header2"), HeaderValue.FromInt32(14 + i) } - })); - } - - return messages.ToArray(); - } -} +// // Licensed to the Apache Software Foundation (ASF) under one +// // or more contributor license agreements. See the NOTICE file +// // distributed with this work for additional information +// // regarding copyright ownership. The ASF licenses this file +// // to you under the Apache License, Version 2.0 (the +// // "License"); you may not use this file except in compliance +// // with the License. You may obtain a copy of the License at +// // +// // http://www.apache.org/licenses/LICENSE-2.0 +// // +// // Unless required by applicable law or agreed to in writing, +// // software distributed under the License is distributed on an +// // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// // KIND, either express or implied. See the License for the +// // specific language governing permissions and limitations +// // under the License. + +using System.Text; +using Apache.Iggy.Contracts; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; +using Apache.Iggy.Messages; +using Apache.Iggy.Tests.Integrations.Fixtures; +using Shouldly; +using Partitioning = Apache.Iggy.Kinds.Partitioning; + +namespace Apache.Iggy.Tests.Integrations; + +public class FetchMessagesTests +{ + private const int MessageCount = 20; + private const string TopicName = "topic"; + private const string HeadersTopicName = "headers-topic"; + + [ClassDataSource(Shared = SharedType.PerAssembly)] + public required IggyServerFixture Fixture { get; init; } + + private async Task<(IIggyClient client, string streamName)> CreateStreamWithMessages(Protocol protocol) + { + var client = await Fixture.CreateAuthenticatedClient(protocol); + + var streamName = $"fetch-msg-{Guid.NewGuid():N}"; + + await client.CreateStreamAsync(streamName); + await client.CreateTopicAsync(Identifier.String(streamName), TopicName, 1); + await client.CreateTopicAsync(Identifier.String(streamName), HeadersTopicName, 1); + + await client.SendMessagesAsync(Identifier.String(streamName), + Identifier.String(TopicName), Partitioning.None(), + CreateMessagesWithoutHeader(MessageCount)); + + await client.SendMessagesAsync(Identifier.String(streamName), + Identifier.String(HeadersTopicName), Partitioning.None(), + CreateMessagesWithHeader(MessageCount)); + + return (client, streamName); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task PollMessages_WithNoHeaders_Should_PollMessages_Successfully(Protocol protocol) + { + var (client, streamName) = await CreateStreamWithMessages(protocol); + + var response = await client.PollMessagesAsync(new MessageFetchRequest + { + Count = 10, + AutoCommit = true, + Consumer = Consumer.New(1), + PartitionId = 0, + PollingStrategy = PollingStrategy.Next(), + StreamId = Identifier.String(streamName), + TopicId = Identifier.String(TopicName) + }); + + response.Messages.Count.ShouldBe(10); + response.PartitionId.ShouldBe(0); + response.CurrentOffset.ShouldBe(19u); + + foreach (var responseMessage in response.Messages) + { + responseMessage.UserHeaders.ShouldBeNull(); + responseMessage.Payload.ShouldNotBeNull(); + responseMessage.Payload.Length.ShouldBeGreaterThan(0); + } + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task PollMessages_InvalidTopic_Should_Throw_InvalidResponse(Protocol protocol) + { + var (client, streamName) = await CreateStreamWithMessages(protocol); + + var invalidFetchRequest = new MessageFetchRequest + { + Count = 10, + AutoCommit = true, + Consumer = Consumer.New(1), + PartitionId = 0, + PollingStrategy = PollingStrategy.Next(), + StreamId = Identifier.String(streamName), + TopicId = Identifier.Numeric(2137) + }; + + await Should.ThrowAsync(() => + client.PollMessagesAsync(invalidFetchRequest)); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task PollMessages_WithHeaders_Should_PollMessages_Successfully(Protocol protocol) + { + var (client, streamName) = await CreateStreamWithMessages(protocol); + + var headersMessageFetchRequest = new MessageFetchRequest + { + Count = 10, + AutoCommit = true, + Consumer = Consumer.New(1), + PartitionId = 0, + PollingStrategy = PollingStrategy.Next(), + StreamId = Identifier.String(streamName), + TopicId = Identifier.String(HeadersTopicName) + }; + + var response = await client.PollMessagesAsync(headersMessageFetchRequest); + response.Messages.Count.ShouldBe(10); + response.PartitionId.ShouldBe(0); + response.CurrentOffset.ShouldBe(19u); + foreach (var responseMessage in response.Messages) + { + responseMessage.UserHeaders.ShouldNotBeNull(); + responseMessage.UserHeaders.Count.ShouldBe(2); + responseMessage.UserHeaders[HeaderKey.FromString("header1")].ToString().ShouldBe("value1"); + responseMessage.UserHeaders[HeaderKey.FromString("header2")].ToInt32().ShouldBeGreaterThan(0); + } + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task PollMessages_WithHeaders_Should_AutoParseUserHeadersFromRaw(Protocol protocol) + { + var (client, streamName) = await CreateStreamWithMessages(protocol); + + var response = await client.PollMessagesAsync(new MessageFetchRequest + { + Count = 1, + AutoCommit = true, + Consumer = Consumer.New(2), + PartitionId = 0, + PollingStrategy = PollingStrategy.First(), + StreamId = Identifier.String(streamName), + TopicId = Identifier.String(HeadersTopicName) + }); + + response.Messages.Count.ShouldBe(1); + var msg = response.Messages[0]; + + Dictionary? parsed = msg.UserHeaders; + parsed.ShouldNotBeNull(); + parsed!.Count.ShouldBe(2); + parsed[HeaderKey.FromString("header1")].ToString().ShouldBe("value1"); + parsed[HeaderKey.FromString("header2")].ToInt32().ShouldBe(14); + + Dictionary? second = msg.UserHeaders; + second.ShouldBeSameAs(parsed); + } + + private static Message[] CreateMessagesWithoutHeader(int count) + { + var messages = new List(); + for (var i = 0; i < count; i++) + { + var dummyJson = $$""" + { + "userId": {{i + 1}}, + "id": {{i + 1}}, + "title": "delete", + "completed": false + } + """; + messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson))); + } + + return messages.ToArray(); + } + + private static Message[] CreateMessagesWithHeader(int count) + { + var messages = new List(); + for (var i = 0; i < count; i++) + { + var dummyJson = $$""" + { + "userId": {{i + 1}}, + "id": {{i + 1}}, + "title": "delete", + "completed": false + } + """; + messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson), + new Dictionary + { + { HeaderKey.FromString("header1"), HeaderValue.FromString("value1") }, + { HeaderKey.FromString("header2"), HeaderValue.FromInt32(14 + i) } + })); + } + + return messages.ToArray(); + } +} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs new file mode 100644 index 0000000000..8dd3160ffd --- /dev/null +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text; +using Apache.Iggy.Consumers; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; +using Apache.Iggy.Messages; +using Apache.Iggy.Tests.Integrations.Fixtures; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Partitioning = Apache.Iggy.Kinds.Partitioning; + +namespace Apache.Iggy.Tests.Integrations; + +public class IggyTypedConsumerTests +{ + [ClassDataSource(Shared = SharedType.PerAssembly)] + public required IggyServerFixture Fixture { get; init; } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task ReceiveRentedDeserializedAsync_Should_YieldMessages_WithCorrectData(Protocol protocol) + { + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); + + var testStream = await CreateTestStreamWithMessages(client, protocol); + + var consumerId = protocol == Protocol.Tcp ? 200 : 300; + IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), + AutoCommitMode.Disabled, new Utf8StringDeserializer()); + + await consumer.InitAsync(); + + var received = new List>(); + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + await foreach (DeserializedMessage msg in consumer.ReceiveRentedDeserializedAsync(cts.Token)) + { + msg.ShouldNotBeNull(); + msg.Status.ShouldBe(MessageStatus.Success); + msg.Data.ShouldNotBeNull(); + msg.Data.ShouldStartWith("Test message"); + msg.PartitionId.ShouldBe(1u); + msg.Error.ShouldBeNull(); + received.Add(msg); + if (received.Count >= 10) + { + break; + } + } + + received.Count.ShouldBeGreaterThanOrEqualTo(10); + await consumer.DisposeAsync(); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task ReceiveRentedDeserializedAsync_WithoutInit_Should_Throw_ConsumerNotInitializedException( + Protocol protocol) + { + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); + + var testStream = await CreateTestStreamWithMessages(client, protocol); + + var consumerId = protocol == Protocol.Tcp ? 201 : 301; + IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), + AutoCommitMode.Disabled, new Utf8StringDeserializer()); + + await Should.ThrowAsync(async () => + { + await foreach (DeserializedMessage _ in consumer.ReceiveRentedDeserializedAsync()) + { + } + }); + + await consumer.DisposeAsync(); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task ReceiveRentedDeserializedAsync_WithAutoCommitAfterReceive_Should_StoreOffset(Protocol protocol) + { + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); + + var testStream = await CreateTestStreamWithMessages(client, protocol); + + var consumerId = protocol == Protocol.Tcp ? 202 : 302; + IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), + AutoCommitMode.AfterReceive, new Utf8StringDeserializer(), + PollingStrategy.First()); + + await consumer.InitAsync(); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var count = 0; + + await foreach (DeserializedMessage _ in consumer.ReceiveRentedDeserializedAsync(cts.Token)) + { + count++; + if (count >= 5) + { + break; + } + } + + await consumer.DisposeAsync(); + + var offset = await client.GetOffsetAsync(Consumer.New(consumerId), + Identifier.String(testStream.StreamId), + Identifier.String(testStream.TopicId), + 1u); + + offset.ShouldNotBeNull(); + offset.StoredOffset.ShouldBe(3ul); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task ReceiveRentedDeserializedAsync_WithFailingDeserializer_Should_YieldDeserializationFailed( + Protocol protocol) + { + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); + + var testStream = await CreateTestStreamWithMessages(client, protocol); + + var consumerId = protocol == Protocol.Tcp ? 203 : 303; + IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), + AutoCommitMode.Disabled, new FailingDeserializer()); + + await consumer.InitAsync(); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + await foreach (DeserializedMessage msg in consumer.ReceiveRentedDeserializedAsync(cts.Token)) + { + msg.Status.ShouldBe(MessageStatus.DeserializationFailed); + msg.Data.ShouldBeNull(); + msg.Error.ShouldNotBeNull(); + msg.Error.ShouldBeOfType(); + break; + } + + await consumer.DisposeAsync(); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task ReceiveRentedDeserializedAsync_Should_StopCleanly_OnCancellation(Protocol protocol) + { + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); + + var testStream = await CreateTestStreamWithMessages(client, protocol); + + var consumerId = protocol == Protocol.Tcp ? 204 : 304; + IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), + AutoCommitMode.Disabled, new Utf8StringDeserializer()); + + await consumer.InitAsync(); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); + + await Should.NotThrowAsync(async () => + { + try + { + await foreach (DeserializedMessage _ in consumer.ReceiveRentedDeserializedAsync(cts.Token)) + { + } + } + catch (OperationCanceledException) + { + } + }); + + await consumer.DisposeAsync(); + } + + private static IggyConsumer BuildTypedConsumer(IIggyClient client, + TestStreamInfo stream, + Consumer consumer, + AutoCommitMode autoCommitMode, + IDeserializer deserializer, + PollingStrategy? pollingStrategy = null) + { + var config = new IggyConsumerConfig + { + StreamId = Identifier.String(stream.StreamId), + TopicId = Identifier.String(stream.TopicId), + Consumer = consumer, + PollingStrategy = pollingStrategy ?? PollingStrategy.Next(), + BatchSize = 10, + PartitionId = 1, + AutoCommitMode = autoCommitMode, + AutoCommit = autoCommitMode != AutoCommitMode.Disabled, + PollingIntervalMs = 0, + Deserializer = deserializer + }; + return new IggyConsumer(client, config, NullLoggerFactory.Instance); + } + + private async Task CreateTestStreamWithMessages(IIggyClient client, Protocol protocol, + uint partitionsCount = 5, int messagesPerPartition = 100) + { + var streamId = $"typed_stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}"; + var topicId = "test_topic"; + + await client.CreateStreamAsync(streamId); + await client.CreateTopicAsync(Identifier.String(streamId), topicId, partitionsCount); + + for (uint partitionId = 0; partitionId < partitionsCount; partitionId++) + { + var messages = new List(); + for (var i = 0; i < messagesPerPartition; i++) + { + messages.Add(new Message(Guid.NewGuid(), + Encoding.UTF8.GetBytes($"Test message {i} for partition {partitionId}"))); + } + + await client.SendMessagesAsync(Identifier.String(streamId), + Identifier.String(topicId), + Partitioning.PartitionId((int)partitionId), + messages); + } + + return new TestStreamInfo(streamId, topicId, partitionsCount, messagesPerPartition); + } + + private record TestStreamInfo(string StreamId, string TopicId, uint PartitionsCount, int MessagesPerPartition); + + private sealed class Utf8StringDeserializer : IDeserializer + { + public string Deserialize(ReadOnlyMemory data) + { + return Encoding.UTF8.GetString(data.Span); + } + } + + private sealed class FailingDeserializer : IDeserializer + { + public string Deserialize(ReadOnlyMemory data) + { + throw new InvalidOperationException("Intentional deserialization failure"); + } + } +} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj b/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj index 1b229b82c6..a0bc77def7 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj @@ -1,45 +1,44 @@ - - - - enable - enable - Exe - net8.0;net10.0 - Apache.Iggy.Tests.Integrations - Apache.Iggy.Tests.Integrations - true - - - - - - - - - - - - - - - - - - Certs\iggy.pfx - Always - - - Certs\iggy_ca_cert.pem - Always - - - Certs\iggy_cert.pem - Always - - - Certs\iggy_key.pem - Always - - - - + + + + enable + enable + Exe + net8.0;net10.0 + Apache.Iggy.Tests.Integrations + Apache.Iggy.Tests.Integrations + true + + + + + + + + + + + + + + + + + Certs\iggy.pfx + Always + + + Certs\iggy_ca_cert.pem + Always + + + Certs\iggy_cert.pem + Always + + + Certs\iggy_key.pem + Always + + + + diff --git a/foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs b/foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs new file mode 100644 index 0000000000..b7e8cdbb83 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Headers; +using Apache.Iggy.Messages; + +namespace Apache.Iggy.Consumers; + +/// +/// Represents a message whose payload was deserialized directly from rented memory. The rented buffer has +/// already been returned to the pool by the time this message is yielded, so only the header, user headers, +/// and the deserialized are available; the raw payload bytes are not retained. +/// +/// The deserialized payload type. +public sealed class DeserializedMessage +{ + /// + /// Message header. + /// + public required MessageHeader Header { get; init; } + + /// + /// The deserialized payload. Null if is not . + /// + public T? Data { get; init; } + + /// + /// Parsed user headers, if present. + /// + public Dictionary? UserHeaders { get; init; } + + /// + /// The current offset of this message in the partition. + /// + public required ulong CurrentOffset { get; init; } + + /// + /// The partition ID from which this message was consumed. + /// + public uint PartitionId { get; init; } + + /// + /// The status of the message (Success, DecryptionFailed, DeserializationFailed). + /// + public MessageStatus Status { get; init; } = MessageStatus.Success; + + /// + /// The exception that occurred during processing, if any. + /// + public Exception? Error { get; init; } +} diff --git a/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs b/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs index 4a82d11082..e22372144d 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs @@ -1,54 +1,50 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -namespace Apache.Iggy.Consumers; - -/// -/// Interface for deserializing message payloads from byte arrays to type T. -/// -/// No type constraints are enforced on T to provide maximum flexibility. -/// Implementations are responsible for ensuring that the provided byte data can be properly deserialized to the target type. -/// -/// -/// -/// The target type for deserialization. Can be any type - reference or value type, nullable or non-nullable. -/// The deserializer implementation must be able to produce instances of the specific type. -/// -/// -/// Implementations should throw appropriate exceptions (e.g., , -/// , or ) -/// if the provided data cannot be deserialized to type T. These exceptions will be caught and logged by -/// during message processing. -/// -public interface IDeserializer -{ - /// - /// Deserializes a byte array into an instance of type T. - /// - /// The byte array containing the serialized data to deserialize. - /// An instance of type T representing the deserialized data. - /// - /// Thrown when the data format is invalid and cannot be deserialized. - /// - /// - /// Thrown when the data cannot be deserialized due to invalid content or structure. - /// - /// - /// Thrown when the deserialization operation fails due to state issues. - /// - T Deserialize(byte[] data); -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.Iggy.Consumers; + +/// +/// Interface for deserializing message payloads from byte arrays to type T. +/// +/// No type constraints are enforced on T to provide maximum flexibility. +/// Implementations are responsible for ensuring that the provided byte data can be properly deserialized to the +/// target type. +/// +/// +/// +/// The target type for deserialization. Can be any type - reference or value type, nullable or non-nullable. +/// The deserializer implementation must be able to produce instances of the specific type. +/// +/// +/// Implementations should throw appropriate exceptions (e.g., , +/// , or ) +/// if the provided data cannot be deserialized to type T. These exceptions will be caught and logged by +/// during message processing. +/// +public interface IDeserializer +{ + /// + /// Deserializes a read-only memory into an instance of type T. Callers may pass a byte[] directly + /// thanks to the implicit conversion to . + /// + /// + /// Read-only memory containing the serialized data. The implementation MUST NOT retain a reference to + /// the span after returning. + /// + /// An instance of type T representing the deserialized data. + T Deserialize(ReadOnlyMemory data); +} diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs new file mode 100644 index 0000000000..a18f4c784e --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs @@ -0,0 +1,250 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Apache.Iggy.Contracts; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Headers; +using Apache.Iggy.Kinds; +using Apache.Iggy.Mappers; +using Microsoft.Extensions.Logging; + +namespace Apache.Iggy.Consumers; + +public partial class IggyConsumer +{ + private readonly Channel _rentedChannel = Channel.CreateUnbounded(); + + /// + /// Receives messages asynchronously as an async stream of rented messages. Each yielded + /// shares its underlying pooled buffer with the other messages from the + /// same poll and MUST be disposed by the caller when processing is complete. The buffer is returned to the + /// pool once every message of its batch has been disposed. + /// + /// Cancellation token to stop receiving messages. + /// An async enumerable of rented messages. + /// Thrown when has not been called. + public async IAsyncEnumerable ReceiveRentedAsync( + [EnumeratorCancellation] CancellationToken ct = default) + { + if (!_isInitialized) + { + throw new ConsumerNotInitializedException(); + } + + do + { + if (!_rentedChannel.Reader.TryRead(out var message)) + { + await PollRentedMessagesAsync(ct); + continue; + } + + yield return message; + + if (_config.AutoCommitMode == AutoCommitMode.AfterReceive) + { + await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, false, ct); + } + } while (!ct.IsCancellationRequested); + } + + /// + /// Publishes a single rented message from a polled batch to the consumer channel. Called once per + /// message during . The caller acquires a reference on + /// before invocation; the channel reader is expected to release that + /// reference by disposing the produced . Override to redirect + /// rented batches to a different sink (e.g. typed deserialization) — overrides must ensure the + /// acquired reference is released exactly once on every path. + /// + /// Reference-counted handle around the polled batch. Caller has already acquired one reference. + /// + /// The rented message to publish. Payload and raw headers are slices of pooled memory tied to + /// . + /// + /// Partition the message was polled from. + /// Outcome of any prior processing (e.g. decryption). + /// Exception captured if is non-success; otherwise null. + /// Cancellation token. + protected virtual async Task PublishRentedAsync(RentedBatchHandle rental, + RentedMessageResponse message, + uint partitionId, + MessageStatus status, + Exception? error, + CancellationToken ct) + { + await _rentedChannel.Writer.WriteAsync(new ReceivedRentedMessage + { + Handle = rental, + Message = message, + CurrentOffset = message.Header.Offset, + PartitionId = partitionId, + Status = status, + Error = error + }, ct); + } + + /// + /// Polls a rented batch from the server and publishes it via . + /// Handles decryption, offset tracking, and auto-commit logic. Rental lifetime is managed via + /// shared by every produced message. + /// + protected async Task PollRentedMessagesAsync(CancellationToken ct) + { + if (!_joinedConsumerGroup) + { + LogConsumerGroupNotJoinedYetSkippingPolling(); + return; + } + + await _pollingSemaphore.WaitAsync(ct); + + PolledMessagesRental? rental = null; + RentedBatchHandle? batchHandle = null; + try + { + if (_config.PollingIntervalMs > 0) + { + await WaitBeforePollingAsync(ct); + } + + rental = await _client.PollMessagesRentedAsync(_config.StreamId, _config.TopicId, + _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize, + _config.AutoCommit, ct); + + if (rental.Messages.Count == 0) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug("No messages received from poll for partition {PartitionId}", rental.PartitionId); + } + + return; + } + + var partitionId = (uint)rental.PartitionId; + + var hasLastOffset = _lastPolledOffset.TryGetValue(rental.PartitionId, out var lastPolledPartitionOffset); + + var currentOffset = 0ul; + + batchHandle = new RentedBatchHandle(rental); + var anyNewMessages = false; + foreach (var message in rental.Messages) + { + if (hasLastOffset && message.Header.Offset <= lastPolledPartitionOffset) + { + continue; + } + + var processedMessage = message; + var status = MessageStatus.Success; + Exception? error = null; + + if (_config.MessageEncryptor != null) + { + try + { + var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload.Span); + + Dictionary? decryptedHeaders = null; + if (!message.RawUserHeaders.IsEmpty) + { + var decryptedHeaderBytes = + _config.MessageEncryptor.Decrypt(message.RawUserHeaders.Span); + decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes); + } + + processedMessage = new RentedMessageResponse + { + Header = message.Header, + Payload = decryptedPayload, + RawUserHeaders = ReadOnlyMemory.Empty, + UserHeaders = decryptedHeaders + }; + } + catch (Exception ex) + { + LogFailedToDecryptMessage(ex, message.Header.Offset); + status = MessageStatus.DecryptionFailed; + error = ex; + } + } + + currentOffset = message.Header.Offset; + batchHandle.Acquire(); + try + { + await PublishRentedAsync(batchHandle, processedMessage, partitionId, status, error, ct); + } + catch + { + batchHandle.Release(); + throw; + } + + anyNewMessages = true; + } + + if (!anyNewMessages + && _config.AutoCommitMode != AutoCommitMode.Disabled) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}", + lastPolledPartitionOffset, rental.PartitionId); + } + + await StoreOffsetAsync(lastPolledPartitionOffset, partitionId, false, ct); + } + + if (anyNewMessages) + { + _lastPolledOffset.AddOrUpdate(rental.PartitionId, currentOffset, (_, _) => currentOffset); + } + + if (_config.PollingStrategy.Kind == MessagePolling.Offset) + { + _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1); + } + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + LogFailedToPollMessages(ex); + _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages")); + } + finally + { + if (batchHandle is not null) + { + batchHandle.Release(); + } + else + { + rental?.Dispose(); + } + + _pollingSemaphore.Release(); + } + } +} diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs index 76264f988c..47349f9cf4 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs @@ -1,536 +1,545 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Collections.Concurrent; -using System.Runtime.CompilerServices; -using System.Threading.Channels; -using Apache.Iggy.Contracts; -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Headers; -using Apache.Iggy.IggyClient; -using Apache.Iggy.Kinds; -using Apache.Iggy.Mappers; -using Apache.Iggy.Utils; -using Microsoft.Extensions.Logging; - -namespace Apache.Iggy.Consumers; - -/// -/// High-level consumer for receiving messages from Iggy streams. -/// Provides automatic polling, offset management, and consumer group support. -/// -public partial class IggyConsumer : IAsyncDisposable -{ - private readonly Channel _channel; - private readonly IIggyClient _client; - private readonly IggyConsumerConfig _config; - private readonly SemaphoreSlim _connectionStateSemaphore = new(1, 1); - private readonly EventAggregator _consumerErrorEvents; - private readonly ConcurrentDictionary _lastPolledOffset = new(); - private readonly ILogger _logger; - private readonly SemaphoreSlim _pollingSemaphore = new(1, 1); - private string? _consumerGroupName; - private int _disposeState; - private volatile bool _isInitialized; - private volatile bool _joinedConsumerGroup; - private long _lastPolledAtMs; - - /// - /// Initializes a new instance of the class - /// - /// The Iggy client for server communication - /// Consumer configuration settings - /// Logger for creating loggers - public IggyConsumer(IIggyClient client, IggyConsumerConfig config, ILoggerFactory loggerFactory) - { - _client = client; - _config = config; - _logger = loggerFactory.CreateLogger(); - - _channel = Channel.CreateUnbounded(); - _consumerErrorEvents = new EventAggregator(loggerFactory); - } - - /// - /// Disposes the consumer, leaving consumer groups and logging out if applicable - /// - public async ValueTask DisposeAsync() - { - if (Interlocked.Exchange(ref _disposeState, 1) == 1) - { - return; - } - - if (_isInitialized) - { - _client.UnsubscribeConnectionEvents(OnClientConnectionStateChangedAsync); - } - - if (!string.IsNullOrEmpty(_consumerGroupName) && _isInitialized) - { - try - { - await _client.LeaveConsumerGroupAsync(_config.StreamId, _config.TopicId, - Identifier.String(_consumerGroupName)); - - LogLeftConsumerGroup(_consumerGroupName); - } - catch (Exception e) - { - LogFailedToLeaveConsumerGroup(e, _consumerGroupName); - } - } - - if (_config.CreateIggyClient && _isInitialized) - { - try - { - await _client.LogoutUserAsync(); - _client.Dispose(); - } - catch (Exception e) - { - LogFailedToLogoutOrDispose(e); - } - } - - _consumerErrorEvents.Clear(); - _pollingSemaphore.Dispose(); - _connectionStateSemaphore.Dispose(); - - } - - /// - /// Initializes the consumer by logging in (if needed) and setting up consumer groups - /// - /// Cancellation token - /// Thrown when consumer group name is invalid - /// Thrown when consumer group doesn't exist and auto-creation is disabled - public async Task InitAsync(CancellationToken ct = default) - { - if (_isInitialized) - { - return; - } - - await _connectionStateSemaphore.WaitAsync(ct); - try - { - if (_isInitialized) - { - return; - } - - if (_config.Consumer.Type == ConsumerType.ConsumerGroup && _config.PartitionId != null) - { - LogPartitionIdIsIgnoredWhenConsumerTypeIsConsumerGroup(); - _config.PartitionId = null; - } - - await _client.ConnectAsync(ct); - - if (_config.CreateIggyClient) - { - await _client.LoginUserAsync(_config.Login, _config.Password, ct); - } - - await InitializeConsumerGroupAsync(ct); - - _client.SubscribeConnectionEvents(OnClientConnectionStateChangedAsync); - - _isInitialized = true; - } - finally - { - _connectionStateSemaphore.Release(); - } - } - - /// - /// Receives messages asynchronously from the consumer as an async stream. - /// Messages are automatically polled from the server and buffered in a bounded channel. - /// - /// Cancellation token to stop receiving messages - /// An async enumerable of received messages - /// Thrown when InitAsync has not been called - public async IAsyncEnumerable ReceiveAsync([EnumeratorCancellation] CancellationToken ct = default) - { - if (!_isInitialized) - { - throw new ConsumerNotInitializedException(); - } - - do - { - if (!_channel.Reader.TryRead(out var message)) - { - await PollMessagesAsync(ct); - continue; - } - - yield return message; - - if (_config.AutoCommitMode == AutoCommitMode.AfterReceive) - { - await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, ct); - } - } while (!ct.IsCancellationRequested); - } - - /// - /// Manually stores the consumer offset for a specific partition. - /// Use this when auto-commit is disabled or when you need manual offset control. - /// - /// The offset to store - /// The partition ID - /// Cancellation token - public async Task StoreOffsetAsync(ulong offset, uint partitionId, CancellationToken ct = default) - { - await _client.StoreOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, offset, partitionId, ct); - } - - /// - /// Deletes the stored consumer offset for a specific partition. - /// The next poll will start from the beginning or based on the polling strategy. - /// - /// The partition ID - /// Cancellation token - public async Task DeleteOffsetAsync(uint partitionId, CancellationToken ct = default) - { - await _client.DeleteOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, partitionId, ct); - } - - /// - /// Event raised when an error occurs during polling. - /// - /// Callback method - public void SubscribeToErrorEvents(Func callback) - { - _consumerErrorEvents.Subscribe(callback); - } - - /// - /// Unsubscribe from error events - /// - /// - public void UnsubscribeFromErrorEvents(Func callback) - { - _consumerErrorEvents.Unsubscribe(callback); - } - - /// - /// Initializes consumer group if configured, creating and joining as needed - /// - private async Task InitializeConsumerGroupAsync(CancellationToken ct = default) - { - if (_joinedConsumerGroup) - { - return; - } - - if (_config.Consumer.Type == ConsumerType.Consumer) - { - _joinedConsumerGroup = true; - return; - } - - _consumerGroupName = _config.Consumer.ConsumerId.Kind == IdKind.String - ? _config.Consumer.ConsumerId.GetString() - : _config.ConsumerGroupName; - - if (string.IsNullOrEmpty(_consumerGroupName)) - { - throw new InvalidConsumerGroupNameException("Consumer group name is empty or null."); - } - - try - { - var existingGroup = await _client.GetConsumerGroupByIdAsync(_config.StreamId, _config.TopicId, - Identifier.String(_consumerGroupName), ct); - - if (existingGroup == null && _config.CreateConsumerGroupIfNotExists) - { - LogCreatingConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId); - - var createdGroup = await TryCreateConsumerGroupAsync(_consumerGroupName, ct); - - if (createdGroup) - { - LogConsumerGroupCreated(_consumerGroupName); - } - } - else if (existingGroup == null) - { - throw new ConsumerGroupNotFoundException(_consumerGroupName); - } - - if (_config.JoinConsumerGroup) - { - LogJoiningConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId); - - await _client.JoinConsumerGroupAsync(_config.StreamId, _config.TopicId, - Identifier.String(_consumerGroupName), ct); - - _joinedConsumerGroup = true; - LogConsumerGroupJoined(_consumerGroupName); - } - } - catch (Exception ex) - { - LogFailedToInitializeConsumerGroup(ex, _consumerGroupName); - throw; - } - } - - /// - /// Attempts to create a consumer group, handling the case where it already exists - /// - /// True if the group was created or already exists, false on error - private async Task TryCreateConsumerGroupAsync(string groupName, CancellationToken ct) - { - try - { - await _client.CreateConsumerGroupAsync(_config.StreamId, _config.TopicId, - groupName, ct); - } - catch (IggyInvalidStatusCodeException ex) - { - // 5004 - Consumer group already exists TODO: refactor errors - if (ex.StatusCode != 5004) - { - LogFailedToCreateConsumerGroup(ex, groupName); - return false; - } - - return true; - } - - return true; - } - - /// - /// Polls messages from the server and writes them to the internal channel. - /// Handles decryption, offset tracking, and auto-commit logic. - /// Uses semaphore to ensure single concurrent polling operation. - /// - private async Task PollMessagesAsync(CancellationToken ct) - { - if (!_joinedConsumerGroup) - { - LogConsumerGroupNotJoinedYetSkippingPolling(); - return; - } - - await _pollingSemaphore.WaitAsync(ct); - - try - { - if (_config.PollingIntervalMs > 0) - { - await WaitBeforePollingAsync(ct); - } - - var messages = await _client.PollMessagesAsync(_config.StreamId, _config.TopicId, - _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize, - _config.AutoCommit, ct); - - var receiveMessages = messages.Messages.Count > 0; - - if (_lastPolledOffset.TryGetValue(messages.PartitionId, out var lastPolledPartitionOffset)) - { - messages.Messages = messages.Messages.Where(x => x.Header.Offset > lastPolledPartitionOffset).ToList(); - } - - if (messages.Messages.Count == 0 - && receiveMessages - && _config.AutoCommitMode != AutoCommitMode.Disabled) - { - _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}", - lastPolledPartitionOffset, messages.PartitionId); - await StoreOffsetAsync(lastPolledPartitionOffset, (uint)messages.PartitionId, ct); - } - - if (messages.Messages.Count == 0) - { - return; - } - - var currentOffset = 0ul; - foreach (var message in messages.Messages) - { - var processedMessage = message; - var status = MessageStatus.Success; - Exception? error = null; - - if (_config.MessageEncryptor != null) - { - try - { - var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload); - - Dictionary? decryptedHeaders = null; - if (message.RawUserHeaders is { Length: > 0 }) - { - var decryptedHeaderBytes = _config.MessageEncryptor.Decrypt(message.RawUserHeaders); - decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes); - } - - processedMessage = new MessageResponse - { - Header = message.Header, - Payload = decryptedPayload, - UserHeaders = decryptedHeaders - }; - } - catch (Exception ex) - { - LogFailedToDecryptMessage(ex, message.Header.Offset); - status = MessageStatus.DecryptionFailed; - error = ex; - } - } - - var receivedMessage = new ReceivedMessage - { - Message = processedMessage, - CurrentOffset = processedMessage.Header.Offset, - PartitionId = (uint)messages.PartitionId, - Status = status, - Error = error - }; - - await _channel.Writer.WriteAsync(receivedMessage, ct); - currentOffset = receivedMessage.CurrentOffset; - } - - _lastPolledOffset.AddOrUpdate(messages.PartitionId, currentOffset, - (_, _) => currentOffset); - - if (_config.PollingStrategy.Kind == MessagePolling.Offset) - { - _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1); - } - } - catch (Exception ex) - { - LogFailedToPollMessages(ex); - _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages")); - } - finally - { - _pollingSemaphore.Release(); - } - } - - /// - /// Implements polling interval throttling to avoid excessive server requests. - /// Uses monotonic time tracking to ensure proper intervals even with clock adjustments. - /// - private async Task WaitBeforePollingAsync(CancellationToken ct) - { - var intervalMs = _config.PollingIntervalMs; - if (intervalMs <= 0) - { - return; - } - - var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - var lastPolledAtMs = Interlocked.Read(ref _lastPolledAtMs); - - if (nowMs < lastPolledAtMs) - { - LogMonotonicTimeWentBackwards(nowMs, lastPolledAtMs); - await Task.Delay(intervalMs, ct); - Interlocked.Exchange(ref _lastPolledAtMs, nowMs); - return; - } - - var elapsedMs = nowMs - lastPolledAtMs; - if (elapsedMs >= intervalMs) - { - LogNoNeedToWaitBeforePolling(nowMs, lastPolledAtMs, elapsedMs); - Interlocked.Exchange(ref _lastPolledAtMs, nowMs); - return; - } - - var remainingMs = intervalMs - elapsedMs; - LogWaitingBeforePolling(remainingMs); - - if (remainingMs > 0) - { - await Task.Delay((int)remainingMs, ct); - } - - Interlocked.Exchange(ref _lastPolledAtMs, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); - } - - /// - /// Handles connection state changes from the client. - /// - /// Event object - private async Task OnClientConnectionStateChangedAsync(ConnectionStateChangedEventArgs e) - { - LogConnectionStateChanged(e.PreviousState, e.CurrentState); - - await _connectionStateSemaphore.WaitAsync(); - try - { - if (e.CurrentState == ConnectionState.Disconnected) - { - _joinedConsumerGroup = false; - } - - if (e.CurrentState != ConnectionState.Authenticated - || e.PreviousState == ConnectionState.Authenticated - || _joinedConsumerGroup) - { - return; - } - - await RejoinConsumerGroupOnReconnectionAsync(); - } - finally - { - _connectionStateSemaphore.Release(); - } - } - - /// - /// Asynchronously rejoins the consumer group after a client reconnection. - /// This restores the consumer group membership that was lost during the connection failure. - /// - private async Task RejoinConsumerGroupOnReconnectionAsync() - { - if (string.IsNullOrEmpty(_consumerGroupName)) - { - LogConsumerGroupNameIsEmptySkippingRejoiningConsumerGroup(); - return; - } - - try - { - await InitializeConsumerGroupAsync(); - } - catch (Exception ex) - { - LogFailedToRejoinConsumerGroup(ex, _consumerGroupName); - _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, - "Failed to rejoin consumer group after reconnection")); - } - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Apache.Iggy.Contracts; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; +using Apache.Iggy.Mappers; +using Apache.Iggy.Utils; +using Microsoft.Extensions.Logging; + +namespace Apache.Iggy.Consumers; + +/// +/// High-level consumer for receiving messages from Iggy streams. +/// Provides automatic polling, offset management, and consumer group support. +/// +public partial class IggyConsumer : IAsyncDisposable +{ + private readonly Channel _channel; + private readonly IIggyClient _client; + private readonly IggyConsumerConfig _config; + private readonly SemaphoreSlim _connectionStateSemaphore = new(1, 1); + private readonly EventAggregator _consumerErrorEvents; + private readonly ConcurrentDictionary _lastPolledOffset = new(); + private readonly ILogger _logger; + private readonly SemaphoreSlim _pollingSemaphore = new(1, 1); + private string? _consumerGroupName; + private int _disposeState; + private volatile bool _isInitialized; + private volatile bool _joinedConsumerGroup; + private long _lastPolledAtMs; + + /// Whether this consumer has been initialized via . + protected bool IsInitialized => _isInitialized; + + /// + /// Initializes a new instance of the class + /// + /// The Iggy client for server communication + /// Consumer configuration settings + /// Logger for creating loggers + public IggyConsumer(IIggyClient client, IggyConsumerConfig config, ILoggerFactory loggerFactory) + { + _client = client; + _config = config; + _logger = loggerFactory.CreateLogger(); + + _channel = Channel.CreateUnbounded(); + _consumerErrorEvents = new EventAggregator(loggerFactory); + } + + /// + /// Disposes the consumer, leaving consumer groups and logging out if applicable + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposeState, 1) == 1) + { + return; + } + + if (_isInitialized) + { + _client.UnsubscribeConnectionEvents(OnClientConnectionStateChangedAsync); + } + + if (!string.IsNullOrEmpty(_consumerGroupName) && _isInitialized) + { + try + { + await _client.LeaveConsumerGroupAsync(_config.StreamId, _config.TopicId, + Identifier.String(_consumerGroupName)); + + LogLeftConsumerGroup(_consumerGroupName); + } + catch (Exception e) + { + LogFailedToLeaveConsumerGroup(e, _consumerGroupName); + } + } + + if (_config.CreateIggyClient && _isInitialized) + { + try + { + await _client.LogoutUserAsync(); + _client.Dispose(); + } + catch (Exception e) + { + LogFailedToLogoutOrDispose(e); + } + } + + _consumerErrorEvents.Clear(); + _pollingSemaphore.Dispose(); + _connectionStateSemaphore.Dispose(); + } + + /// + /// Initializes the consumer by logging in (if needed) and setting up consumer groups + /// + /// Cancellation token + /// Thrown when consumer group name is invalid + /// Thrown when consumer group doesn't exist and auto-creation is disabled + public async Task InitAsync(CancellationToken ct = default) + { + if (_isInitialized) + { + return; + } + + await _connectionStateSemaphore.WaitAsync(ct); + try + { + if (_isInitialized) + { + return; + } + + if (_config.Consumer.Type == ConsumerType.ConsumerGroup && _config.PartitionId != null) + { + LogPartitionIdIsIgnoredWhenConsumerTypeIsConsumerGroup(); + _config.PartitionId = null; + } + + await _client.ConnectAsync(ct); + + if (_config.CreateIggyClient) + { + await _client.LoginUserAsync(_config.Login, _config.Password, ct); + } + + await InitializeConsumerGroupAsync(ct); + + _client.SubscribeConnectionEvents(OnClientConnectionStateChangedAsync); + + _isInitialized = true; + } + finally + { + _connectionStateSemaphore.Release(); + } + } + + /// + /// Receives messages asynchronously from the consumer as an async stream. + /// Messages are automatically polled from the server and buffered in a bounded channel. + /// + /// Cancellation token to stop receiving messages + /// An async enumerable of received messages + /// Thrown when InitAsync has not been called + public async IAsyncEnumerable ReceiveAsync([EnumeratorCancellation] CancellationToken ct = default) + { + if (!_isInitialized) + { + throw new ConsumerNotInitializedException(); + } + + do + { + if (!_channel.Reader.TryRead(out var message)) + { + await PollMessagesAsync(ct); + continue; + } + + yield return message; + + if (_config.AutoCommitMode == AutoCommitMode.AfterReceive) + { + await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, false, ct); + } + } while (!ct.IsCancellationRequested); + } + + /// + /// Manually stores the consumer offset for a specific partition. + /// Use this when auto-commit is disabled or when you need manual offset control. + /// + /// The offset to store + /// The partition ID + /// + /// Cancellation token + public async Task StoreOffsetAsync(ulong offset, uint partitionId, bool resetLastPooled = false, + CancellationToken ct = default) + { + await _client.StoreOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, offset, partitionId, ct); + + if (resetLastPooled) + { + _lastPolledOffset[(int)partitionId] = offset; + } + } + + /// + /// Deletes the stored consumer offset for a specific partition. + /// The next poll will start from the beginning or based on the polling strategy. + /// + /// The partition ID + /// Cancellation token + public async Task DeleteOffsetAsync(uint partitionId, CancellationToken ct = default) + { + await _client.DeleteOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, partitionId, ct); + } + + /// + /// Event raised when an error occurs during polling. + /// + /// Callback method + public void SubscribeToErrorEvents(Func callback) + { + _consumerErrorEvents.Subscribe(callback); + } + + /// + /// Unsubscribe from error events + /// + /// + public void UnsubscribeFromErrorEvents(Func callback) + { + _consumerErrorEvents.Unsubscribe(callback); + } + + /// + /// Initializes consumer group if configured, creating and joining as needed + /// + private async Task InitializeConsumerGroupAsync(CancellationToken ct = default) + { + if (_joinedConsumerGroup) + { + return; + } + + if (_config.Consumer.Type == ConsumerType.Consumer) + { + _joinedConsumerGroup = true; + return; + } + + _consumerGroupName = _config.Consumer.ConsumerId.Kind == IdKind.String + ? _config.Consumer.ConsumerId.GetString() + : _config.ConsumerGroupName; + + if (string.IsNullOrEmpty(_consumerGroupName)) + { + throw new InvalidConsumerGroupNameException("Consumer group name is empty or null."); + } + + try + { + var existingGroup = await _client.GetConsumerGroupByIdAsync(_config.StreamId, _config.TopicId, + Identifier.String(_consumerGroupName), ct); + + if (existingGroup == null && _config.CreateConsumerGroupIfNotExists) + { + LogCreatingConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId); + + var createdGroup = await TryCreateConsumerGroupAsync(_consumerGroupName, ct); + + if (createdGroup) + { + LogConsumerGroupCreated(_consumerGroupName); + } + } + else if (existingGroup == null) + { + throw new ConsumerGroupNotFoundException(_consumerGroupName); + } + + if (_config.JoinConsumerGroup) + { + LogJoiningConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId); + + await _client.JoinConsumerGroupAsync(_config.StreamId, _config.TopicId, + Identifier.String(_consumerGroupName), ct); + + _joinedConsumerGroup = true; + LogConsumerGroupJoined(_consumerGroupName); + } + } + catch (Exception ex) + { + LogFailedToInitializeConsumerGroup(ex, _consumerGroupName); + throw; + } + } + + /// + /// Attempts to create a consumer group, handling the case where it already exists + /// + /// True if the group was created or already exists, false on error + private async Task TryCreateConsumerGroupAsync(string groupName, CancellationToken ct) + { + try + { + await _client.CreateConsumerGroupAsync(_config.StreamId, _config.TopicId, + groupName, ct); + } + catch (IggyInvalidStatusCodeException ex) + { + // 5004 - Consumer group already exists TODO: refactor errors + if (ex.StatusCode != 5004) + { + LogFailedToCreateConsumerGroup(ex, groupName); + return false; + } + + return true; + } + + return true; + } + + /// + /// Polls messages from the server and writes them to the internal channel. + /// Handles decryption, offset tracking, and auto-commit logic. + /// Uses semaphore to ensure single concurrent polling operation. + /// + private async Task PollMessagesAsync(CancellationToken ct) + { + if (!_joinedConsumerGroup) + { + LogConsumerGroupNotJoinedYetSkippingPolling(); + return; + } + + await _pollingSemaphore.WaitAsync(ct); + + try + { + if (_config.PollingIntervalMs > 0) + { + await WaitBeforePollingAsync(ct); + } + + var messages = await _client.PollMessagesAsync(_config.StreamId, _config.TopicId, + _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize, + _config.AutoCommit, ct); + + var receiveMessages = messages.Messages.Count > 0; + + if (_lastPolledOffset.TryGetValue(messages.PartitionId, out var lastPolledPartitionOffset)) + { + messages.Messages = messages.Messages.Where(x => x.Header.Offset > lastPolledPartitionOffset).ToList(); + } + + if (messages.Messages.Count == 0 + && receiveMessages + && _config.AutoCommitMode != AutoCommitMode.Disabled) + { + _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}", + lastPolledPartitionOffset, messages.PartitionId); + await StoreOffsetAsync(lastPolledPartitionOffset, (uint)messages.PartitionId, false, ct); + } + + if (messages.Messages.Count == 0) + { + return; + } + + var currentOffset = 0ul; + foreach (var message in messages.Messages) + { + var processedMessage = message; + var status = MessageStatus.Success; + Exception? error = null; + + if (_config.MessageEncryptor != null) + { + try + { + var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload); + + Dictionary? decryptedHeaders = null; + if (message.RawUserHeaders is { Length: > 0 }) + { + var decryptedHeaderBytes = _config.MessageEncryptor.Decrypt(message.RawUserHeaders); + decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes); + } + + processedMessage = new MessageResponse + { + Header = message.Header, + Payload = decryptedPayload, + UserHeaders = decryptedHeaders + }; + } + catch (Exception ex) + { + LogFailedToDecryptMessage(ex, message.Header.Offset); + status = MessageStatus.DecryptionFailed; + error = ex; + } + } + + var receivedMessage = new ReceivedMessage + { + Message = processedMessage, + CurrentOffset = processedMessage.Header.Offset, + PartitionId = (uint)messages.PartitionId, + Status = status, + Error = error + }; + + await _channel.Writer.WriteAsync(receivedMessage, ct); + currentOffset = receivedMessage.CurrentOffset; + } + + _lastPolledOffset.AddOrUpdate(messages.PartitionId, currentOffset, + (_, _) => currentOffset); + + if (_config.PollingStrategy.Kind == MessagePolling.Offset) + { + _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1); + } + } + catch (Exception ex) + { + LogFailedToPollMessages(ex); + _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages")); + } + finally + { + _pollingSemaphore.Release(); + } + } + + /// + /// Implements polling interval throttling to avoid excessive server requests. + /// Uses monotonic time tracking to ensure proper intervals even with clock adjustments. + /// + private async Task WaitBeforePollingAsync(CancellationToken ct) + { + var intervalMs = _config.PollingIntervalMs; + if (intervalMs <= 0) + { + return; + } + + var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var lastPolledAtMs = Interlocked.Read(ref _lastPolledAtMs); + + if (nowMs < lastPolledAtMs) + { + LogMonotonicTimeWentBackwards(nowMs, lastPolledAtMs); + await Task.Delay(intervalMs, ct); + Interlocked.Exchange(ref _lastPolledAtMs, nowMs); + return; + } + + var elapsedMs = nowMs - lastPolledAtMs; + if (elapsedMs >= intervalMs) + { + LogNoNeedToWaitBeforePolling(nowMs, lastPolledAtMs, elapsedMs); + Interlocked.Exchange(ref _lastPolledAtMs, nowMs); + return; + } + + var remainingMs = intervalMs - elapsedMs; + LogWaitingBeforePolling(remainingMs); + + if (remainingMs > 0) + { + await Task.Delay((int)remainingMs, ct); + } + + Interlocked.Exchange(ref _lastPolledAtMs, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + } + + /// + /// Handles connection state changes from the client. + /// + /// Event object + private async Task OnClientConnectionStateChangedAsync(ConnectionStateChangedEventArgs e) + { + LogConnectionStateChanged(e.PreviousState, e.CurrentState); + + await _connectionStateSemaphore.WaitAsync(); + try + { + if (e.CurrentState == ConnectionState.Disconnected) + { + _joinedConsumerGroup = false; + } + + if (e.CurrentState != ConnectionState.Authenticated + || e.PreviousState == ConnectionState.Authenticated + || _joinedConsumerGroup) + { + return; + } + + await RejoinConsumerGroupOnReconnectionAsync(); + } + finally + { + _connectionStateSemaphore.Release(); + } + } + + /// + /// Asynchronously rejoins the consumer group after a client reconnection. + /// This restores the consumer group membership that was lost during the connection failure. + /// + private async Task RejoinConsumerGroupOnReconnectionAsync() + { + if (string.IsNullOrEmpty(_consumerGroupName)) + { + LogConsumerGroupNameIsEmptySkippingRejoiningConsumerGroup(); + return; + } + + try + { + await InitializeConsumerGroupAsync(); + } + catch (Exception ex) + { + LogFailedToRejoinConsumerGroup(ex, _consumerGroupName); + _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, + "Failed to rejoin consumer group after reconnection")); + } + } +} diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs index afaa8cd94b..b8ea65b622 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs @@ -1,107 +1,193 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Runtime.CompilerServices; -using Apache.Iggy.IggyClient; -using Microsoft.Extensions.Logging; - -namespace Apache.Iggy.Consumers; - -/// -/// Typed consumer that automatically deserializes message payloads to type T. -/// Extends with deserialization capabilities. -/// -/// The type to deserialize message payloads to -public class IggyConsumer : IggyConsumer -{ - private readonly IggyConsumerConfig _typedConfig; - private readonly ILogger> _typedLogger; - - /// - /// Initializes a new instance of the typed class - /// - /// The Iggy client for server communication - /// Typed consumer configuration including deserializer - /// Logger instance for diagnostic output - public IggyConsumer(IIggyClient client, IggyConsumerConfig config, ILoggerFactory logger) : base( - client, config, logger) - { - _typedConfig = config; - _typedLogger = logger.CreateLogger>(); - } - - /// - /// Receives and deserializes messages from the consumer - /// - /// Cancellation token - /// Async enumerable of deserialized messages with status - public async IAsyncEnumerable> ReceiveDeserializedAsync( - [EnumeratorCancellation] CancellationToken ct = default) - { - await foreach (var message in ReceiveAsync(ct)) - { - if (message.Status != MessageStatus.Success) - { - yield return new ReceivedMessage - { - Data = default, - Message = message.Message, - CurrentOffset = message.CurrentOffset, - PartitionId = message.PartitionId, - Status = message.Status, - Error = message.Error - }; - continue; - } - - T? deserializedPayload = default; - Exception? deserializationError = null; - var status = MessageStatus.Success; - - try - { - deserializedPayload = Deserialize(message.Message.Payload); - } - catch (Exception ex) - { - _typedLogger.LogError(ex, "Failed to deserialize message at offset {Offset}", message.CurrentOffset); - status = MessageStatus.DeserializationFailed; - deserializationError = ex; - } - - yield return new ReceivedMessage - { - Data = deserializedPayload, - Message = message.Message, - CurrentOffset = message.CurrentOffset, - PartitionId = message.PartitionId, - Status = status, - Error = deserializationError - }; - } - } - - /// - /// Deserializes a message payload using the configured deserializer - /// - /// The raw byte array payload to deserialize - /// The deserialized object of type T - public T Deserialize(byte[] payload) - { - return _typedConfig.Deserializer.Deserialize(payload); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Apache.Iggy.Contracts; +using Apache.Iggy.Exceptions; +using Apache.Iggy.IggyClient; +using Microsoft.Extensions.Logging; + +namespace Apache.Iggy.Consumers; + +/// +/// Typed consumer that automatically deserializes message payloads to type T. +/// Extends with deserialization capabilities. +/// +/// The type to deserialize message payloads to +public class IggyConsumer : IggyConsumer +{ + private readonly Channel> _deserializedChannel = + Channel.CreateUnbounded>(); + + private readonly IggyConsumerConfig _typedConfig; + private readonly ILogger> _typedLogger; + + /// + /// Initializes a new instance of the typed class + /// + /// The Iggy client for server communication + /// Typed consumer configuration including deserializer + /// Logger instance for diagnostic output + public IggyConsumer(IIggyClient client, IggyConsumerConfig config, ILoggerFactory logger) : base(client, config, + logger) + { + _typedConfig = config; + _typedLogger = logger.CreateLogger>(); + } + + /// + /// Receives and deserializes messages from the consumer + /// + /// Cancellation token + /// Async enumerable of deserialized messages with status + public async IAsyncEnumerable> ReceiveDeserializedAsync( + [EnumeratorCancellation] CancellationToken ct = default) + { + await foreach (var message in ReceiveAsync(ct)) + { + if (message.Status != MessageStatus.Success) + { + yield return new ReceivedMessage + { + Data = default, + Message = message.Message, + CurrentOffset = message.CurrentOffset, + PartitionId = message.PartitionId, + Status = message.Status, + Error = message.Error + }; + continue; + } + + T? deserializedPayload = default; + Exception? deserializationError = null; + var status = MessageStatus.Success; + + try + { + deserializedPayload = Deserialize(message.Message.Payload); + } + catch (Exception ex) + { + _typedLogger.LogError(ex, "Failed to deserialize message at offset {Offset}", message.CurrentOffset); + status = MessageStatus.DeserializationFailed; + deserializationError = ex; + } + + yield return new ReceivedMessage + { + Data = deserializedPayload, + Message = message.Message, + CurrentOffset = message.CurrentOffset, + PartitionId = message.PartitionId, + Status = status, + Error = deserializationError + }; + } + } + + /// + /// Receives and deserializes messages via the rented poll path. Each polled batch is deserialized + /// in full before any message is yielded — the rented buffer is returned immediately after + /// deserialization, independently of how fast the caller iterates. The caller does not need to + /// dispose anything. + /// + /// Cancellation token. + /// Async enumerable of deserialized messages with status. + public async IAsyncEnumerable> ReceiveRentedDeserializedAsync( + [EnumeratorCancellation] CancellationToken ct = default) + { + if (!IsInitialized) + { + throw new ConsumerNotInitializedException(); + } + + do + { + if (!_deserializedChannel.Reader.TryRead(out DeserializedMessage? message)) + { + await PollRentedMessagesAsync(ct); + continue; + } + + yield return message; + + if (_typedConfig.AutoCommitMode == AutoCommitMode.AfterReceive) + { + await StoreOffsetAsync(message.Header.Offset, message.PartitionId, false, ct); + } + } while (!ct.IsCancellationRequested); + } + + /// + /// Overrides the base batch-publishing step: instead of routing rented messages through the base + /// class channel, deserializes the entire batch immediately (releasing all rented buffer refs via + /// ) and writes the deserialized results to + /// . Auto-commit is also handled here since the base-class + /// yield path is bypassed. + /// + protected override async Task PublishRentedAsync(RentedBatchHandle rental, RentedMessageResponse message, + uint partitionId, MessageStatus status, + Exception? error, CancellationToken ct) + { + T? data = default; + var deserError = status != MessageStatus.Success ? error : null; + var msgStatus = status; + + if (status == MessageStatus.Success) + { + try + { + data = Deserialize(message.Payload); + } + catch (Exception ex) + { + _typedLogger.LogError(ex, "Failed to deserialize message at offset {Offset}", + message.Header.Offset); + msgStatus = MessageStatus.DeserializationFailed; + deserError = ex; + } + } + + var deserialized = new DeserializedMessage + { + Data = data, + Header = message.Header, + UserHeaders = message.UserHeaders, + CurrentOffset = message.Header.Offset, + PartitionId = partitionId, + Status = msgStatus, + Error = deserError + }; + + await _deserializedChannel.Writer.WriteAsync(deserialized, ct); + + rental.Release(); + } + + /// + /// Deserializes a message payload from a span using the configured deserializer. Zero-copy when the + /// deserializer overrides the span overload; otherwise falls back to a one-time array copy. + /// + /// The payload memory to deserialize. + /// The deserialized object of type T. + public T Deserialize(ReadOnlyMemory payload) + { + return _typedConfig.Deserializer.Deserialize(payload); + } +} diff --git a/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs b/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs new file mode 100644 index 0000000000..ac0f92e6c1 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Contracts; + +namespace Apache.Iggy.Consumers; + +/// +/// Represents a message received from the Iggy consumer whose payload and raw headers are backed by rented memory +/// shared across a polled batch. The caller SHOULD dispose the message once processing is complete for deterministic +/// release of the underlying pool buffer. If the caller forgets, the buffer is still returned to the pool when the +/// owning becomes unreachable and its finalizer runs - non-deterministic but safe. +/// +public sealed class ReceivedRentedMessage : IDisposable +{ + private int _disposed; + internal RentedBatchHandle? Handle { get; init; } + + /// + /// The underlying rented message response containing headers and rented payload memory. + /// + public required RentedMessageResponse Message { get; init; } + + /// + /// The current offset of this message in the partition. + /// + public required ulong CurrentOffset { get; init; } + + /// + /// The partition ID from which this message was consumed. + /// + public uint PartitionId { get; init; } + + /// + /// The status of the message (Success, DecryptionFailed). + /// + public MessageStatus Status { get; init; } = MessageStatus.Success; + + /// + /// The exception that occurred during processing, if any. + /// + public Exception? Error { get; init; } + + /// + /// Releases this message's reference on the underlying rental. When the final message of a batch is disposed, + /// the rented buffer is returned to the pool and any payload/raw header slices are invalidated. + /// + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + Handle?.Release(); + } +} + +/// +/// Reference-counted handle around a . Shared by all +/// instances produced from one poll; the rental is disposed +/// when the reference count drops to zero. +/// +public sealed class RentedBatchHandle : IDisposable +{ + private readonly PolledMessagesRental _rental; + private int _refCount = 1; + + /// + /// Creates a new handle with a single self-reference held by the constructing producer. The producer + /// must call before each publish and release the self-reference (via + /// or ) once it has finished producing. + /// + /// The polled messages rental whose lifetime this handle manages. + public RentedBatchHandle(PolledMessagesRental rental) + { + _rental = rental; + } + + /// + /// Releases one reference on the underlying rental. Equivalent to . + /// + public void Dispose() + { + Release(); + } + + /// + /// Acquires an additional reference. Must be balanced by a matching . + /// + public void Acquire() + { + Interlocked.Increment(ref _refCount); + } + + /// + /// Decrements the reference count. When the count reaches zero, the underlying rental is disposed + /// and its pool buffer returned. + /// + public void Release() + { + var remaining = Interlocked.Decrement(ref _refCount); + if (remaining == 0) + { + _rental.Dispose(); + } + else if (remaining < 0) + { + throw new InvalidOperationException("RentedBatchHandle released more times than acquired."); + } + } +} diff --git a/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs index 9a5eb55afd..7895e7921d 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs @@ -1,52 +1,96 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Text.Json.Serialization; -using Apache.Iggy.Headers; -using Apache.Iggy.JsonConverters; -using Apache.Iggy.Messages; - -namespace Apache.Iggy.Contracts; - -/// -/// Response from the server containing a message payload. -/// -[JsonConverter(typeof(MessageResponseConverter))] -public sealed class MessageResponse -{ - /// - /// Message header. - /// - public required MessageHeader Header { get; set; } - - /// - /// Message payload. - /// - public required byte[] Payload { get; set; } = []; - - /// - /// Headers defined by the user. - /// - public Dictionary? UserHeaders { get; set; } - - /// - /// Raw user header bytes before deserialization. - /// Used internally for decrypting encrypted headers. - /// - [JsonIgnore] - internal byte[]? RawUserHeaders { get; set; } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text.Json.Serialization; +using Apache.Iggy.Headers; +using Apache.Iggy.JsonConverters; +using Apache.Iggy.Mappers; +using Apache.Iggy.Messages; + +namespace Apache.Iggy.Contracts; + +/// +/// Response from the server containing a message payload. +/// +[JsonConverter(typeof(MessageResponseConverter))] +public sealed class MessageResponse +{ + private byte[]? _rawUserHeaders; + private Dictionary? _userHeaders; + private bool _userHeadersInitialized; + + /// + /// Message header. + /// + public required MessageHeader Header { get; set; } + + /// + /// Message payload. + /// + public required byte[] Payload { get; set; } = []; + + /// + /// Headers defined by the user. + /// + public Dictionary? UserHeaders + { + get + { + if (!_userHeadersInitialized) + { + _userHeaders = _rawUserHeaders is { Length: > 0 } + ? BinaryMapper.TryMapHeaders(_rawUserHeaders) + : null; + _userHeadersInitialized = true; + } + + return _userHeaders; + } + set + { + _userHeaders = value; + _userHeadersInitialized = true; + } + } + + /// + /// Raw user header bytes before deserialization. + /// Used internally for decrypting encrypted headers. + /// + [JsonIgnore] + internal byte[]? RawUserHeaders + { + get => _rawUserHeaders; + set + { + _rawUserHeaders = value; + _userHeaders = null; + _userHeadersInitialized = false; + } + } + + internal void ParseUserHeaders() + { + if (!_userHeadersInitialized) + { + _userHeaders = _rawUserHeaders is { Length: > 0 } + ? BinaryMapper.TryMapHeaders(_rawUserHeaders) + : null; + _userHeadersInitialized = true; + } + } +} diff --git a/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs b/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs new file mode 100644 index 0000000000..04a82834bf --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers; + +namespace Apache.Iggy.Contracts; + +/// +/// Represents a rented poll result whose payload and raw header memory remain valid until disposed. +/// +public sealed class PolledMessagesRental : IDisposable +{ + private readonly IMemoryOwner _owner; + private int _disposed; + + /// + /// Partition identifier for the messages. + /// + public required int PartitionId { get; init; } + + /// + /// Current offset for the partition. + /// + public required ulong CurrentOffset { get; init; } + + /// + /// Rented messages. + /// + public required IReadOnlyList Messages { get; init; } + + internal PolledMessagesRental(IMemoryOwner owner) + { + _owner = owner; + } + + /// + /// Disposes the rental and returns the underlying buffer to the pool. + /// + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + _owner.Dispose(); + GC.SuppressFinalize(this); + } + + /// + /// Finalizer fallback that returns the buffer to the pool if the caller forgot to dispose. + /// + ~PolledMessagesRental() + { + Dispose(); + } +} diff --git a/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs new file mode 100644 index 0000000000..43c629380e --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Headers; +using Apache.Iggy.Mappers; +using Apache.Iggy.Messages; + +namespace Apache.Iggy.Contracts; + +/// +/// Response containing rented payload and raw header memory. +/// The payload and raw headers are only valid while the owning is alive. +/// +public sealed class RentedMessageResponse +{ + private Dictionary? _userHeaders; + private bool _userHeadersInitialized; + + /// + /// Message header. + /// + public required MessageHeader Header { get; init; } + + /// + /// Message payload backed by rented memory. + /// + public required ReadOnlyMemory Payload { get; init; } + + /// + /// Raw user header bytes backed by rented memory. + /// + public ReadOnlyMemory RawUserHeaders { get; init; } + + /// + /// Parsed user headers. Parsed lazily and cached on first access. + /// + public Dictionary? UserHeaders + { + get + { + if (!_userHeadersInitialized) + { + _userHeaders = RawUserHeaders.IsEmpty + ? null + : BinaryMapper.TryMapHeaders(RawUserHeaders.Span); + _userHeadersInitialized = true; + } + + return _userHeaders; + } + init + { + _userHeaders = value; + _userHeadersInitialized = true; + } + } +} diff --git a/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs b/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs index 8bd8ec2a6f..6aaf4d8224 100644 --- a/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs +++ b/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs @@ -1,128 +1,129 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Security.Cryptography; - -namespace Apache.Iggy.Encryption; - -/// -/// AES-256-GCM based message encryptor for secure message encryption/decryption. -/// Uses AES-GCM (Galois/Counter Mode) which provides both confidentiality and authenticity. -/// -public sealed class AesMessageEncryptor : IMessageEncryptor -{ - private readonly byte[] _key; - private const int NonceSize = 12; // 96 bits - recommended for GCM - private const int TagSize = 16; // 128 bits authentication tag - - /// - /// Creates a new AES message encryptor with the specified key. - /// - /// The encryption key. Must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256 respectively. - /// Thrown when key length is invalid - public AesMessageEncryptor(byte[] key) - { - if (key.Length != 16 && key.Length != 24 && key.Length != 32) - { - throw new ArgumentException("Key must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256", nameof(key)); - } - - _key = key; - } - - /// - /// Creates a new AES-256 message encryptor with the specified base64-encoded key. - /// - /// The base64-encoded encryption key - /// A new AesMessageEncryptor instance - public static AesMessageEncryptor FromBase64Key(string base64Key) - { - return new AesMessageEncryptor(Convert.FromBase64String(base64Key)); - } - - /// - /// Generates a new random AES-256 key. - /// - /// A 32-byte random key suitable for AES-256 - public static byte[] GenerateKey() - { - var key = new byte[32]; - using var rng = RandomNumberGenerator.Create(); - rng.GetBytes(key); - return key; - } - - /// - /// Encrypts the provided plain data using AES-GCM. - /// Format: [12-byte nonce][encrypted data][16-byte authentication tag] - /// - /// The plain data to encrypt - /// The encrypted data with nonce and tag - public byte[] Encrypt(byte[] plainData) - { - using var aesGcm = new AesGcm(_key, TagSize); - - var nonce = new byte[NonceSize]; - RandomNumberGenerator.Fill(nonce); - - var ciphertext = new byte[plainData.Length]; - var tag = new byte[TagSize]; - - aesGcm.Encrypt(nonce, plainData, ciphertext, tag); - - // Combine: nonce + ciphertext + tag - var result = new byte[NonceSize + ciphertext.Length + TagSize]; - Buffer.BlockCopy(nonce, 0, result, 0, NonceSize); - Buffer.BlockCopy(ciphertext, 0, result, NonceSize, ciphertext.Length); - Buffer.BlockCopy(tag, 0, result, NonceSize + ciphertext.Length, TagSize); - - return result; - } - - /// - /// Decrypts the provided encrypted data using AES-GCM. - /// Expected format: [12-byte nonce][encrypted data][16-byte authentication tag] - /// - /// The encrypted data with nonce and tag - /// The decrypted plain data - /// Thrown when encrypted data format is invalid - /// Thrown when decryption or authentication fails - public byte[] Decrypt(byte[] encryptedData) - { - if (encryptedData.Length < NonceSize + TagSize) - { - throw new ArgumentException("Encrypted data is too short to contain nonce and tag", nameof(encryptedData)); - } - - using var aesGcm = new AesGcm(_key, TagSize); - - // Extract nonce, ciphertext, and tag - var nonce = new byte[NonceSize]; - var tag = new byte[TagSize]; - var ciphertextLength = encryptedData.Length - NonceSize - TagSize; - var ciphertext = new byte[ciphertextLength]; - - Buffer.BlockCopy(encryptedData, 0, nonce, 0, NonceSize); - Buffer.BlockCopy(encryptedData, NonceSize, ciphertext, 0, ciphertextLength); - Buffer.BlockCopy(encryptedData, NonceSize + ciphertextLength, tag, 0, TagSize); - - var plaintext = new byte[ciphertextLength]; - aesGcm.Decrypt(nonce, ciphertext, tag, plaintext); - - return plaintext; - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Security.Cryptography; + +namespace Apache.Iggy.Encryption; + +/// +/// AES-256-GCM based message encryptor for secure message encryption/decryption. +/// Uses AES-GCM (Galois/Counter Mode) which provides both confidentiality and authenticity. +/// +public sealed class AesMessageEncryptor : IMessageEncryptor +{ + private const int NonceSize = 12; // 96 bits - recommended for GCM + private const int TagSize = 16; // 128 bits authentication tag + private readonly byte[] _key; + + /// + /// Creates a new AES message encryptor with the specified key. + /// + /// The encryption key. Must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256 respectively. + /// Thrown when key length is invalid + public AesMessageEncryptor(byte[] key) + { + if (key.Length != 16 && key.Length != 24 && key.Length != 32) + { + throw new ArgumentException("Key must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256", + nameof(key)); + } + + _key = key; + } + + /// + /// Encrypts the provided plain data using AES-GCM. + /// Format: [12-byte nonce][encrypted data][16-byte authentication tag] + /// + /// The plain data to encrypt + /// The encrypted data with nonce and tag + public byte[] Encrypt(Span plainData) + { + using var aesGcm = new AesGcm(_key, TagSize); + + var nonce = new byte[NonceSize]; + RandomNumberGenerator.Fill(nonce); + + var ciphertext = new byte[plainData.Length]; + var tag = new byte[TagSize]; + + aesGcm.Encrypt(nonce, plainData, ciphertext, tag); + + // Combine: nonce + ciphertext + tag + var result = new byte[NonceSize + ciphertext.Length + TagSize]; + Buffer.BlockCopy(nonce, 0, result, 0, NonceSize); + Buffer.BlockCopy(ciphertext, 0, result, NonceSize, ciphertext.Length); + Buffer.BlockCopy(tag, 0, result, NonceSize + ciphertext.Length, TagSize); + + return result; + } + + /// + /// Decrypts the provided encrypted data using AES-GCM. + /// Expected format: [12-byte nonce][encrypted data][16-byte authentication tag] + /// + /// The encrypted data with nonce and tag + /// The decrypted plain data + /// Thrown when encrypted data format is invalid + /// Thrown when decryption or authentication fails + public byte[] Decrypt(ReadOnlySpan encryptedData) + { + if (encryptedData.Length < NonceSize + TagSize) + { + throw new ArgumentException("Encrypted data is too short to contain nonce and tag", nameof(encryptedData)); + } + + using var aesGcm = new AesGcm(_key, TagSize); + + // Extract nonce, ciphertext, and tag + Span nonce = stackalloc byte[NonceSize]; + Span tag = stackalloc byte[TagSize]; + var ciphertextLength = encryptedData.Length - NonceSize - TagSize; + Span ciphertext = stackalloc byte[ciphertextLength]; + + encryptedData[..NonceSize].CopyTo(nonce); + encryptedData.Slice(NonceSize, ciphertextLength).CopyTo(ciphertext); + encryptedData.Slice(NonceSize + ciphertextLength, TagSize).CopyTo(tag); + + var plaintext = new byte[ciphertextLength]; + aesGcm.Decrypt(nonce, ciphertext, tag, plaintext); + + return plaintext; + } + + /// + /// Creates a new AES-256 message encryptor with the specified base64-encoded key. + /// + /// The base64-encoded encryption key + /// A new AesMessageEncryptor instance + public static AesMessageEncryptor FromBase64Key(string base64Key) + { + return new AesMessageEncryptor(Convert.FromBase64String(base64Key)); + } + + /// + /// Generates a new random AES-256 key. + /// + /// A 32-byte random key suitable for AES-256 + public static byte[] GenerateKey() + { + var key = new byte[32]; + using var rng = RandomNumberGenerator.Create(); + rng.GetBytes(key); + return key; + } +} diff --git a/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs b/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs index fbb99c6262..f6111f89d2 100644 --- a/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs +++ b/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs @@ -1,40 +1,40 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -namespace Apache.Iggy.Encryption; - -/// -/// Interface for encrypting and decrypting message payloads in Iggy messaging system. -/// Implementations of this interface can be used with IggyPublisher and IggyConsumer -/// to provide end-to-end encryption of message data. -/// -public interface IMessageEncryptor -{ - /// - /// Encrypts the provided plain data. - /// - /// The plain data to encrypt - /// The encrypted data - byte[] Encrypt(byte[] plainData); - - /// - /// Decrypts the provided encrypted data. - /// - /// The encrypted data to decrypt - /// The decrypted plain data - byte[] Decrypt(byte[] encryptedData); -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.Iggy.Encryption; + +/// +/// Interface for encrypting and decrypting message payloads in Iggy messaging system. +/// Implementations of this interface can be used with IggyPublisher and IggyConsumer +/// to provide end-to-end encryption of message data. +/// +public interface IMessageEncryptor +{ + /// + /// Encrypts the provided plain data. + /// + /// The plain data to encrypt + /// The encrypted data + byte[] Encrypt(Span plainData); + + /// + /// Decrypts the provided encrypted data. + /// + /// The encrypted data to decrypt + /// The decrypted plain data + byte[] Decrypt(ReadOnlySpan encryptedData); +} diff --git a/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs b/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs index 82d1a8139c..c585e1fb50 100644 --- a/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs +++ b/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs @@ -1,74 +1,74 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Consumers; -using Apache.Iggy.IggyClient; -using Apache.Iggy.Kinds; -using Apache.Iggy.Publishers; - -namespace Apache.Iggy.Extensions; - -/// -/// Extension methods for -/// -public static class IggyClientExtension -{ - /// - /// Creates a new from for the specified stream and - /// topic. - /// - /// Existing iggy client - /// Stream identifier from which to consume - /// Topic identifier from which to consume - /// Consumer - /// Iggy consumer builder - public static IggyConsumerBuilder CreateConsumerBuilder(this IIggyClient client, Identifier streamId, - Identifier topicId, Consumer consumer) - { - return IggyConsumerBuilder.Create(client, streamId, topicId, consumer); - } - - /// - /// Creates a new from for the specified stream and - /// topic. - /// - /// Existing iggy client - /// Stream identifier from which to consume - /// Topic identifier from which to consume - /// Consumer - /// Optional deserializer - /// Iggy consumer builder - public static IggyConsumerBuilder CreateConsumerBuilder(this IIggyClient client, Identifier streamId, - Identifier topicId, Consumer consumer, IDeserializer deserializer) where T : IDeserializer - { - return IggyConsumerBuilder.Create(client, streamId, topicId, consumer); - } - - /// - /// Creates a new from for the specified stream and - /// topic. - /// - /// Existing iggy client - /// Stream identifier to publish to - /// >Topic identifier to publish to - /// Iggy publisher builder - public static IggyPublisherBuilder CreatePublisherBuilder(this IIggyClient client, Identifier streamId, - Identifier topicId) - { - return IggyPublisherBuilder.Create(client, streamId, topicId); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Consumers; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; +using Apache.Iggy.Publishers; + +namespace Apache.Iggy.Extensions; + +/// +/// Extension methods for +/// +public static class IggyClientExtension +{ + /// + /// Creates a new from for the specified stream and + /// topic. + /// + /// Existing iggy client + /// Stream identifier from which to consume + /// Topic identifier from which to consume + /// Consumer + /// Iggy consumer builder + public static IggyConsumerBuilder CreateConsumerBuilder(this IIggyClient client, Identifier streamId, + Identifier topicId, Consumer consumer) + { + return IggyConsumerBuilder.Create(client, streamId, topicId, consumer); + } + + /// + /// Creates a new from for the specified stream and + /// topic. + /// + /// Existing iggy client + /// Stream identifier from which to consume + /// Topic identifier from which to consume + /// Consumer + /// Optional deserializer + /// Iggy consumer builder + public static IggyConsumerBuilder CreateConsumerBuilder(this IIggyClient client, Identifier streamId, + Identifier topicId, Consumer consumer, IDeserializer deserializer) + { + return IggyConsumerBuilder.Create(client, streamId, topicId, consumer, deserializer); + } + + /// + /// Creates a new from for the specified stream and + /// topic. + /// + /// Existing iggy client + /// Stream identifier to publish to + /// >Topic identifier to publish to + /// Iggy publisher builder + public static IggyPublisherBuilder CreatePublisherBuilder(this IIggyClient client, Identifier streamId, + Identifier topicId) + { + return IggyPublisherBuilder.Create(client, streamId, topicId); + } +} diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs index 476b09b1b1..1898bdbd35 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs @@ -1,63 +1,85 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Contracts; -using Apache.Iggy.Kinds; - -namespace Apache.Iggy.IggyClient; - -/// -/// Defines methods for consuming messages from topics in an Iggy client. -/// -public interface IIggyConsumer -{ - /// - /// Polls messages from a specified topic and partition with a given polling strategy. - /// - /// - /// This method retrieves messages from a topic based on the specified consumer and polling strategy. - /// The polling strategy determines where to start reading messages (e.g., from a specific offset, latest, earliest). - /// If a partition ID is not specified, messages can be consumed from any partition. - /// - /// The identifier of the stream containing the topic (numeric ID or name). - /// The identifier of the topic to consume from (numeric ID or name). - /// The specific partition to consume from, or null to consume from any partition. - /// The consumer identifier (group ID or member ID). - /// The strategy for determining where to start reading messages. - /// The maximum number of messages to retrieve. - /// If true, automatically commit the offset after polling. - /// The cancellation token to cancel the operation. - /// A task that represents the asynchronous operation and returns the polled messages. - Task PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId, - Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit, - CancellationToken token = default); - - /// - /// Polls messages from a specified topic using a pre-constructed request. - /// - /// - /// This is a convenience method that wraps the full PollMessagesAsync method using a request object. - /// - /// The message fetch request containing all polling parameters. - /// The cancellation token to cancel the operation. - /// A task that represents the asynchronous operation and returns the polled messages. - Task PollMessagesAsync(MessageFetchRequest request, CancellationToken token = default) - { - return PollMessagesAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer, - request.PollingStrategy, request.Count, request.AutoCommit, token); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Contracts; +using Apache.Iggy.Kinds; + +namespace Apache.Iggy.IggyClient; + +/// +/// Defines methods for consuming messages from topics in an Iggy client. +/// +public interface IIggyConsumer +{ + /// + /// Polls messages from a specified topic and partition with a given polling strategy. + /// + /// + /// This method retrieves messages from a topic based on the specified consumer and polling strategy. + /// The polling strategy determines where to start reading messages (e.g., from a specific offset, latest, earliest). + /// If a partition ID is not specified, messages can be consumed from any partition. + /// + /// The identifier of the stream containing the topic (numeric ID or name). + /// The identifier of the topic to consume from (numeric ID or name). + /// The specific partition to consume from, or null to consume from any partition. + /// The consumer identifier (group ID or member ID). + /// The strategy for determining where to start reading messages. + /// The maximum number of messages to retrieve. + /// If true, automatically commit the offset after polling. + /// The cancellation token to cancel the operation. + /// A task that represents the asynchronous operation and returns the polled messages. + Task PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId, + Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit, + CancellationToken token = default); + + /// + /// Polls messages from a specified topic and partition while renting the payload buffers from a shared pool + /// instead of copying them into byte arrays. + /// + /// + /// The returned rental must be disposed when the caller is done reading the payload and raw header memory. + /// Payload and raw header slices are invalidated once the rental is disposed. + /// + Task PollMessagesRentedAsync(Identifier streamId, Identifier topicId, uint? partitionId, + Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit, + CancellationToken token = default); + + /// + /// Polls messages from a specified topic using a pre-constructed request. + /// + /// + /// This is a convenience method that wraps the full PollMessagesAsync method using a request object. + /// + /// The message fetch request containing all polling parameters. + /// The cancellation token to cancel the operation. + /// A task that represents the asynchronous operation and returns the polled messages. + Task PollMessagesAsync(MessageFetchRequest request, CancellationToken token = default) + { + return PollMessagesAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer, + request.PollingStrategy, request.Count, request.AutoCommit, token); + } + + /// + /// Polls messages from a specified topic using a pre-constructed request while renting the payload buffers + /// from a shared pool. + /// + Task PollMessagesRentedAsync(MessageFetchRequest request, CancellationToken token = default) + { + return PollMessagesRentedAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer, + request.PollingStrategy, request.Count, request.AutoCommit, token); + } +} diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs index fac18e1de0..e7e7e77a76 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs @@ -1,850 +1,862 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Net; -using System.Net.Http.Headers; -using System.Net.Http.Json; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using Apache.Iggy.Contracts; -using Apache.Iggy.Contracts.Auth; -using Apache.Iggy.Contracts.Http; -using Apache.Iggy.Contracts.Http.Auth; -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Kinds; -using Apache.Iggy.Messages; -using Apache.Iggy.StringHandlers; -using Apache.Iggy.Utils; -using Partitioning = Apache.Iggy.Kinds.Partitioning; - -namespace Apache.Iggy.IggyClient.Implementations; - -/// -/// Implementation of that uses to communicate with the server. -/// -public class HttpMessageStream : IIggyClient -{ - private const string Context = "csharp-sdk"; - - private readonly HttpClient _httpClient; - - //TODO - create mechanism for refreshing jwt token - //TODO - replace the HttpClient with IHttpClientFactory, when implementing support for ASP.NET Core DI - //TODO - the error handling pattern is pretty ugly, look into moving it into an extension method - private readonly JsonSerializerOptions _jsonSerializerOptions; - - internal HttpMessageStream(HttpClient httpClient) - { - _httpClient = httpClient; - - _jsonSerializerOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, - Converters = { new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) } - }; - } - - /// - public async Task CreateStreamAsync(string name, CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new CreateStreamRequest(name), _jsonSerializerOptions); - - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response = await _httpClient.PostAsync("/streams", data, token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - - return null; - } - - /// - public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default) - { - var response = await _httpClient.DeleteAsync($"/streams/{streamId}/purge", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default) - { - var response = await _httpClient.DeleteAsync($"/streams/{streamId}", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task GetStreamByIdAsync(Identifier streamId, CancellationToken token = default) - { - var response = await _httpClient.GetAsync($"/streams/{streamId}", token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - - return null; - } - - /// - public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new UpdateStreamRequest(name), _jsonSerializerOptions); - - var data = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PutAsync($"/streams/{streamId}", data, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task> GetStreamsAsync(CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/streams", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, - token) - ?? Array.Empty(); - } - - await HandleResponseAsync(response); - return Array.Empty(); - } - - /// - public async Task CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null, - TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new CreateTopicRequest - { - Name = name, - CompressionAlgorithm = compressionAlgorithm, - MaxTopicSize = maxTopicSize, - MessageExpiry = DurationHelpers.ToDuration(messageExpiry), - PartitionsCount = partitionsCount, - ReplicationFactor = replicationFactor - }, _jsonSerializerOptions); - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response = await _httpClient.PostAsync($"/streams/{streamId}/topics", data, token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - - return null; - } - - /// - public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, - ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null, - CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new UpdateTopicRequest(name, compressionAlgorithm, maxTopicSize, - DurationHelpers.ToDuration(messageExpiry), - replicationFactor), - _jsonSerializerOptions); - var data = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}", data, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) - { - var response = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) - { - return _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/purge", token) - .ContinueWith(async response => - { - if (!response.Result.IsSuccessStatusCode) - { - await HandleResponseAsync(response.Result); - } - }, token); - } - - /// - public async Task> GetTopicsAsync(Identifier streamId, - CancellationToken token = default) - { - var response = await _httpClient.GetAsync($"/streams/{streamId}/topics", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, token) - ?? Array.Empty(); - } - - await HandleResponseAsync(response); - return Array.Empty(); - } - - /// - public async Task GetTopicByIdAsync(Identifier streamId, Identifier topicId, - CancellationToken token = default) - { - var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}", token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - - return null; - } - - /// - public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, - IList messages, - CancellationToken token = default) - { - var request = new MessageSendRequest - { - StreamId = streamId, - TopicId = topicId, - Partitioning = partitioning, - Messages = messages - }; - var json = JsonSerializer.Serialize(request, _jsonSerializerOptions); - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response = await _httpClient.PostAsync($"/streams/{request.StreamId}/topics/{request.TopicId}/messages", - data, - token); - - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, - CancellationToken token = default) - { - var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages/flush/{partitionId}/{fsync}"); - - var response = await _httpClient.GetAsync(url, token); - - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response, true); - } - } - - /// - public async Task PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId, - Consumer consumer, - PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) - { - var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty; - var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages?consumer_id={consumer.ConsumerId}" + - $"{partitionIdParam}&kind={pollingStrategy.Kind}&value={pollingStrategy.Value}&count={count}&auto_commit={autoCommit}"); - - var response = await _httpClient.GetAsync(url, token); - if (response.IsSuccessStatusCode) - { - var pollMessages = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token) - ?? PolledMessages.Empty; - - return pollMessages; - } - - await HandleResponseAsync(response, true); - return PolledMessages.Empty; - } - - /// - public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset, - uint? partitionId, CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new StoreOffsetRequest(consumer, partitionId, offset), - _jsonSerializerOptions); - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response - = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}/consumer-offsets", data, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, - uint? partitionId, CancellationToken token = default) - { - var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty; - var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/" + - $"consumer-offsets?consumer_id={consumer.ConsumerId}{partitionIdParam}", - token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId, - CancellationToken token = default) - { - var partitionIdParam = partitionId.HasValue ? $"?partition_id={partitionId.Value}" : string.Empty; - var response = await _httpClient.DeleteAsync( - $"/streams/{streamId}/topics/{topicId}/consumer-offsets/{consumer}{partitionIdParam}", token); - await HandleResponseAsync(response); - } - - /// - public async Task> GetConsumerGroupsAsync(Identifier streamId, - Identifier topicId, CancellationToken token = default) - { - var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync>( - _jsonSerializerOptions, token) - ?? Array.Empty(); - } - - await HandleResponseAsync(response); - return Array.Empty(); - } - - /// - public async Task GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId, - Identifier groupId, CancellationToken token = default) - { - var response - = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task CreateConsumerGroupAsync(Identifier streamId, Identifier topicId, - string name, CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new CreateConsumerGroupRequest(name), _jsonSerializerOptions); - - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response - = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", data, token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, - CancellationToken token = default) - { - var response - = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token); - await HandleResponseAsync(response); - } - - /// - /// This method is only supported in TCP protocol - /// - /// - /// - /// - public Task GetMeAsync(CancellationToken token = default) - { - throw new FeatureUnavailableException(); - } - - /// - public async Task GetStatsAsync(CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/stats", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task GetClusterMetadataAsync(CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/cluster/metadata", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - - return null; - } - - /// - public async Task PingAsync(CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/ping", token); - - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task GetSnapshotAsync(SnapshotCompression compression, - IList snapshotTypes, CancellationToken token = default) - { - // Rust serde uses default derive (PascalCase) for these enums, not snake_case. - // We use .ToString() to produce PascalCase names matching Rust's serde expectations. - var request = new - { - compression = compression.ToString(), - snapshot_types = snapshotTypes.Select(t => t.ToString()).ToList() - }; - var json = JsonSerializer.Serialize(request, _jsonSerializerOptions); - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response = await _httpClient.PostAsync("/snapshot", data, token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadAsByteArrayAsync(token); - } - - await HandleResponseAsync(response); - return []; - } - - /// - public Task ConnectAsync(CancellationToken token = default) - { - return Task.CompletedTask; - } - - /// - public async Task> GetClientsAsync(CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/clients", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, - token) - ?? Array.Empty(); - } - - await HandleResponseAsync(response); - return Array.Empty(); - } - - /// - public async Task GetClientByIdAsync(uint clientId, CancellationToken token = default) - { - var response = await _httpClient.GetAsync($"/clients/{clientId}", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - /// This method is only supported in TCP protocol - /// - /// The identifier of the stream containing the topic (numeric ID or name). - /// The identifier of the topic (numeric ID or name). - /// The identifier of the consumer group to join (numeric ID or name). - /// The cancellation token to cancel the operation. - /// A task representing the asynchronous operation. - /// - [Obsolete("This method is only supported in TCP protocol", true)] - public Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, - CancellationToken token = default) - { - throw new FeatureUnavailableException(); - } - - /// - /// This method is only supported in TCP protocol - /// - /// The identifier of the stream containing the topic (numeric ID or name). - /// The identifier of the topic (numeric ID or name). - /// The identifier of the consumer group to leave (numeric ID or name). - /// The cancellation token to cancel the operation. - /// A task representing the asynchronous operation. - /// - [Obsolete("This method is only supported in TCP protocol", true)] - public Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, - CancellationToken token = default) - { - throw new FeatureUnavailableException(); - } - - /// - public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, - CancellationToken token = default) - { - var response - = await _httpClient.DeleteAsync( - $"/streams/{streamId}/topics/{topicId}/partitions?partitions_count={partitionsCount}", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - /// This method is only supported in TCP protocol - /// - /// The identifier of the stream containing the topic (numeric ID or name). - /// The identifier of the topic containing the partition (numeric ID or name). - /// The unique partition ID. - /// The number of segments to delete. - /// The cancellation token to cancel the operation. - /// A task representing the asynchronous operation. - /// - public Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId, - uint segmentsCount, CancellationToken token = default) - { - throw new FeatureUnavailableException(); - } - - /// - public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, - CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new CreatePartitionsRequest(partitionsCount), _jsonSerializerOptions); - - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/partitions", data, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task GetUserAsync(Identifier userId, CancellationToken token = default) - { - //TODO - this doesn't work prob needs a custom json serializer - var response = await _httpClient.GetAsync($"/users/{userId}", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task> GetUsersAsync(CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/users", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, token) - ?? Array.Empty(); - } - - await HandleResponseAsync(response); - return Array.Empty(); - } - - /// - public async Task CreateUserAsync(string userName, string password, UserStatus status, - Permissions? permissions = null, CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new CreateUserRequest(userName, password, status, permissions), - _jsonSerializerOptions); - - var content = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PostAsync("/users", content, token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default) - { - var response = await _httpClient.DeleteAsync($"/users/{userId}", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null, - CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new UpdateUserRequest(userName, status), _jsonSerializerOptions); - var content = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PutAsync($"/users/{userId}", content, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null, - CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new UpdateUserPermissionsRequest(permissions), _jsonSerializerOptions); - var content = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PutAsync($"/users/{userId}/permissions", content, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword, - CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new ChangePasswordRequest(currentPassword, newPassword), - _jsonSerializerOptions); - var content = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PutAsync($"/users/{userId}/password", content, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task LoginUserAsync(string userName, string password, CancellationToken token = default) - { - // TODO: Add binary protocol version - var json = JsonSerializer.Serialize(new LoginUserRequest(userName, password, SdkVersion.Value, Context), - _jsonSerializerOptions); - - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response = await _httpClient.PostAsync("users/login", data, token); - if (response.IsSuccessStatusCode) - { - var authResponse = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - var jwtToken = authResponse!.AccessToken?.Token; - if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token)) - { - _httpClient.DefaultRequestHeaders.Authorization = - new AuthenticationHeaderValue("Bearer", jwtToken); - } - else - { - throw new Exception("The JWT token is missing."); - } - - return authResponse; - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task LogoutUserAsync(CancellationToken token = default) - { - var response = await _httpClient.DeleteAsync("users/logout", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - - _httpClient.DefaultRequestHeaders.Authorization = null; - } - - /// - public async Task> GetPersonalAccessTokensAsync( - CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/personal-access-tokens", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync>( - _jsonSerializerOptions, token) - ?? Array.Empty(); - } - - await HandleResponseAsync(response); - return Array.Empty(); - } - - /// - public async Task CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null, - CancellationToken token = default) - { - var json = JsonSerializer.Serialize( - new CreatePersonalAccessTokenRequest(name, DurationHelpers.ToDuration(expiry)), _jsonSerializerOptions); - - var content = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PostAsync("/personal-access-tokens", content, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - /// - public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default) - { - var response = await _httpClient.DeleteAsync($"/personal-access-tokens/{name}", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default) - { - var json = JsonSerializer.Serialize(new LoginWithPersonalAccessTokenRequest(token), _jsonSerializerOptions); - - var content = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PostAsync("/personal-access-tokens/login", content, ct); - if (response.IsSuccessStatusCode) - { - var authResponse = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, ct); - var jwtToken = authResponse!.AccessToken?.Token; - if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token)) - { - _httpClient.DefaultRequestHeaders.Authorization = - new AuthenticationHeaderValue("Bearer", jwtToken); - } - else - { - throw new Exception("The JWT token is missing."); - } - - return authResponse; - } - - await HandleResponseAsync(response); - - return null; - } - - /// - /// Dispose the client. - /// - public void Dispose() - { - } - - /// - public void SubscribeConnectionEvents(Func callback) - { - } - - /// - public void UnsubscribeConnectionEvents(Func callback) - { - } - - /// - public string GetCurrentAddress() - { - return _httpClient.BaseAddress?.ToString() ?? string.Empty; - } - - private static async Task HandleResponseAsync(HttpResponseMessage response, bool shouldThrowOnGetNotFound = false) - { - if ((int)response.StatusCode > 300 - && (int)response.StatusCode < 500 - && !(response.RequestMessage!.Method == HttpMethod.Get && response.StatusCode == HttpStatusCode.NotFound && - !shouldThrowOnGetNotFound)) - { - var err = await response.Content.ReadAsStringAsync(); - var errorModel = JsonSerializer.Deserialize(err); - throw new IggyInvalidStatusCodeException(errorModel?.Id ?? -1, err); - } - - if (response.StatusCode == HttpStatusCode.InternalServerError) - { - throw new Exception("Internal server error"); - } - } - - private static string CreateUrl(ref MessageRequestInterpolationHandler message) - { - return message.ToString(); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Apache.Iggy.Contracts; +using Apache.Iggy.Contracts.Auth; +using Apache.Iggy.Contracts.Http; +using Apache.Iggy.Contracts.Http.Auth; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Kinds; +using Apache.Iggy.Mappers; +using Apache.Iggy.Messages; +using Apache.Iggy.StringHandlers; +using Apache.Iggy.Utils; +using Partitioning = Apache.Iggy.Kinds.Partitioning; + +namespace Apache.Iggy.IggyClient.Implementations; + +/// +/// Implementation of that uses to communicate with the server. +/// +public class HttpMessageStream : IIggyClient +{ + private const string Context = "csharp-sdk"; + + private readonly HttpClient _httpClient; + + //TODO - create mechanism for refreshing jwt token + //TODO - replace the HttpClient with IHttpClientFactory, when implementing support for ASP.NET Core DI + //TODO - the error handling pattern is pretty ugly, look into moving it into an extension method + private readonly JsonSerializerOptions _jsonSerializerOptions; + + internal HttpMessageStream(HttpClient httpClient) + { + _httpClient = httpClient; + + _jsonSerializerOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) } + }; + } + + /// + public async Task CreateStreamAsync(string name, CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new CreateStreamRequest(name), _jsonSerializerOptions); + + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync("/streams", data, token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + + return null; + } + + /// + public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default) + { + var response = await _httpClient.DeleteAsync($"/streams/{streamId}/purge", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default) + { + var response = await _httpClient.DeleteAsync($"/streams/{streamId}", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task GetStreamByIdAsync(Identifier streamId, CancellationToken token = default) + { + var response = await _httpClient.GetAsync($"/streams/{streamId}", token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + + return null; + } + + /// + public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new UpdateStreamRequest(name), _jsonSerializerOptions); + + var data = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PutAsync($"/streams/{streamId}", data, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task> GetStreamsAsync(CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/streams", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, + token) + ?? Array.Empty(); + } + + await HandleResponseAsync(response); + return Array.Empty(); + } + + /// + public async Task CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, + CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null, + TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new CreateTopicRequest + { + Name = name, + CompressionAlgorithm = compressionAlgorithm, + MaxTopicSize = maxTopicSize, + MessageExpiry = DurationHelpers.ToDuration(messageExpiry), + PartitionsCount = partitionsCount, + ReplicationFactor = replicationFactor + }, _jsonSerializerOptions); + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/streams/{streamId}/topics", data, token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + + return null; + } + + /// + public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, + CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, + ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null, + CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new UpdateTopicRequest(name, compressionAlgorithm, maxTopicSize, + DurationHelpers.ToDuration(messageExpiry), + replicationFactor), + _jsonSerializerOptions); + var data = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}", data, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) + { + var response = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) + { + return _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/purge", token) + .ContinueWith(async response => + { + if (!response.Result.IsSuccessStatusCode) + { + await HandleResponseAsync(response.Result); + } + }, token); + } + + /// + public async Task> GetTopicsAsync(Identifier streamId, + CancellationToken token = default) + { + var response = await _httpClient.GetAsync($"/streams/{streamId}/topics", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, token) + ?? Array.Empty(); + } + + await HandleResponseAsync(response); + return Array.Empty(); + } + + /// + public async Task GetTopicByIdAsync(Identifier streamId, Identifier topicId, + CancellationToken token = default) + { + var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}", token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + + return null; + } + + /// + public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, + IList messages, + CancellationToken token = default) + { + var request = new MessageSendRequest + { + StreamId = streamId, + TopicId = topicId, + Partitioning = partitioning, + Messages = messages + }; + var json = JsonSerializer.Serialize(request, _jsonSerializerOptions); + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/streams/{request.StreamId}/topics/{request.TopicId}/messages", + data, + token); + + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, + CancellationToken token = default) + { + var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages/flush/{partitionId}/{fsync}"); + + var response = await _httpClient.GetAsync(url, token); + + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response, true); + } + } + + /// + public async Task PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId, + Consumer consumer, + PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) + { + var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty; + var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages?consumer_id={consumer.ConsumerId}" + + $"{partitionIdParam}&kind={pollingStrategy.Kind}&value={pollingStrategy.Value}&count={count}&auto_commit={autoCommit}"); + + var response = await _httpClient.GetAsync(url, token); + if (response.IsSuccessStatusCode) + { + var pollMessages = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token) + ?? PolledMessages.Empty; + + return pollMessages; + } + + await HandleResponseAsync(response, true); + return PolledMessages.Empty; + } + + /// + public async Task PollMessagesRentedAsync(Identifier streamId, Identifier topicId, + uint? partitionId, + Consumer consumer, + PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) + { + var messages = await PollMessagesAsync(streamId, topicId, partitionId, consumer, pollingStrategy, count, + autoCommit, token); + return BinaryMapper.ToRentedMessages(messages); + } + + /// + public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset, + uint? partitionId, CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new StoreOffsetRequest(consumer, partitionId, offset), + _jsonSerializerOptions); + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response + = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}/consumer-offsets", data, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, + uint? partitionId, CancellationToken token = default) + { + var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty; + var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/" + + $"consumer-offsets?consumer_id={consumer.ConsumerId}{partitionIdParam}", + token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId, + CancellationToken token = default) + { + var partitionIdParam = partitionId.HasValue ? $"?partition_id={partitionId.Value}" : string.Empty; + var response = await _httpClient.DeleteAsync( + $"/streams/{streamId}/topics/{topicId}/consumer-offsets/{consumer}{partitionIdParam}", token); + await HandleResponseAsync(response); + } + + /// + public async Task> GetConsumerGroupsAsync(Identifier streamId, + Identifier topicId, CancellationToken token = default) + { + var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync>( + _jsonSerializerOptions, token) + ?? Array.Empty(); + } + + await HandleResponseAsync(response); + return Array.Empty(); + } + + /// + public async Task GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId, + Identifier groupId, CancellationToken token = default) + { + var response + = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task CreateConsumerGroupAsync(Identifier streamId, Identifier topicId, + string name, CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new CreateConsumerGroupRequest(name), _jsonSerializerOptions); + + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response + = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", data, token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, + CancellationToken token = default) + { + var response + = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token); + await HandleResponseAsync(response); + } + + /// + /// This method is only supported in TCP protocol + /// + /// + /// + /// + public Task GetMeAsync(CancellationToken token = default) + { + throw new FeatureUnavailableException(); + } + + /// + public async Task GetStatsAsync(CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/stats", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task GetClusterMetadataAsync(CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/cluster/metadata", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + + return null; + } + + /// + public async Task PingAsync(CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/ping", token); + + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task GetSnapshotAsync(SnapshotCompression compression, + IList snapshotTypes, CancellationToken token = default) + { + // Rust serde uses default derive (PascalCase) for these enums, not snake_case. + // We use .ToString() to produce PascalCase names matching Rust's serde expectations. + var request = new + { + compression = compression.ToString(), + snapshot_types = snapshotTypes.Select(t => t.ToString()).ToList() + }; + var json = JsonSerializer.Serialize(request, _jsonSerializerOptions); + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync("/snapshot", data, token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadAsByteArrayAsync(token); + } + + await HandleResponseAsync(response); + return []; + } + + /// + public Task ConnectAsync(CancellationToken token = default) + { + return Task.CompletedTask; + } + + /// + public async Task> GetClientsAsync(CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/clients", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, + token) + ?? Array.Empty(); + } + + await HandleResponseAsync(response); + return Array.Empty(); + } + + /// + public async Task GetClientByIdAsync(uint clientId, CancellationToken token = default) + { + var response = await _httpClient.GetAsync($"/clients/{clientId}", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + /// This method is only supported in TCP protocol + /// + /// The identifier of the stream containing the topic (numeric ID or name). + /// The identifier of the topic (numeric ID or name). + /// The identifier of the consumer group to join (numeric ID or name). + /// The cancellation token to cancel the operation. + /// A task representing the asynchronous operation. + /// + [Obsolete("This method is only supported in TCP protocol", true)] + public Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, + CancellationToken token = default) + { + throw new FeatureUnavailableException(); + } + + /// + /// This method is only supported in TCP protocol + /// + /// The identifier of the stream containing the topic (numeric ID or name). + /// The identifier of the topic (numeric ID or name). + /// The identifier of the consumer group to leave (numeric ID or name). + /// The cancellation token to cancel the operation. + /// A task representing the asynchronous operation. + /// + [Obsolete("This method is only supported in TCP protocol", true)] + public Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, + CancellationToken token = default) + { + throw new FeatureUnavailableException(); + } + + /// + public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, + CancellationToken token = default) + { + var response + = await _httpClient.DeleteAsync( + $"/streams/{streamId}/topics/{topicId}/partitions?partitions_count={partitionsCount}", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + /// This method is only supported in TCP protocol + /// + /// The identifier of the stream containing the topic (numeric ID or name). + /// The identifier of the topic containing the partition (numeric ID or name). + /// The unique partition ID. + /// The number of segments to delete. + /// The cancellation token to cancel the operation. + /// A task representing the asynchronous operation. + /// + public Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId, + uint segmentsCount, CancellationToken token = default) + { + throw new FeatureUnavailableException(); + } + + /// + public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, + CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new CreatePartitionsRequest(partitionsCount), _jsonSerializerOptions); + + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/partitions", data, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task GetUserAsync(Identifier userId, CancellationToken token = default) + { + //TODO - this doesn't work prob needs a custom json serializer + var response = await _httpClient.GetAsync($"/users/{userId}", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task> GetUsersAsync(CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/users", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, token) + ?? Array.Empty(); + } + + await HandleResponseAsync(response); + return Array.Empty(); + } + + /// + public async Task CreateUserAsync(string userName, string password, UserStatus status, + Permissions? permissions = null, CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new CreateUserRequest(userName, password, status, permissions), + _jsonSerializerOptions); + + var content = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PostAsync("/users", content, token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default) + { + var response = await _httpClient.DeleteAsync($"/users/{userId}", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null, + CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new UpdateUserRequest(userName, status), _jsonSerializerOptions); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PutAsync($"/users/{userId}", content, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null, + CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new UpdateUserPermissionsRequest(permissions), _jsonSerializerOptions); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PutAsync($"/users/{userId}/permissions", content, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword, + CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new ChangePasswordRequest(currentPassword, newPassword), + _jsonSerializerOptions); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PutAsync($"/users/{userId}/password", content, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task LoginUserAsync(string userName, string password, CancellationToken token = default) + { + // TODO: Add binary protocol version + var json = JsonSerializer.Serialize(new LoginUserRequest(userName, password, SdkVersion.Value, Context), + _jsonSerializerOptions); + + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync("users/login", data, token); + if (response.IsSuccessStatusCode) + { + var authResponse = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + var jwtToken = authResponse!.AccessToken?.Token; + if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token)) + { + _httpClient.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("Bearer", jwtToken); + } + else + { + throw new Exception("The JWT token is missing."); + } + + return authResponse; + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task LogoutUserAsync(CancellationToken token = default) + { + var response = await _httpClient.DeleteAsync("users/logout", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + + _httpClient.DefaultRequestHeaders.Authorization = null; + } + + /// + public async Task> GetPersonalAccessTokensAsync( + CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/personal-access-tokens", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync>( + _jsonSerializerOptions, token) + ?? Array.Empty(); + } + + await HandleResponseAsync(response); + return Array.Empty(); + } + + /// + public async Task CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null, + CancellationToken token = default) + { + var json = JsonSerializer.Serialize( + new CreatePersonalAccessTokenRequest(name, DurationHelpers.ToDuration(expiry)), _jsonSerializerOptions); + + var content = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PostAsync("/personal-access-tokens", content, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + /// + public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default) + { + var response = await _httpClient.DeleteAsync($"/personal-access-tokens/{name}", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default) + { + var json = JsonSerializer.Serialize(new LoginWithPersonalAccessTokenRequest(token), _jsonSerializerOptions); + + var content = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PostAsync("/personal-access-tokens/login", content, ct); + if (response.IsSuccessStatusCode) + { + var authResponse = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, ct); + var jwtToken = authResponse!.AccessToken?.Token; + if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token)) + { + _httpClient.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("Bearer", jwtToken); + } + else + { + throw new Exception("The JWT token is missing."); + } + + return authResponse; + } + + await HandleResponseAsync(response); + + return null; + } + + /// + /// Dispose the client. + /// + public void Dispose() + { + } + + /// + public void SubscribeConnectionEvents(Func callback) + { + } + + /// + public void UnsubscribeConnectionEvents(Func callback) + { + } + + /// + public string GetCurrentAddress() + { + return _httpClient.BaseAddress?.ToString() ?? string.Empty; + } + + private static async Task HandleResponseAsync(HttpResponseMessage response, bool shouldThrowOnGetNotFound = false) + { + if ((int)response.StatusCode > 300 + && (int)response.StatusCode < 500 + && !(response.RequestMessage!.Method == HttpMethod.Get && response.StatusCode == HttpStatusCode.NotFound && + !shouldThrowOnGetNotFound)) + { + var err = await response.Content.ReadAsStringAsync(); + var errorModel = JsonSerializer.Deserialize(err); + throw new IggyInvalidStatusCodeException(errorModel?.Id ?? -1, err); + } + + if (response.StatusCode == HttpStatusCode.InternalServerError) + { + throw new Exception("Internal server error"); + } + } + + private static string CreateUrl(ref MessageRequestInterpolationHandler message) + { + return message.ToString(); + } +} diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs index ed8d424451..72d0d9b511 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs @@ -1,1239 +1,1317 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers; -using System.Buffers.Binary; -using System.Net.Security; -using System.Net.Sockets; -using System.Security.Cryptography.X509Certificates; -using System.Text; -using Apache.Iggy.Configuration; -using Apache.Iggy.ConnectionStream; -using Apache.Iggy.Contracts; -using Apache.Iggy.Contracts.Auth; -using Apache.Iggy.Contracts.Tcp; -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Kinds; -using Apache.Iggy.Mappers; -using Apache.Iggy.Messages; -using Apache.Iggy.Utils; -using Microsoft.Extensions.Logging; -using Partitioning = Apache.Iggy.Kinds.Partitioning; - -namespace Apache.Iggy.IggyClient.Implementations; - -/// -/// A TCP client for interacting with the Iggy server. -/// -public sealed class TcpMessageStream : IIggyClient -{ - private readonly IggyClientConfigurator _configuration; - private readonly EventAggregator _connectionEvents; - private readonly SemaphoreSlim _connectionSemaphore; - private readonly ILogger _logger; - private readonly SemaphoreSlim _sendingSemaphore; - private string _currentAddress = string.Empty; - private X509Certificate2Collection _customCaStore = []; - private bool _isConnecting; - private DateTimeOffset _lastConnectionTime; - private ConnectionState _state = ConnectionState.Disconnected; - private TcpConnectionStream _stream = null!; - - internal TcpMessageStream(IggyClientConfigurator configuration, ILoggerFactory loggerFactory) - { - _configuration = configuration; - _logger = loggerFactory.CreateLogger(); - _sendingSemaphore = new SemaphoreSlim(1, 1); - _connectionSemaphore = new SemaphoreSlim(1, 1); - _lastConnectionTime = DateTimeOffset.MinValue; - _connectionEvents = new EventAggregator(loggerFactory); - } - - /// - /// Fired whenever the connection state changes. - /// - //public event EventHandler? OnConnectionStateChanged; - public void Dispose() - { - _stream?.Close(); - _stream?.Dispose(); - _sendingSemaphore.Dispose(); - _connectionSemaphore.Dispose(); - _connectionEvents.Clear(); - } - - /// - public void SubscribeConnectionEvents(Func callback) - { - _connectionEvents.Subscribe(callback); - } - - /// - public void UnsubscribeConnectionEvents(Func callback) - { - _connectionEvents.Unsubscribe(callback); - } - - /// - public string GetCurrentAddress() - { - return _currentAddress; - } - - /// - public async Task CreateStreamAsync(string name, CancellationToken token = default) - { - var message = TcpContracts.CreateStream(name); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_STREAM_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - throw new InvalidResponseException("Received empty response while trying to create stream."); - } - - return BinaryMapper.MapStream(responseBuffer); - } - - /// - public async Task GetStreamByIdAsync(Identifier streamId, CancellationToken token = default) - { - var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAM_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return null; - } - - return BinaryMapper.MapStream(responseBuffer); - } - - /// - public async Task> GetStreamsAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAMS_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return []; - } - - return BinaryMapper.MapStreams(responseBuffer); - } - - /// - public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default) - { - var message = TcpContracts.UpdateStream(streamId, name); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_STREAM_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default) - { - var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_STREAM_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default) - { - var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_STREAM_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task> GetTopicsAsync(Identifier streamId, - CancellationToken token = default) - { - var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPICS_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return []; - } - - return BinaryMapper.MapTopics(responseBuffer); - } - - /// - public async Task GetTopicByIdAsync(Identifier streamId, Identifier topicId, - CancellationToken token = default) - { - var message = TcpContracts.GetTopicById(streamId, topicId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPIC_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return null; - } - - return BinaryMapper.MapTopic(responseBuffer); - } - - /// - public async Task CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null, - TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default) - { - var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry); - var message = TcpContracts.CreateTopic(streamId, name, partitionsCount, compressionAlgorithm, - replicationFactor, messageExpiryValue, maxTopicSize); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_TOPIC_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return null; - } - - return BinaryMapper.MapTopic(responseBuffer); - } - - /// - public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, - ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null, - CancellationToken token = default) - { - var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry); - var message = TcpContracts.UpdateTopic(streamId, topicId, name, compressionAlgorithm, maxTopicSize, - messageExpiryValue, replicationFactor); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_TOPIC_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) - { - var message = TcpContracts.DeleteTopic(streamId, topicId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_TOPIC_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) - { - var message = TcpContracts.PurgeTopic(streamId, topicId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_TOPIC_CODE); - - await SendWithResponseAsync(payload, token); - } - - - /// - public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, - IList messages, CancellationToken token = default) - { - var metadataLength = 2 + streamId.Length + 2 + topicId.Length - + 2 + partitioning.Length + 4 + 4; - var messageBufferSize = TcpMessageStreamHelpers.CalculateMessageBytesCount(messages) - + metadataLength; - var payloadBufferSize = messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH; - - IMemoryOwner messageBuffer = MemoryPool.Shared.Rent(messageBufferSize); - IMemoryOwner payloadBuffer = MemoryPool.Shared.Rent(payloadBufferSize); - try - { - TcpContracts.CreateMessage(messageBuffer.Memory.Span[..messageBufferSize], streamId, - topicId, partitioning, messages); - - TcpMessageStreamHelpers.CreatePayload(payloadBuffer.Memory.Span[..payloadBufferSize], - messageBuffer.Memory.Span[..messageBufferSize], CommandCodes.SEND_MESSAGES_CODE); - - await SendWithResponseAsync(payloadBuffer.Memory[..payloadBufferSize].ToArray(), token); - } - finally - { - messageBuffer.Dispose(); - payloadBuffer.Dispose(); - } - } - - /// - public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, - CancellationToken token = default) - { - var message = TcpContracts.FlushUnsavedBuffer(streamId, topicId, partitionId, fsync); - - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.FLUSH_UNSAVED_BUFFER_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId, - Consumer consumer, - PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) - { - var messageBufferSize = CalculateMessageBufferSize(streamId, topicId, consumer); - var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize); - var message = new byte[messageBufferSize]; - var payload = new byte[payloadBufferSize]; - - TcpContracts.GetMessages(message.AsSpan()[..messageBufferSize], consumer, streamId, - topicId, pollingStrategy, count, autoCommit, partitionId); - TcpMessageStreamHelpers.CreatePayload(payload, message.AsSpan()[..messageBufferSize], - CommandCodes.POLL_MESSAGES_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - return BinaryMapper.MapMessages(responseBuffer); - } - - /// - public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset, - uint? partitionId, CancellationToken token = default) - { - var message = TcpContracts.UpdateOffset(streamId, topicId, consumer, offset, partitionId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.STORE_CONSUMER_OFFSET_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, - uint? partitionId, CancellationToken token = default) - { - var message = TcpContracts.GetOffset(streamId, topicId, consumer, partitionId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_OFFSET_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return null; - } - - return BinaryMapper.MapOffsets(responseBuffer); - } - - /// - public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId, - CancellationToken token = default) - { - var message = TcpContracts.DeleteOffset(streamId, topicId, consumer, partitionId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_OFFSET_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task> GetConsumerGroupsAsync(Identifier streamId, - Identifier topicId, - CancellationToken token = default) - { - var message = TcpContracts.GetGroups(streamId, topicId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUPS_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return []; - } - - return BinaryMapper.MapConsumerGroups(responseBuffer); - } - - /// - public async Task GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId, - Identifier groupId, CancellationToken token = default) - { - var message = TcpContracts.GetGroup(streamId, topicId, groupId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUP_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return null; - } - - return BinaryMapper.MapConsumerGroup(responseBuffer); - } - - /// - public async Task CreateConsumerGroupAsync(Identifier streamId, Identifier topicId, - string name, CancellationToken token = default) - { - var message = TcpContracts.CreateGroup(streamId, topicId, name); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_CONSUMER_GROUP_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return null; - } - - return BinaryMapper.MapConsumerGroup(responseBuffer); - } - - /// - public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, - CancellationToken token = default) - { - var message = TcpContracts.DeleteGroup(streamId, topicId, groupId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_GROUP_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, - CancellationToken token = default) - { - var message = TcpContracts.JoinGroup(streamId, topicId, groupId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.JOIN_CONSUMER_GROUP_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, - CancellationToken token = default) - { - var message = TcpContracts.LeaveGroup(streamId, topicId, groupId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LEAVE_CONSUMER_GROUP_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, - CancellationToken token = default) - { - var message = TcpContracts.DeletePartitions(streamId, topicId, partitionsCount); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PARTITIONS_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, - CancellationToken token = default) - { - var message = TcpContracts.CreatePartitions(streamId, topicId, partitionsCount); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PARTITIONS_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId, - uint segmentsCount, CancellationToken token = default) - { - var message = TcpContracts.DeleteSegments(streamId, topicId, partitionId, segmentsCount); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_SEGMENTS_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task GetMeAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_ME_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return null; - } - - return BinaryMapper.MapClient(responseBuffer); - } - - /// - public async Task GetStatsAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STATS_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return null; - } - - return BinaryMapper.MapStats(responseBuffer); - } - - /// - public async Task GetClusterMetadataAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLUSTER_METADATA_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return null; - } - - return BinaryMapper.MapClusterMetadata(responseBuffer); - } - - /// - public async Task PingAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PING_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task GetSnapshotAsync(SnapshotCompression compression, - IList snapshotTypes, CancellationToken token = default) - { - var message = TcpContracts.GetSnapshot(compression, snapshotTypes); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_SNAPSHOT_CODE); - - return await SendWithResponseAsync(payload, token); - } - - /// - public async Task ConnectAsync(CancellationToken token = default) - { - if (_state is ConnectionState.Connected - or ConnectionState.Authenticating - or ConnectionState.Authenticated) - { - _logger.LogWarning("Connection is already connected"); - return; - } - - if (_lastConnectionTime != DateTimeOffset.MinValue) - { - await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token); - } - - SetConnectionStateAsync(ConnectionState.Connecting); - _isConnecting = true; - try - { - await TryEstablishConnectionAsync(token); - } - finally - { - _isConnecting = false; - } - } - - /// - public async Task> GetClientsAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENTS_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return []; - } - - return BinaryMapper.MapClients(responseBuffer); - } - - /// - public async Task GetClientByIdAsync(uint clientId, CancellationToken token = default) - { - var message = TcpContracts.GetClient(clientId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENT_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return null; - } - - return BinaryMapper.MapClient(responseBuffer); - } - - /// - public async Task GetUserAsync(Identifier userId, CancellationToken token = default) - { - var message = TcpContracts.GetUser(userId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USER_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return null; - } - - return BinaryMapper.MapUser(responseBuffer); - } - - /// - public async Task> GetUsersAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USERS_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return []; - } - - return BinaryMapper.MapUsers(responseBuffer); - } - - /// - public async Task CreateUserAsync(string userName, string password, UserStatus status, - Permissions? permissions = null, CancellationToken token = default) - { - var message = TcpContracts.CreateUser(userName, password, status, permissions); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_USER_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return null; - } - - return BinaryMapper.MapUser(responseBuffer); - } - - /// - public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default) - { - var message = TcpContracts.DeleteUser(userId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_USER_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null, - CancellationToken token = default) - { - var message = TcpContracts.UpdateUser(userId, userName, status); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_USER_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null, - CancellationToken token = default) - { - var message = TcpContracts.UpdatePermissions(userId, permissions); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_PERMISSIONS_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword, - CancellationToken token = default) - { - var message = TcpContracts.ChangePassword(userId, currentPassword, newPassword); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CHANGE_PASSWORD_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task LoginUserAsync(string userName, string password, CancellationToken token = default) - { - if (_state == ConnectionState.Disconnected) - { - throw new NotConnectedException(); - } - - // TODO: Add binary protocol version - var message = TcpContracts.LoginUser(userName, password, SdkVersion.Value, "csharp-sdk"); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_USER_CODE); - - SetConnectionStateAsync(ConnectionState.Authenticating); - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length <= 0) - { - return null; - } - - var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.AsSpan()[..responseBuffer.Length]); - SetConnectionStateAsync(ConnectionState.Authenticated); - - if (await RedirectAsync(token)) - { - await ConnectAsync(token); - return await LoginUserAsync(userName, password, token); - } - - var authResponse = new AuthResponse(userId, null); - return authResponse; - } - - /// - public async Task LogoutUserAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGOUT_USER_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task> GetPersonalAccessTokensAsync( - CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_PERSONAL_ACCESS_TOKENS_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return []; - } - - return BinaryMapper.MapPersonalAccessTokens(responseBuffer); - } - - /// - public async Task CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null, - CancellationToken token = default) - { - var message = TcpContracts.CreatePersonalAccessToken(name, DurationHelpers.ToDuration(expiry)); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE); - - var responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Length == 0) - { - return null; - } - - return BinaryMapper.MapRawPersonalAccessToken(responseBuffer); - } - - /// - public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default) - { - var message = TcpContracts.DeletePersonalRequestToken(name); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PERSONAL_ACCESS_TOKEN_CODE); - - await SendWithResponseAsync(payload, token); - } - - /// - public async Task LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default) - { - var message = TcpContracts.LoginWithPersonalAccessToken(token); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE); - - SetConnectionStateAsync(ConnectionState.Authenticating); - var responseBuffer = await SendWithResponseAsync(payload, ct); - - if (responseBuffer.Length <= 1) - { - return null; - } - - var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.AsSpan()[..4]); - - SetConnectionStateAsync(ConnectionState.Authenticated); - - if (await RedirectAsync(ct)) - { - await ConnectAsync(ct); - return await LoginWithPersonalAccessTokenAsync(token, ct); - } - - return new AuthResponse(userId, null); - } - - private async Task TryEstablishConnectionAsync(CancellationToken token) - { - var retryCount = 0; - var delay = _configuration.ReconnectionSettings.InitialDelay; - do - { - // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract - _stream?.Close(); - _stream?.Dispose(); - - if (string.IsNullOrEmpty(_currentAddress)) - { - _currentAddress = _configuration.BaseAddress; - } - - var urlPortSplitter = _currentAddress.Split(":"); - if (urlPortSplitter.Length > 2) - { - throw new InvalidBaseAddressException(); - } - - try - { - var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - socket.SendBufferSize = _configuration.SendBufferSize; - socket.ReceiveBufferSize = _configuration.ReceiveBufferSize; - - await socket.ConnectAsync(urlPortSplitter[0], int.Parse(urlPortSplitter[1]), token); - - socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true); - socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 5); - - SetConnectionStateAsync(ConnectionState.Connected); - _lastConnectionTime = DateTimeOffset.UtcNow; - - _stream = _configuration.TlsSettings.Enabled switch - { - true => await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings), - false => new TcpConnectionStream(new NetworkStream(socket, true)) - }; - - if (_configuration.AutoLoginSettings.Enabled) - { - _logger.LogInformation("Auto login enabled. Trying to login with credentials: {Username}", - _configuration.AutoLoginSettings.Username); - await LoginUserAsync(_configuration.AutoLoginSettings.Username, - _configuration.AutoLoginSettings.Password, token); - } - - break; - } - catch (Exception e) - { - _logger.LogError(e, "Failed to connect"); - - if (!_configuration.ReconnectionSettings.Enabled || - (_configuration.ReconnectionSettings.MaxRetries > 0 && - retryCount >= _configuration.ReconnectionSettings.MaxRetries)) - { - SetConnectionStateAsync(ConnectionState.Disconnected); - throw; - } - - retryCount++; - if (_configuration.ReconnectionSettings.UseExponentialBackoff) - { - delay *= _configuration.ReconnectionSettings.BackoffMultiplier; - - if (delay > _configuration.ReconnectionSettings.MaxDelay) - { - delay = _configuration.ReconnectionSettings.MaxDelay; - } - } - - if (_logger.IsEnabled(LogLevel.Information)) - { - _logger.LogInformation("Retrying connection attempt {RetryCount} with delay {Delay}", retryCount, - delay); - } - - await Task.Delay(delay, token); - } - } while (true); - } - - private async Task GetCurrentLeaderNodeAsync(CancellationToken token) - { - try - { - var clusterMetadata = await GetClusterMetadataAsync(token); - if (clusterMetadata == null) - { - return null; - } - - // Single-node cluster (clustering disabled) - no redirection needed - if (clusterMetadata.Nodes.Count() == 1) - { - return null; - } - - var leaderNode = clusterMetadata.Nodes.FirstOrDefault(x => x.Role == ClusterNodeRole.Leader); - if (leaderNode == null) - { - throw new MissingLeaderException(); - } - - return leaderNode; - } - // todo: change after error refactoring, error code 5 is for feature not supported - catch (IggyInvalidStatusCodeException e) when (e.StatusCode == 5) - { - return null; - } - } - - private async Task CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings) - { - ValidateCertificatePath(tlsSettings.CertificatePath); - - _customCaStore = new X509Certificate2Collection(); - _customCaStore.ImportFromPemFile(tlsSettings.CertificatePath); - var stream = new NetworkStream(socket, true); - var sslStream = new SslStream(stream, false, RemoteCertificateValidationCallback); - - await sslStream.AuthenticateAsClientAsync(tlsSettings.Hostname); - - return new TcpConnectionStream(sslStream); - } - - private async Task SendWithResponseAsync(byte[] payload, CancellationToken token = default) - { - try - { - return await SendRawAsync(payload, token); - } - catch (Exception e) when (IsConnectionException(e) && !_isConnecting) - { - _logger.LogWarning("Connection lost"); - if (!_configuration.ReconnectionSettings.Enabled) - { - _logger.LogWarning("Reconnection is disabled"); - SetConnectionStateAsync(ConnectionState.Disconnected); - throw; - } - - return await HandleReconnectionAsync(payload, token); - } - } - - private async Task HandleReconnectionAsync(byte[] payload, CancellationToken token) - { - var currentTime = DateTimeOffset.UtcNow; - await _connectionSemaphore.WaitAsync(token); - - try - { - if (_state is ConnectionState.Connected or ConnectionState.Authenticated - && _lastConnectionTime > currentTime) - { - _logger.LogInformation("Connection already established, sending payload"); - return await SendRawAsync(payload, token); - } - - SetConnectionStateAsync(ConnectionState.Disconnected); - _logger.LogInformation("Reconnecting to the server"); - await ConnectAsync(token); - - _logger.LogInformation("Reconnected to the server"); - - await Task.Delay(_configuration.ReconnectionSettings.WaitAfterReconnect, token); - - return await SendRawAsync(payload, token); - } - finally - { - _connectionSemaphore.Release(); - } - } - - private async Task SendRawAsync(byte[] payload, CancellationToken token) - { - if (_state is ConnectionState.Disconnected or ConnectionState.Connecting) - { - throw new NotConnectedException(); - } - - try - { - await _sendingSemaphore.WaitAsync(token); - await _stream.SendAsync(payload, token); - await _stream.FlushAsync(token); - - // Read the 8-byte header (4 bytes status + 4 bytes length) - var buffer = new byte[BufferSizes.EXPECTED_RESPONSE_SIZE]; - var totalRead = 0; - while (totalRead < BufferSizes.EXPECTED_RESPONSE_SIZE) - { - var readBytes - = await _stream.ReadAsync( - buffer.AsMemory(totalRead, BufferSizes.EXPECTED_RESPONSE_SIZE - totalRead), - token); - if (readBytes == 0) - { - throw new IggyZeroBytesException(); - } - - totalRead += readBytes; - } - - var response = TcpMessageStreamHelpers.GetResponseLengthAndStatus(buffer); - - if (response.Status != 0) - { - if (response.Length == 0) - { - throw new IggyInvalidStatusCodeException(response.Status, - $"Invalid response status code: {response.Status}"); - } - - var errorBuffer = new byte[response.Length]; - totalRead = 0; - while (totalRead < response.Length) - { - var readBytes - = await _stream.ReadAsync(errorBuffer.AsMemory(totalRead, response.Length - totalRead), token); - if (readBytes == 0) - { - throw new IggyZeroBytesException(); - } - - totalRead += readBytes; - } - - throw new InvalidResponseException(Encoding.UTF8.GetString(errorBuffer)); - } - - if (response.Length == 0) - { - return []; - } - - var responseBuffer = new byte[response.Length]; - totalRead = 0; - while (totalRead < response.Length) - { - var readBytes = await _stream.ReadAsync(responseBuffer.AsMemory(totalRead, response.Length - totalRead), - token); - if (readBytes == 0) - { - throw new IggyZeroBytesException(); - } - - totalRead += readBytes; - } - - return responseBuffer; - } - finally - { - _sendingSemaphore.Release(); - } - } - - private static bool IsConnectionException(Exception ex) - { - return ex is IggyZeroBytesException or - NotConnectedException or - SocketException or - IOException or - ObjectDisposedException; - } - - private static int CalculatePayloadBufferSize(int messageBufferSize) - { - return messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH; - } - - private static int CalculateMessageBufferSize(Identifier streamId, Identifier topicId, Consumer consumer) - { - // Original: 14 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.Id.Length - // Added 1 byte for partition flag - return 15 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.ConsumerId.Length; - } - - /// - /// Sets the connection state and fires the OnConnectionStateChanged event. - /// - /// The new connection state - private void SetConnectionStateAsync(ConnectionState newState) - { - if (_state == newState) - { - return; - } - - var previousState = _state; - _state = newState; - - _logger.LogInformation("Connection state changed: {PreviousState} -> {CurrentState}", previousState, newState); - _connectionEvents.Publish(new ConnectionStateChangedEventArgs(previousState, newState)); - } - - private void ValidateCertificatePath(string tlsCertificatePath) - { - if (string.IsNullOrEmpty(tlsCertificatePath) - || !File.Exists(tlsCertificatePath)) - { - throw new InvalidCertificatePathException(tlsCertificatePath); - } - } - - private bool RemoteCertificateValidationCallback(object sender, X509Certificate? certificate, X509Chain? chain, - SslPolicyErrors sslPolicyErrors) - { - if (sslPolicyErrors == SslPolicyErrors.None) - { - return true; - } - - if (certificate is null) - { - return false; - } - - if (certificate is not X509Certificate2 serverCert) - { - serverCert = new X509Certificate2(certificate); - } - - if (_customCaStore.Any(ca => ca.Thumbprint == serverCert.Thumbprint)) - { - if (DateTime.UtcNow <= serverCert.NotAfter && DateTime.UtcNow >= serverCert.NotBefore) - { - return true; - } - - _logger.LogError( - "Server certificate matches trusted key but is expired. Valid from {NotBefore} to {NotAfter}", - serverCert.NotBefore, serverCert.NotAfter); - return false; - } - - - using var customChain = new X509Chain(); - customChain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; - customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; - foreach (var ca in _customCaStore) - { - customChain.ChainPolicy.CustomTrustStore.Add(ca); - customChain.ChainPolicy.ExtraStore.Add(ca); - } - - customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; - - if (customChain.Build(new X509Certificate2(certificate))) - { - if (!sslPolicyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch)) - { - return true; - } - - _logger.LogError("Custom CA chain is valid, but hostname does not match"); - return false; - } - - foreach (var chainStatus in customChain.ChainStatus) - { - _logger.LogWarning("Certificate validation failed: {ChainStatus} - {StatusInformation}", chainStatus.Status, - chainStatus.StatusInformation); - } - - return false; - } - - private async Task RedirectAsync(CancellationToken token) - { - var currentLeaderNode = await GetCurrentLeaderNodeAsync(token); - if (currentLeaderNode == null) - { - return false; - } - - var leaderAddress = $"{currentLeaderNode.Ip}:{currentLeaderNode.Endpoints.Tcp}"; - if (leaderAddress == _currentAddress) - { - return false; - } - - _currentAddress = leaderAddress; - - _logger.LogInformation("Leader address changed. Trying to reconnect to {Address}", - leaderAddress); - - _stream.Close(); - SetConnectionStateAsync(ConnectionState.Disconnected); - return true; - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers; +using System.Buffers.Binary; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using Apache.Iggy.Configuration; +using Apache.Iggy.ConnectionStream; +using Apache.Iggy.Contracts; +using Apache.Iggy.Contracts.Auth; +using Apache.Iggy.Contracts.Tcp; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Kinds; +using Apache.Iggy.Mappers; +using Apache.Iggy.Messages; +using Apache.Iggy.Utils; +using Microsoft.Extensions.Logging; +using Partitioning = Apache.Iggy.Kinds.Partitioning; + +namespace Apache.Iggy.IggyClient.Implementations; + +/// +/// A TCP client for interacting with the Iggy server. +/// +public sealed class TcpMessageStream : IIggyClient +{ + private readonly IggyClientConfigurator _configuration; + private readonly EventAggregator _connectionEvents; + private readonly SemaphoreSlim _connectionSemaphore; + private readonly ILogger _logger; + private readonly byte[] _responseHeaderBuffer = new byte[BufferSizes.EXPECTED_RESPONSE_SIZE]; + private readonly SemaphoreSlim _sendingSemaphore; + private string _currentAddress = string.Empty; + private X509Certificate2Collection _customCaStore = []; + private bool _isConnecting; + private DateTimeOffset _lastConnectionTime; + private ConnectionState _state = ConnectionState.Disconnected; + private TcpConnectionStream _stream = null!; + + internal TcpMessageStream(IggyClientConfigurator configuration, ILoggerFactory loggerFactory) + { + _configuration = configuration; + _logger = loggerFactory.CreateLogger(); + _sendingSemaphore = new SemaphoreSlim(1, 1); + _connectionSemaphore = new SemaphoreSlim(1, 1); + _lastConnectionTime = DateTimeOffset.MinValue; + _connectionEvents = new EventAggregator(loggerFactory); + } + + /// + /// Fired whenever the connection state changes. + /// + //public event EventHandler? OnConnectionStateChanged; + public void Dispose() + { + _stream?.Close(); + _stream?.Dispose(); + _sendingSemaphore.Dispose(); + _connectionSemaphore.Dispose(); + _connectionEvents.Clear(); + } + + /// + public void SubscribeConnectionEvents(Func callback) + { + _connectionEvents.Subscribe(callback); + } + + /// + public void UnsubscribeConnectionEvents(Func callback) + { + _connectionEvents.Unsubscribe(callback); + } + + /// + public string GetCurrentAddress() + { + return _currentAddress; + } + + /// + public async Task CreateStreamAsync(string name, CancellationToken token = default) + { + var message = TcpContracts.CreateStream(name); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_STREAM_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + throw new InvalidResponseException("Received empty response while trying to create stream."); + } + + return BinaryMapper.MapStream(responseBuffer.Memory.Span); + } + + /// + public async Task GetStreamByIdAsync(Identifier streamId, CancellationToken token = default) + { + var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAM_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapStream(responseBuffer.Memory.Span); + } + + /// + public async Task> GetStreamsAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAMS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return []; + } + + return BinaryMapper.MapStreams(responseBuffer.Memory.Span); + } + + /// + public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default) + { + var message = TcpContracts.UpdateStream(streamId, name); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_STREAM_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default) + { + var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_STREAM_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default) + { + var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_STREAM_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task> GetTopicsAsync(Identifier streamId, + CancellationToken token = default) + { + var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPICS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return []; + } + + return BinaryMapper.MapTopics(responseBuffer.Memory.Span); + } + + /// + public async Task GetTopicByIdAsync(Identifier streamId, Identifier topicId, + CancellationToken token = default) + { + var message = TcpContracts.GetTopicById(streamId, topicId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPIC_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapTopic(responseBuffer.Memory.Span); + } + + /// + public async Task CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, + CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null, + TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default) + { + var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry); + var message = TcpContracts.CreateTopic(streamId, name, partitionsCount, compressionAlgorithm, + replicationFactor, messageExpiryValue, maxTopicSize); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_TOPIC_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapTopic(responseBuffer.Memory.Span); + } + + /// + public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, + CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, + ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null, + CancellationToken token = default) + { + var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry); + var message = TcpContracts.UpdateTopic(streamId, topicId, name, compressionAlgorithm, maxTopicSize, + messageExpiryValue, replicationFactor); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_TOPIC_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) + { + var message = TcpContracts.DeleteTopic(streamId, topicId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_TOPIC_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) + { + var message = TcpContracts.PurgeTopic(streamId, topicId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_TOPIC_CODE); + + await SendAckAsync(payload, token); + } + + + /// + public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, + IList messages, CancellationToken token = default) + { + var metadataLength = 2 + streamId.Length + 2 + topicId.Length + + 2 + partitioning.Length + 4 + 4; + var messageBufferSize = TcpMessageStreamHelpers.CalculateMessageBytesCount(messages) + + metadataLength; + var payloadBufferSize = messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH; + + IMemoryOwner messageBuffer = MemoryPool.Shared.Rent(messageBufferSize); + IMemoryOwner payloadBuffer = MemoryPool.Shared.Rent(payloadBufferSize); + try + { + TcpContracts.CreateMessage(messageBuffer.Memory.Span[..messageBufferSize], streamId, + topicId, partitioning, messages); + + TcpMessageStreamHelpers.CreatePayload(payloadBuffer.Memory.Span[..payloadBufferSize], + messageBuffer.Memory.Span[..messageBufferSize], CommandCodes.SEND_MESSAGES_CODE); + + await SendAckAsync(payloadBuffer.Memory[..payloadBufferSize], token); + } + finally + { + messageBuffer.Dispose(); + payloadBuffer.Dispose(); + } + } + + /// + public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, + CancellationToken token = default) + { + var message = TcpContracts.FlushUnsavedBuffer(streamId, topicId, partitionId, fsync); + + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.FLUSH_UNSAVED_BUFFER_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId, + Consumer consumer, + PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) + { + using var rental = await PollMessagesRentedAsync(streamId, topicId, partitionId, consumer, pollingStrategy, + count, autoCommit, token); + return BinaryMapper.MaterializeMessages(rental); + } + + /// + public async Task PollMessagesRentedAsync(Identifier streamId, Identifier topicId, + uint? partitionId, + Consumer consumer, + PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) + { + var messageBufferSize = CalculateMessageBufferSize(streamId, topicId, consumer); + var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize); + var payload = ArrayPool.Shared.Rent(payloadBufferSize); + + try + { + TcpContracts.GetMessages(payload.AsSpan().Slice(8, messageBufferSize), consumer, streamId, + topicId, pollingStrategy, count, autoCommit, partitionId); + BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[..4], messageBufferSize + 4); + BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[4..8], CommandCodes.POLL_MESSAGES_CODE); + + IMemoryOwner responseBuffer + = await SendWithResponseAsync(payload.AsMemory(0, payloadBufferSize), token); + return BinaryMapper.MapRentedMessages(responseBuffer); + } + finally + { + ArrayPool.Shared.Return(payload); + } + } + + /// + public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset, + uint? partitionId, CancellationToken token = default) + { + var message = TcpContracts.UpdateOffset(streamId, topicId, consumer, offset, partitionId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.STORE_CONSUMER_OFFSET_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, + uint? partitionId, CancellationToken token = default) + { + var message = TcpContracts.GetOffset(streamId, topicId, consumer, partitionId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_OFFSET_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapOffsets(responseBuffer.Memory.Span); + } + + /// + public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId, + CancellationToken token = default) + { + var message = TcpContracts.DeleteOffset(streamId, topicId, consumer, partitionId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_OFFSET_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task> GetConsumerGroupsAsync(Identifier streamId, + Identifier topicId, + CancellationToken token = default) + { + var message = TcpContracts.GetGroups(streamId, topicId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUPS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return []; + } + + return BinaryMapper.MapConsumerGroups(responseBuffer.Memory.Span); + } + + /// + public async Task GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId, + Identifier groupId, CancellationToken token = default) + { + var message = TcpContracts.GetGroup(streamId, topicId, groupId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUP_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapConsumerGroup(responseBuffer.Memory.Span); + } + + /// + public async Task CreateConsumerGroupAsync(Identifier streamId, Identifier topicId, + string name, CancellationToken token = default) + { + var message = TcpContracts.CreateGroup(streamId, topicId, name); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_CONSUMER_GROUP_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapConsumerGroup(responseBuffer.Memory.Span); + } + + /// + public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, + CancellationToken token = default) + { + var message = TcpContracts.DeleteGroup(streamId, topicId, groupId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_GROUP_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, + CancellationToken token = default) + { + var message = TcpContracts.JoinGroup(streamId, topicId, groupId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.JOIN_CONSUMER_GROUP_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, + CancellationToken token = default) + { + var message = TcpContracts.LeaveGroup(streamId, topicId, groupId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LEAVE_CONSUMER_GROUP_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, + CancellationToken token = default) + { + var message = TcpContracts.DeletePartitions(streamId, topicId, partitionsCount); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PARTITIONS_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, + CancellationToken token = default) + { + var message = TcpContracts.CreatePartitions(streamId, topicId, partitionsCount); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PARTITIONS_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId, + uint segmentsCount, CancellationToken token = default) + { + var message = TcpContracts.DeleteSegments(streamId, topicId, partitionId, segmentsCount); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_SEGMENTS_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task GetMeAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_ME_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapClient(responseBuffer.Memory.Span); + } + + /// + public async Task GetStatsAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STATS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapStats(responseBuffer.Memory.Span); + } + + /// + public async Task GetClusterMetadataAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLUSTER_METADATA_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapClusterMetadata(responseBuffer.Memory.Span); + } + + /// + public async Task PingAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PING_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task GetSnapshotAsync(SnapshotCompression compression, + IList snapshotTypes, CancellationToken token = default) + { + var message = TcpContracts.GetSnapshot(compression, snapshotTypes); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_SNAPSHOT_CODE); + + using IMemoryOwner result = await SendWithResponseAsync(payload, token); + + return result.Memory.Span.ToArray(); + } + + /// + public async Task ConnectAsync(CancellationToken token = default) + { + if (_state is ConnectionState.Connected + or ConnectionState.Authenticating + or ConnectionState.Authenticated) + { + _logger.LogWarning("Connection is already connected"); + return; + } + + if (_lastConnectionTime != DateTimeOffset.MinValue) + { + await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token); + } + + SetConnectionStateAsync(ConnectionState.Connecting); + _isConnecting = true; + try + { + await TryEstablishConnectionAsync(token); + } + finally + { + _isConnecting = false; + } + } + + /// + public async Task> GetClientsAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENTS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return []; + } + + return BinaryMapper.MapClients(responseBuffer.Memory.Span); + } + + /// + public async Task GetClientByIdAsync(uint clientId, CancellationToken token = default) + { + var message = TcpContracts.GetClient(clientId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENT_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapClient(responseBuffer.Memory.Span); + } + + /// + public async Task GetUserAsync(Identifier userId, CancellationToken token = default) + { + var message = TcpContracts.GetUser(userId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USER_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapUser(responseBuffer.Memory.Span); + } + + /// + public async Task> GetUsersAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USERS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return []; + } + + return BinaryMapper.MapUsers(responseBuffer.Memory.Span); + } + + /// + public async Task CreateUserAsync(string userName, string password, UserStatus status, + Permissions? permissions = null, CancellationToken token = default) + { + var message = TcpContracts.CreateUser(userName, password, status, permissions); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_USER_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapUser(responseBuffer.Memory.Span); + } + + /// + public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default) + { + var message = TcpContracts.DeleteUser(userId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_USER_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null, + CancellationToken token = default) + { + var message = TcpContracts.UpdateUser(userId, userName, status); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_USER_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null, + CancellationToken token = default) + { + var message = TcpContracts.UpdatePermissions(userId, permissions); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_PERMISSIONS_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword, + CancellationToken token = default) + { + var message = TcpContracts.ChangePassword(userId, currentPassword, newPassword); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CHANGE_PASSWORD_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task LoginUserAsync(string userName, string password, CancellationToken token = default) + { + if (_state == ConnectionState.Disconnected) + { + throw new NotConnectedException(); + } + + // TODO: Add binary protocol version + var message = TcpContracts.LoginUser(userName, password, SdkVersion.Value, "csharp-sdk"); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_USER_CODE); + + SetConnectionStateAsync(ConnectionState.Authenticating); + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..responseBuffer.Memory.Length]); + SetConnectionStateAsync(ConnectionState.Authenticated); + + if (await RedirectAsync(token)) + { + await ConnectAsync(token); + return await LoginUserAsync(userName, password, token); + } + + var authResponse = new AuthResponse(userId, null); + return authResponse; + } + + /// + public async Task LogoutUserAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGOUT_USER_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task> GetPersonalAccessTokensAsync( + CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_PERSONAL_ACCESS_TOKENS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return []; + } + + return BinaryMapper.MapPersonalAccessTokens(responseBuffer.Memory.Span); + } + + /// + public async Task CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null, + CancellationToken token = default) + { + var message = TcpContracts.CreatePersonalAccessToken(name, DurationHelpers.ToDuration(expiry)); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapRawPersonalAccessToken(responseBuffer.Memory.Span); + } + + /// + public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default) + { + var message = TcpContracts.DeletePersonalRequestToken(name); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PERSONAL_ACCESS_TOKEN_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default) + { + var message = TcpContracts.LoginWithPersonalAccessToken(token); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE); + + SetConnectionStateAsync(ConnectionState.Authenticating); + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, ct); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..4]); + + SetConnectionStateAsync(ConnectionState.Authenticated); + + if (await RedirectAsync(ct)) + { + await ConnectAsync(ct); + return await LoginWithPersonalAccessTokenAsync(token, ct); + } + + return new AuthResponse(userId, null); + } + + private async Task TryEstablishConnectionAsync(CancellationToken token) + { + var retryCount = 0; + var delay = _configuration.ReconnectionSettings.InitialDelay; + do + { + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract + _stream?.Close(); + _stream?.Dispose(); + + if (string.IsNullOrEmpty(_currentAddress)) + { + _currentAddress = _configuration.BaseAddress; + } + + var urlPortSplitter = _currentAddress.Split(":"); + if (urlPortSplitter.Length > 2) + { + throw new InvalidBaseAddressException(); + } + + try + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + socket.SendBufferSize = _configuration.SendBufferSize; + socket.ReceiveBufferSize = _configuration.ReceiveBufferSize; + + await socket.ConnectAsync(urlPortSplitter[0], int.Parse(urlPortSplitter[1]), token); + + socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true); + socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 5); + + SetConnectionStateAsync(ConnectionState.Connected); + _lastConnectionTime = DateTimeOffset.UtcNow; + + _stream = _configuration.TlsSettings.Enabled switch + { + true => await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings), + false => new TcpConnectionStream(new NetworkStream(socket, true)) + }; + + if (_configuration.AutoLoginSettings.Enabled) + { + _logger.LogInformation("Auto login enabled. Trying to login with credentials: {Username}", + _configuration.AutoLoginSettings.Username); + await LoginUserAsync(_configuration.AutoLoginSettings.Username, + _configuration.AutoLoginSettings.Password, token); + } + + break; + } + catch (Exception e) + { + _logger.LogError(e, "Failed to connect"); + + if (!_configuration.ReconnectionSettings.Enabled || + (_configuration.ReconnectionSettings.MaxRetries > 0 && + retryCount >= _configuration.ReconnectionSettings.MaxRetries)) + { + SetConnectionStateAsync(ConnectionState.Disconnected); + throw; + } + + retryCount++; + if (_configuration.ReconnectionSettings.UseExponentialBackoff) + { + delay *= _configuration.ReconnectionSettings.BackoffMultiplier; + + if (delay > _configuration.ReconnectionSettings.MaxDelay) + { + delay = _configuration.ReconnectionSettings.MaxDelay; + } + } + + if (_logger.IsEnabled(LogLevel.Information)) + { + _logger.LogInformation("Retrying connection attempt {RetryCount} with delay {Delay}", retryCount, + delay); + } + + await Task.Delay(delay, token); + } + } while (true); + } + + private async Task GetCurrentLeaderNodeAsync(CancellationToken token) + { + try + { + var clusterMetadata = await GetClusterMetadataAsync(token); + if (clusterMetadata == null) + { + return null; + } + + // Single-node cluster (clustering disabled) - no redirection needed + if (clusterMetadata.Nodes.Count() == 1) + { + return null; + } + + var leaderNode = clusterMetadata.Nodes.FirstOrDefault(x => x.Role == ClusterNodeRole.Leader); + if (leaderNode == null) + { + throw new MissingLeaderException(); + } + + return leaderNode; + } + // todo: change after error refactoring, error code 5 is for feature not supported + catch (IggyInvalidStatusCodeException e) when (e.StatusCode == 5) + { + return null; + } + } + + private async Task CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings) + { + ValidateCertificatePath(tlsSettings.CertificatePath); + + _customCaStore = new X509Certificate2Collection(); + _customCaStore.ImportFromPemFile(tlsSettings.CertificatePath); + var stream = new NetworkStream(socket, true); + var sslStream = new SslStream(stream, false, RemoteCertificateValidationCallback); + + await sslStream.AuthenticateAsClientAsync(tlsSettings.Hostname); + + return new TcpConnectionStream(sslStream); + } + + private async Task SendAckAsync(ReadOnlyMemory payload, CancellationToken token = default) + { + using IMemoryOwner _ = await SendWithResponseAsync(payload, token); + } + + private async Task> SendWithResponseAsync(ReadOnlyMemory payload, + CancellationToken token = default) + { + try + { + return await SendRawAsync(payload, token); + } + catch (Exception e) when (IsConnectionException(e) && !_isConnecting) + { + _logger.LogWarning("Connection lost"); + if (!_configuration.ReconnectionSettings.Enabled) + { + _logger.LogWarning("Reconnection is disabled"); + SetConnectionStateAsync(ConnectionState.Disconnected); + throw; + } + + return await HandleReconnectionAsync(payload, token); + } + } + + private async Task> HandleReconnectionAsync(ReadOnlyMemory payload, + CancellationToken token) + { + var currentTime = DateTimeOffset.UtcNow; + await _connectionSemaphore.WaitAsync(token); + + try + { + if (_state is ConnectionState.Connected or ConnectionState.Authenticated + && _lastConnectionTime > currentTime) + { + _logger.LogInformation("Connection already established, sending payload"); + return await SendRawAsync(payload, token); + } + + SetConnectionStateAsync(ConnectionState.Disconnected); + _logger.LogInformation("Reconnecting to the server"); + await ConnectAsync(token); + + _logger.LogInformation("Reconnected to the server"); + + await Task.Delay(_configuration.ReconnectionSettings.WaitAfterReconnect, token); + + return await SendRawAsync(payload, token); + } + finally + { + _connectionSemaphore.Release(); + } + } + + private async Task> SendRawAsync(ReadOnlyMemory payload, CancellationToken token) + { + if (_state is ConnectionState.Disconnected or ConnectionState.Connecting) + { + throw new NotConnectedException(); + } + + await _sendingSemaphore.WaitAsync(token); + + try + { + await _stream.SendAsync(payload, token); + await _stream.FlushAsync(token); + + // Read the 8-byte header (4 bytes status + 4 bytes length) + var totalRead = 0; + while (totalRead < BufferSizes.EXPECTED_RESPONSE_SIZE) + { + var readBytes + = await _stream.ReadAsync( + _responseHeaderBuffer.AsMemory(totalRead, BufferSizes.EXPECTED_RESPONSE_SIZE - totalRead), + token); + if (readBytes == 0) + { + throw new IggyZeroBytesException(); + } + + totalRead += readBytes; + } + + var response = TcpMessageStreamHelpers.GetResponseLengthAndStatus(_responseHeaderBuffer); + + if (response.Status != 0) + { + if (response.Length == 0) + { + throw new IggyInvalidStatusCodeException(response.Status, + $"Invalid response status code: {response.Status}"); + } + + var errorBuffer = new byte[response.Length]; + totalRead = 0; + while (totalRead < response.Length) + { + var readBytes + = await _stream.ReadAsync(errorBuffer.AsMemory(totalRead, response.Length - totalRead), token); + if (readBytes == 0) + { + throw new IggyZeroBytesException(); + } + + totalRead += readBytes; + } + + throw new InvalidResponseException(Encoding.UTF8.GetString(errorBuffer)); + } + + if (response.Length == 0) + { + return EmptyMemoryOwner.Instance; + } + + var responseBuffer = ArrayPoolHelper.Rent(response.Length); + try + { + totalRead = 0; + while (totalRead < response.Length) + { + var readBytes + = await _stream.ReadAsync(responseBuffer.Memory.Slice(totalRead, response.Length - totalRead), + token); + + if (readBytes == 0) + { + throw new IggyZeroBytesException(); + } + + totalRead += readBytes; + } + } + catch + { + responseBuffer.Dispose(); + throw; + } + + return responseBuffer; + } + finally + { + _sendingSemaphore.Release(); + } + } + + private static bool IsConnectionException(Exception ex) + { + return ex is IggyZeroBytesException or + NotConnectedException or + SocketException or + IOException or + ObjectDisposedException; + } + + private static int CalculatePayloadBufferSize(int messageBufferSize) + { + return messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH; + } + + private static int CalculateMessageBufferSize(Identifier streamId, Identifier topicId, Consumer consumer) + { + // Original: 14 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.Id.Length + // Added 1 byte for partition flag + return 15 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.ConsumerId.Length; + } + + /// + /// Sets the connection state and fires the OnConnectionStateChanged event. + /// + /// The new connection state + private void SetConnectionStateAsync(ConnectionState newState) + { + if (_state == newState) + { + return; + } + + var previousState = _state; + _state = newState; + + _logger.LogInformation("Connection state changed: {PreviousState} -> {CurrentState}", previousState, newState); + _connectionEvents.Publish(new ConnectionStateChangedEventArgs(previousState, newState)); + } + + private void ValidateCertificatePath(string tlsCertificatePath) + { + if (string.IsNullOrEmpty(tlsCertificatePath) + || !File.Exists(tlsCertificatePath)) + { + throw new InvalidCertificatePathException(tlsCertificatePath); + } + } + + private bool RemoteCertificateValidationCallback(object sender, X509Certificate? certificate, X509Chain? chain, + SslPolicyErrors sslPolicyErrors) + { + if (sslPolicyErrors == SslPolicyErrors.None) + { + return true; + } + + if (certificate is null) + { + return false; + } + + if (certificate is not X509Certificate2 serverCert) + { + serverCert = new X509Certificate2(certificate); + } + + if (_customCaStore.Any(ca => ca.Thumbprint == serverCert.Thumbprint)) + { + if (DateTime.UtcNow <= serverCert.NotAfter && DateTime.UtcNow >= serverCert.NotBefore) + { + return true; + } + + _logger.LogError( + "Server certificate matches trusted key but is expired. Valid from {NotBefore} to {NotAfter}", + serverCert.NotBefore, serverCert.NotAfter); + return false; + } + + + using var customChain = new X509Chain(); + customChain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; + customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; + foreach (var ca in _customCaStore) + { + customChain.ChainPolicy.CustomTrustStore.Add(ca); + customChain.ChainPolicy.ExtraStore.Add(ca); + } + + customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; + + if (customChain.Build(new X509Certificate2(certificate))) + { + if (!sslPolicyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch)) + { + return true; + } + + _logger.LogError("Custom CA chain is valid, but hostname does not match"); + return false; + } + + foreach (var chainStatus in customChain.ChainStatus) + { + _logger.LogWarning("Certificate validation failed: {ChainStatus} - {StatusInformation}", chainStatus.Status, + chainStatus.StatusInformation); + } + + return false; + } + + private async Task RedirectAsync(CancellationToken token) + { + var currentLeaderNode = await GetCurrentLeaderNodeAsync(token); + if (currentLeaderNode == null) + { + return false; + } + + var leaderAddress = $"{currentLeaderNode.Ip}:{currentLeaderNode.Endpoints.Tcp}"; + if (leaderAddress == _currentAddress) + { + return false; + } + + _currentAddress = leaderAddress; + + _logger.LogInformation("Leader address changed. Trying to reconnect to {Address}", + leaderAddress); + + _stream.Close(); + SetConnectionStateAsync(ConnectionState.Disconnected); + return true; + } + + internal sealed class EmptyMemoryOwner : IMemoryOwner + { + public static readonly EmptyMemoryOwner Instance = new(); + + private EmptyMemoryOwner() + { + } + + public Memory Memory => Memory.Empty; + + public void Dispose() + { + } + } +} + +internal static class ArrayPoolHelper +{ + public static SlicedMemoryOwner Rent(int minimumLength) + { + return new SlicedMemoryOwner(minimumLength); + } + + internal sealed class SlicedMemoryOwner(int minimumLength) : IMemoryOwner + { + private readonly byte[] _value = ArrayPool.Shared.Rent(minimumLength); + private int _disposed; + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + ArrayPool.Shared.Return(_value); + } + + public Memory Memory => _value.AsMemory()[..minimumLength]; + } +} diff --git a/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs b/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs index 9cb9821d82..ffdbf32603 100644 --- a/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs +++ b/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs @@ -1,98 +1,106 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Text.Json; -using System.Text.Json.Serialization; -using Apache.Iggy.Contracts; -using Apache.Iggy.Headers; -using Apache.Iggy.Messages; - -namespace Apache.Iggy.JsonConverters; - -internal sealed class MessageResponseConverter : JsonConverter -{ - private static readonly UserHeadersConverter HeadersConverter = new(); - - public override MessageResponse Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - if (reader.TokenType != JsonTokenType.StartObject) - { - throw new JsonException("Expected start of object for MessageResponse."); - } - - MessageHeader? header = null; - byte[]? payload = null; - Dictionary? userHeaders = null; - byte[]? rawUserHeaders = null; - - while (reader.Read()) - { - if (reader.TokenType == JsonTokenType.EndObject) - { - break; - } - - if (reader.TokenType != JsonTokenType.PropertyName) - { - throw new JsonException("Expected property name."); - } - - var propertyName = reader.GetString(); - reader.Read(); - - switch (propertyName) - { - case "header": - header = JsonSerializer.Deserialize(ref reader, options); - break; - case "payload": - payload = reader.GetBytesFromBase64(); - break; - case "user_headers": - if (reader.TokenType == JsonTokenType.String) - { - rawUserHeaders = reader.GetBytesFromBase64(); - } - else if (reader.TokenType == JsonTokenType.Null) - { - userHeaders = null; - } - else - { - userHeaders = HeadersConverter.Read(ref reader, typeof(Dictionary), options); - } - break; - default: - reader.Skip(); - break; - } - } - - return new MessageResponse - { - Header = header ?? throw new JsonException("Missing 'header' field."), - Payload = payload ?? throw new JsonException("Missing 'payload' field."), - UserHeaders = userHeaders, - RawUserHeaders = rawUserHeaders - }; - } - - public override void Write(Utf8JsonWriter writer, MessageResponse value, JsonSerializerOptions options) - { - JsonSerializer.Serialize(writer, value, options); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text.Json; +using System.Text.Json.Serialization; +using Apache.Iggy.Contracts; +using Apache.Iggy.Headers; +using Apache.Iggy.Messages; + +namespace Apache.Iggy.JsonConverters; + +internal sealed class MessageResponseConverter : JsonConverter +{ + private static readonly UserHeadersConverter HeadersConverter = new(); + + public override MessageResponse Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException("Expected start of object for MessageResponse."); + } + + MessageHeader? header = null; + byte[]? payload = null; + Dictionary? userHeaders = null; + byte[]? rawUserHeaders = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + break; + } + + if (reader.TokenType != JsonTokenType.PropertyName) + { + throw new JsonException("Expected property name."); + } + + var propertyName = reader.GetString(); + reader.Read(); + + switch (propertyName) + { + case "header": + header = JsonSerializer.Deserialize(ref reader, options); + break; + case "payload": + payload = reader.GetBytesFromBase64(); + break; + case "user_headers": + if (reader.TokenType == JsonTokenType.String) + { + rawUserHeaders = reader.GetBytesFromBase64(); + } + else if (reader.TokenType == JsonTokenType.Null) + { + userHeaders = null; + } + else + { + userHeaders = HeadersConverter.Read(ref reader, typeof(Dictionary), + options); + } + + break; + default: + reader.Skip(); + break; + } + } + + var response = new MessageResponse + { + Header = header ?? throw new JsonException("Missing 'header' field."), + Payload = payload ?? throw new JsonException("Missing 'payload' field."), + RawUserHeaders = rawUserHeaders + }; + + if (rawUserHeaders is null) + { + response.UserHeaders = userHeaders; + } + + return response; + } + + public override void Write(Utf8JsonWriter writer, MessageResponse value, JsonSerializerOptions options) + { + JsonSerializer.Serialize(writer, value, options); + } +} diff --git a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs index 890b668022..ad47f03990 100644 --- a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs @@ -23,6 +23,7 @@ using Apache.Iggy.Enums; using Apache.Iggy.Extensions; using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient.Implementations; using Apache.Iggy.Messages; using Apache.Iggy.Utils; @@ -227,34 +228,38 @@ private static (UserResponse response, int position) MapToUserResponse(ReadOnlyS var readBytes = 4 + 8 + 1 + 1 + usernameLength; return (new UserResponse - { - Id = id, - CreatedAt = createdAt, - Status = userStatus, - Username = username - }, + { + Id = id, + CreatedAt = createdAt, + Status = userStatus, + Username = username + }, readBytes); } internal static ClientResponse MapClient(ReadOnlySpan payload) { var (response, position) = MapClientInfo(payload, 0); - var consumerGroups = new List(response.ConsumerGroupsCount); + var consumerGroups = new List(); + var length = payload.Length; - for (var i = 0; i < response.ConsumerGroupsCount; i++) + while (position < length) { - var streamId = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]); - var topicId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 4)..(position + 8)]); - var consumerGroupId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 8)..(position + 12)]); - var consumerGroup - = new ConsumerGroupInfo - { - StreamId = streamId, - TopicId = topicId, - GroupId = consumerGroupId - }; - consumerGroups.Add(consumerGroup); - position += 12; + for (var i = 0; i < response.ConsumerGroupsCount; i++) + { + var streamId = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]); + var topicId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 4)..(position + 8)]); + var consumerGroupId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 8)..(position + 12)]); + var consumerGroup + = new ConsumerGroupInfo + { + StreamId = streamId, + TopicId = topicId, + GroupId = consumerGroupId + }; + consumerGroups.Add(consumerGroup); + position += 12; + } } return new ClientResponse @@ -332,85 +337,88 @@ internal static OffsetResponse MapOffsets(ReadOnlySpan payload) }; } - internal static PolledMessages MapMessages(ReadOnlySpan payload) + internal static PolledMessagesRental MapRentedMessages(IMemoryOwner payloadOwner) { + try + { + return MapRentedMessages(payloadOwner.Memory, payloadOwner); + } + catch + { + payloadOwner.Dispose(); + throw; + } + } + + internal static PolledMessagesRental MapRentedMessages(ReadOnlyMemory payload, + IMemoryOwner payloadOwner) + { + ReadOnlySpan span = payload.Span; var length = payload.Length; - var partitionId = BinaryPrimitives.ReadInt32LittleEndian(payload[..4]); - var currentOffset = BinaryPrimitives.ReadUInt64LittleEndian(payload[4..12]); - var messagesCount = BinaryPrimitives.ReadUInt32LittleEndian(payload[12..16]); + var partitionId = BinaryPrimitives.ReadInt32LittleEndian(span[..4]); + var currentOffset = BinaryPrimitives.ReadUInt64LittleEndian(span[4..12]); + var messagesCount = BinaryPrimitives.ReadUInt32LittleEndian(span[12..16]); var position = 16; if (position >= length) { - return PolledMessages.Empty; + return new PolledMessagesRental(payloadOwner) + { + PartitionId = partitionId, + CurrentOffset = currentOffset, + Messages = [] + }; } - List messages = new(); + List messages = new((int)messagesCount); while (position < length) { - var checksum = BinaryPrimitives.ReadUInt64LittleEndian(payload[position..(position + 8)]); - var id = BinaryPrimitives.ReadUInt128LittleEndian(payload[(position + 8)..(position + 24)]); - var offset = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 24)..(position + 32)]); - var timestamp = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 32)..(position + 40)]); - var originTimestamp = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 40)..(position + 48)]); - var headersLength = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 48)..(position + 52)]); - var payloadLength = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 52)..(position + 56)]); - var reserved = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 56)..(position + 64)]); + var checksum = BinaryPrimitives.ReadUInt64LittleEndian(span[position..(position + 8)]); + var id = BinaryPrimitives.ReadUInt128LittleEndian(span[(position + 8)..(position + 24)]); + var offset = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 24)..(position + 32)]); + var timestamp = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 32)..(position + 40)]); + var originTimestamp = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 40)..(position + 48)]); + var headersLength = BinaryPrimitives.ReadInt32LittleEndian(span[(position + 48)..(position + 52)]); + var payloadLength = BinaryPrimitives.ReadInt32LittleEndian(span[(position + 52)..(position + 56)]); + var reserved = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 56)..(position + 64)]); var wireHeadersLength = headersLength; - byte[]? rawUserHeaders = null; - Dictionary? headers; - if (headersLength == 0) - { - headers = null; - } - else if (headersLength < 0) + if (headersLength < 0) { throw new ArgumentOutOfRangeException(); } - else - { - rawUserHeaders = payload[(position + 64 + payloadLength)..(position + 64 + payloadLength + headersLength)].ToArray(); - headers = TryMapHeaders(rawUserHeaders); - } var payloadRangeStart = position + 64; var payloadRangeEnd = position + 64 + payloadLength; - if (payloadRangeStart > length || payloadRangeEnd > length) + var headersRangeStart = payloadRangeEnd; + var headersRangeEnd = headersRangeStart + headersLength; + if (payloadRangeStart > length || payloadRangeEnd > length || headersRangeStart > length || + headersRangeEnd > length) { break; } - ReadOnlySpan payloadSlice = payload[payloadRangeStart..payloadRangeEnd]; - var messagePayload = ArrayPool.Shared.Rent(payloadSlice.Length); - var payloadSliceLen = payloadSlice.Length; + ReadOnlyMemory payloadSlice = payload.Slice(payloadRangeStart, payloadLength); + ReadOnlyMemory rawHeaders = headersLength > 0 + ? payload.Slice(headersRangeStart, headersLength) + : ReadOnlyMemory.Empty; - try + messages.Add(new RentedMessageResponse { - payloadSlice.CopyTo(messagePayload.AsSpan()[..payloadSliceLen]); - - messages.Add(new MessageResponse + Header = new MessageHeader { - Header = new MessageHeader - { - Checksum = checksum, - Id = id, - Offset = offset, - OriginTimestamp = originTimestamp, - PayloadLength = payloadLength, - Timestamp = DateTimeOffsetUtils.FromUnixTimeMicroSeconds(timestamp), - UserHeadersLength = headersLength, - Reserved = reserved - }, - UserHeaders = headers, - RawUserHeaders = rawUserHeaders, - Payload = messagePayload[..payloadSliceLen] - }); - } - finally - { - ArrayPool.Shared.Return(messagePayload); - } + Checksum = checksum, + Id = id, + Offset = offset, + OriginTimestamp = originTimestamp, + PayloadLength = payloadLength, + Timestamp = DateTimeOffsetUtils.FromUnixTimeMicroSeconds(timestamp), + UserHeadersLength = headersLength, + Reserved = reserved + }, + RawUserHeaders = rawHeaders, + Payload = payloadSlice + }); position += 64 + payloadLength + wireHeadersLength; if (position + PropertiesSize >= length) @@ -419,11 +427,59 @@ internal static PolledMessages MapMessages(ReadOnlySpan payload) } } - return new PolledMessages + return new PolledMessagesRental(payloadOwner) { PartitionId = partitionId, CurrentOffset = currentOffset, - Messages = messages.AsReadOnly() + Messages = messages + }; + } + + internal static PolledMessages MaterializeMessages(PolledMessagesRental rental) + { + if (rental.Messages.Count == 0 && rental.PartitionId == 0 && rental.CurrentOffset == 0) + { + return PolledMessages.Empty; + } + + var messages = new List(rental.Messages.Count); + foreach (var message in rental.Messages) + { + messages.Add(new MessageResponse + { + Header = message.Header, + RawUserHeaders = message.RawUserHeaders.IsEmpty ? null : message.RawUserHeaders.ToArray(), + Payload = message.Payload.ToArray() + }); + } + + return new PolledMessages + { + PartitionId = rental.PartitionId, + CurrentOffset = rental.CurrentOffset, + Messages = messages + }; + } + + internal static PolledMessagesRental ToRentedMessages(PolledMessages messages) + { + var rentedMessages = new List(messages.Messages.Count); + foreach (var message in messages.Messages) + { + rentedMessages.Add(new RentedMessageResponse + { + Header = message.Header, + Payload = message.Payload, + RawUserHeaders = message.RawUserHeaders ?? ReadOnlyMemory.Empty, + UserHeaders = message.UserHeaders + }); + } + + return new PolledMessagesRental(TcpMessageStream.EmptyMemoryOwner.Instance) + { + PartitionId = messages.PartitionId, + CurrentOffset = messages.CurrentOffset, + Messages = rentedMessages }; } @@ -480,36 +536,61 @@ internal static Dictionary MapHeaders(ReadOnlySpan while (position < payload.Length) { if (!TryMapHeaderKind(payload[position], out var keyKind)) + { return null; + } + position++; if (position + 4 > payload.Length) + { return null; + } + var keyLength = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]); if (keyLength is <= 0 or > 255) + { return null; + } position += 4; if (position + keyLength > payload.Length) + { return null; + } + var keyValue = payload[position..(position + keyLength)].ToArray(); position += keyLength; if (position >= payload.Length) + { return null; + } + if (!TryMapHeaderKind(payload[position], out var valueKind)) + { return null; + } + position++; if (position + 4 > payload.Length) + { return null; + } + var valueLength = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]); if (valueLength is <= 0 or > 255) + { return null; + } position += 4; if (position + valueLength > payload.Length) + { return null; + } + ReadOnlySpan value = payload[position..(position + valueLength)]; position += valueLength; @@ -550,6 +631,7 @@ private static bool TryMapHeaderKind(byte value, out HeaderKind kind) kind = MapHeaderKind(value); return true; } + kind = default; return false; } @@ -871,11 +953,11 @@ var partitionId } return (new ConsumerGroupMember - { - Id = id, - PartitionsCount = partitionsCount, - Partitions = partitions - }, + { + Id = id, + PartitionsCount = partitionsCount, + Partitions = partitions + }, 8 + partitionsCount * 4); } diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs new file mode 100644 index 0000000000..e9a8084a8a --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs @@ -0,0 +1,483 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Text; +using Apache.Iggy.Consumers; +using Apache.Iggy.Contracts; +using Apache.Iggy.Encryption; +using Apache.Iggy.Exceptions; +using Apache.Iggy.IggyClient; +using Apache.Iggy.IggyClient.Implementations; +using Apache.Iggy.Kinds; +using Apache.Iggy.Messages; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace Apache.Iggy.Tests.ConsumerTests; + +public class RentedConsumerTests +{ + [Fact] + public async Task ReceiveRentedAsync_Should_YieldMessages_WithExpectedPayloads() + { + var owner = new TrackingMemoryOwner(1024); + IReadOnlyList messages = BuildMessages(owner, 3); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 2, + Messages = messages + }; + Mock client = BuildClientMock(new Queue(new[] { rental })); + var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance); + await consumer.InitAsync(TestContext.Current.CancellationToken); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var got = new List(); + await using (IAsyncEnumerator e = consumer.ReceiveRentedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + for (var i = 0; i < 3; i++) + { + Assert.True(await e.MoveNextAsync()); + got.Add(e.Current); + } + } + + Assert.Equal(3, got.Count); + for (var i = 0; i < 3; i++) + { + Assert.Equal(MessageStatus.Success, got[i].Status); + Assert.Equal($"msg-{i}", Encoding.UTF8.GetString(got[i].Message.Payload.Span)); + Assert.Equal((ulong)i, got[i].CurrentOffset); + Assert.Equal(1u, got[i].PartitionId); + Assert.Null(got[i].Error); + } + + // Buffer still rented — none of the messages have been disposed. + Assert.Equal(0, owner.DisposeCount); + + foreach (var m in got) + { + m.Dispose(); + } + + // After disposing every message of the batch, rental returned to pool exactly once. + Assert.Equal(1, owner.DisposeCount); + await consumer.DisposeAsync(); + } + + [Fact] + public async Task ReceiveRentedAsync_Should_NotReturnBuffer_UntilAllMessagesDisposed() + { + var owner = new TrackingMemoryOwner(1024); + IReadOnlyList messages = BuildMessages(owner, 3); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 2, + Messages = messages + }; + Mock client = BuildClientMock(new Queue(new[] { rental })); + var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance); + await consumer.InitAsync(TestContext.Current.CancellationToken); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var got = new List(); + await using (IAsyncEnumerator e = consumer.ReceiveRentedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + for (var i = 0; i < 3; i++) + { + Assert.True(await e.MoveNextAsync()); + got.Add(e.Current); + } + } + + // Dispose only the first two: buffer must still be alive. + got[0].Dispose(); + Assert.Equal(0, owner.DisposeCount); + got[1].Dispose(); + Assert.Equal(0, owner.DisposeCount); + + // Final dispose returns the buffer. + got[2].Dispose(); + Assert.Equal(1, owner.DisposeCount); + + // Calling Dispose again is a no-op — refcount must not drop below zero. + got[0].Dispose(); + got[1].Dispose(); + got[2].Dispose(); + Assert.Equal(1, owner.DisposeCount); + + await consumer.DisposeAsync(); + } + + [Fact] + public async Task ReceiveRentedAsync_Should_YieldDecryptionFailed_AndStillReleaseBuffer_WhenEncryptorThrows() + { + var owner = new TrackingMemoryOwner(1024); + IReadOnlyList messages = BuildMessages(owner, 1); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = messages + }; + Mock client = BuildClientMock(new Queue(new[] { rental })); + var encryptor = new ThrowingEncryptor(); + var config = BuildConfig(); + config.MessageEncryptor = encryptor; + var consumer = new IggyConsumer(client.Object, config, NullLoggerFactory.Instance); + await consumer.InitAsync(TestContext.Current.CancellationToken); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + ReceivedRentedMessage? got = null; + await using (IAsyncEnumerator e = consumer.ReceiveRentedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + Assert.True(await e.MoveNextAsync()); + got = e.Current; + } + + Assert.NotNull(got); + Assert.Equal(MessageStatus.DecryptionFailed, got!.Status); + Assert.IsType(got.Error); + + Assert.Equal(0, owner.DisposeCount); + got.Dispose(); + Assert.Equal(1, owner.DisposeCount); + + await consumer.DisposeAsync(); + } + + [Fact] + public async Task ReceiveRentedAsync_Should_Throw_WhenConsumerNotInitialized() + { + Mock client = BuildClientMock(new Queue()); + var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance); + + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in consumer.ReceiveRentedAsync(TestContext.Current.CancellationToken)) + { + } + }); + + await consumer.DisposeAsync(); + } + + [Fact] + public void RentedBatchHandle_Release_BeyondAcquired_Throws() + { + var owner = new TrackingMemoryOwner(16); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = Array.Empty() + }; + var handle = new RentedBatchHandle(rental); + + handle.Release(); // releases self-ref -> 0 -> disposes rental + Assert.Equal(1, owner.DisposeCount); + + Assert.Throws(() => handle.Release()); + } + + [Fact] + public void RentedBatchHandle_AcquireRelease_Balanced_DisposesOnce() + { + var owner = new TrackingMemoryOwner(16); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = Array.Empty() + }; + var handle = new RentedBatchHandle(rental); // refCount=1 (self-ref) + + handle.Acquire(); // 2 + handle.Acquire(); // 3 + Assert.Equal(0, owner.DisposeCount); + + handle.Release(); // 2 + handle.Release(); // 1 (self-ref still alive) + Assert.Equal(0, owner.DisposeCount); + + handle.Release(); // 0 -> disposed + Assert.Equal(1, owner.DisposeCount); + } + + [Fact] + public void PolledMessagesRental_Dispose_Idempotent() + { + var owner = new TrackingMemoryOwner(16); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = Array.Empty() + }; + + rental.Dispose(); + rental.Dispose(); + rental.Dispose(); + + Assert.Equal(1, owner.DisposeCount); + } + + [Fact] + public async Task PolledMessagesRental_ConcurrentDispose_ReturnsBufferOnce() + { + var owner = new TrackingMemoryOwner(16); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = Array.Empty() + }; + + using var start = new ManualResetEventSlim(false); + Task[] tasks = Enumerable.Range(0, 64).Select(_ => Task.Run(() => + { + start.Wait(); + rental.Dispose(); + })).ToArray(); + + start.Set(); + await Task.WhenAll(tasks); + + Assert.Equal(1, owner.DisposeCount); + } + + [Fact] + public void PolledMessagesRental_ForgotDispose_FinalizerReturnsBuffer() + { + var owner = new TrackingMemoryOwner(16); + MakeAndDrop(owner); + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + Assert.Equal(1, owner.DisposeCount); + + [MethodImpl(MethodImplOptions.NoInlining)] + static void MakeAndDrop(IMemoryOwner o) + { + _ = new PolledMessagesRental(o) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = Array.Empty() + }; + } + } + + [Fact] + public async Task PollRented_MidLoopPublishFailure_DoesNotLeakBuffer() + { + var owner = new TrackingMemoryOwner(1024); + IReadOnlyList messages = BuildMessages(owner, 5); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 4, + Messages = messages + }; + Mock client = BuildClientMock(new Queue(new[] { rental })); + var consumer + = new FailingPublishConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance) { FailAfter = 2 }; + await consumer.InitAsync(TestContext.Current.CancellationToken); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var got = new List(); + IAsyncEnumerator enumerator = consumer.ReceiveRentedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken); + + for (var i = 0; i < 2; i++) + { + Assert.True(await enumerator.MoveNextAsync()); + got.Add(enumerator.Current); + } + + // First 2 publishes succeeded, 3rd injected failure aborted the loop. + // Producer self-ref was released in finally; consumer refs (2) still hold the buffer. + Assert.Equal(2, got.Count); + Assert.Equal(0, owner.DisposeCount); + + foreach (var m in got) + { + m.Dispose(); + } + + // All refs released -> rental disposed exactly once. + Assert.Equal(1, owner.DisposeCount); + + cts.Cancel(); + try + { + await enumerator.DisposeAsync(); + } + catch (OperationCanceledException) + { + } + + await consumer.DisposeAsync(); + } + + internal static IggyConsumerConfig BuildConfig() + { + return new IggyConsumerConfig + { + StreamId = Identifier.Numeric(1), + TopicId = Identifier.Numeric(1), + Consumer = Consumer.New(1), + PollingStrategy = PollingStrategy.Next(), + BatchSize = 10, + PartitionId = 1, + AutoCommitMode = AutoCommitMode.Disabled, + AutoCommit = false, + PollingIntervalMs = 0 + }; + } + + /// + /// Slices payload bytes into the supplied owner's memory and returns a list of + /// instances backed by that single rented buffer. + /// + internal static IReadOnlyList BuildMessages(TrackingMemoryOwner owner, int count) + { + var list = new List(count); + Memory buffer = owner.Memory; + var written = 0; + for (var i = 0; i < count; i++) + { + var bytes = Encoding.UTF8.GetBytes($"msg-{i}"); + bytes.CopyTo(buffer.Slice(written, bytes.Length)); + Memory slice = buffer.Slice(written, bytes.Length); + written += bytes.Length; + + list.Add(new RentedMessageResponse + { + Header = new MessageHeader + { + Offset = (ulong)i, + PayloadLength = bytes.Length + }, + Payload = slice, + RawUserHeaders = ReadOnlyMemory.Empty + }); + } + + return list; + } + + /// + /// Builds a that dequeues rentals on each + /// PollMessagesRentedAsync call. When the queue is empty, returns an + /// empty rental so the consumer can spin without dereferencing null. + /// + internal static Mock BuildClientMock(Queue rentals) + { + var mock = new Mock(MockBehavior.Loose); + mock.Setup(c => c.ConnectAsync(It.IsAny())).Returns(Task.CompletedTask); + mock.Setup(c => c.PollMessagesRentedAsync(It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny())) + .ReturnsAsync(() => + { + if (rentals.Count > 0) + { + return rentals.Dequeue(); + } + + return new PolledMessagesRental(TcpMessageStream.EmptyMemoryOwner.Instance) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = Array.Empty() + }; + }); + return mock; + } + + /// + /// wrapper that counts how many times + /// has been invoked, so tests can assert that + /// the rental returns to the pool exactly once. + /// + internal sealed class TrackingMemoryOwner : IMemoryOwner + { + private readonly byte[] _buffer; + + public int DisposeCount { get; private set; } + + public TrackingMemoryOwner(int size) + { + _buffer = new byte[size]; + } + + public Memory Memory => _buffer; + + public void Dispose() + { + DisposeCount++; + } + } + + private sealed class ThrowingEncryptor : IMessageEncryptor + { + public byte[] Encrypt(Span plainData) + { + throw new NotSupportedException(); + } + + public byte[] Decrypt(ReadOnlySpan encryptedData) + { + throw new InvalidOperationException("decrypt fail"); + } + } + + private sealed class FailingPublishConsumer : IggyConsumer + { + private int _calls; + + public int FailAfter { get; set; } + + public FailingPublishConsumer(IIggyClient client, IggyConsumerConfig config, + ILoggerFactory loggerFactory) + : base(client, config, loggerFactory) + { + } + + protected override async Task PublishRentedAsync(RentedBatchHandle rental, RentedMessageResponse message, + uint partitionId, MessageStatus status, Exception? error, CancellationToken ct) + { + if (Interlocked.Increment(ref _calls) > FailAfter) + { + throw new InvalidOperationException("inject publish failure"); + } + + await base.PublishRentedAsync(rental, message, partitionId, status, error, ct); + } + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedTypedConsumerTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedTypedConsumerTests.cs new file mode 100644 index 0000000000..26dfd89e50 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedTypedConsumerTests.cs @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text; +using Apache.Iggy.Consumers; +using Apache.Iggy.Contracts; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace Apache.Iggy.Tests.ConsumerTests; + +public class RentedTypedConsumerTests +{ + [Fact] + public async Task ReceiveRentedDeserializedAsync_Should_YieldDeserialized_AndReturnBuffer() + { + var owner = new RentedConsumerTests.TrackingMemoryOwner(1024); + IReadOnlyList messages = RentedConsumerTests.BuildMessages(owner, 3); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 2, + Messages = messages + }; + Mock client + = RentedConsumerTests.BuildClientMock(new Queue(new[] { rental })); + var deserializer = new StringDeserializer(); + var consumer + = new IggyConsumer(client.Object, BuildTypedConfig(deserializer), NullLoggerFactory.Instance); + await consumer.InitAsync(TestContext.Current.CancellationToken); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var got = new List>(); + await using (IAsyncEnumerator> e = consumer + .ReceiveRentedDeserializedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + for (var i = 0; i < 3; i++) + { + Assert.True(await e.MoveNextAsync()); + got.Add(e.Current); + } + } + + Assert.Equal(3, got.Count); + for (var i = 0; i < 3; i++) + { + Assert.Equal(MessageStatus.Success, got[i].Status); + Assert.Equal($"msg-{i}", got[i].Data); + Assert.Equal((ulong)i, got[i].CurrentOffset); + Assert.Equal(1u, got[i].PartitionId); + Assert.Null(got[i].Error); + } + + // Entire batch deserialized before first yield; rental already returned to pool. + Assert.Equal(1, owner.DisposeCount); + await consumer.DisposeAsync(); + } + + [Fact] + public async Task ReceiveRentedDeserializedAsync_Should_ReleaseEntireBatch_BeforeFirstYield() + { + var owner = new RentedConsumerTests.TrackingMemoryOwner(1024); + IReadOnlyList messages = RentedConsumerTests.BuildMessages(owner, 3); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 2, + Messages = messages + }; + Mock client + = RentedConsumerTests.BuildClientMock(new Queue(new[] { rental })); + var deserializer = new StringDeserializer(); + var consumer + = new IggyConsumer(client.Object, BuildTypedConfig(deserializer), NullLoggerFactory.Instance); + await consumer.InitAsync(TestContext.Current.CancellationToken); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await using IAsyncEnumerator> e = consumer.ReceiveRentedDeserializedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken); + + Assert.True(await e.MoveNextAsync()); + Assert.Equal(1, owner.DisposeCount); + Assert.Equal("msg-0", e.Current.Data); + + // Remaining messages come from the pre-deserialized list — no further disposal needed. + Assert.True(await e.MoveNextAsync()); + Assert.Equal(1, owner.DisposeCount); + Assert.Equal("msg-1", e.Current.Data); + + Assert.True(await e.MoveNextAsync()); + Assert.Equal(1, owner.DisposeCount); + Assert.Equal("msg-2", e.Current.Data); + + await consumer.DisposeAsync(); + } + + [Fact] + public async Task DeserializerThrows_Should_YieldDeserializationFailed_AndStillReturnBuffer() + { + var owner = new RentedConsumerTests.TrackingMemoryOwner(1024); + IReadOnlyList messages = RentedConsumerTests.BuildMessages(owner, 1); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = messages + }; + Mock client + = RentedConsumerTests.BuildClientMock(new Queue(new[] { rental })); + var deserializer = new StringDeserializer { ThrowOnNext = true }; + var consumer + = new IggyConsumer(client.Object, BuildTypedConfig(deserializer), NullLoggerFactory.Instance); + await consumer.InitAsync(TestContext.Current.CancellationToken); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + DeserializedMessage? got = null; + await using (IAsyncEnumerator> e = consumer + .ReceiveRentedDeserializedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + Assert.True(await e.MoveNextAsync()); + got = e.Current; + } + + Assert.NotNull(got); + Assert.Equal(MessageStatus.DeserializationFailed, got!.Status); + Assert.Null(got.Data); + Assert.IsType(got.Error); + + // Even on deserialization failure, the using-block inside the typed consumer releases the rental. + Assert.Equal(1, owner.DisposeCount); + await consumer.DisposeAsync(); + } + + private static IggyConsumerConfig BuildTypedConfig(IDeserializer deserializer) + { + return new IggyConsumerConfig + { + StreamId = Identifier.Numeric(1), + TopicId = Identifier.Numeric(1), + Consumer = Consumer.New(1), + PollingStrategy = PollingStrategy.Next(), + BatchSize = 10, + PartitionId = 1, + AutoCommitMode = AutoCommitMode.Disabled, + AutoCommit = false, + PollingIntervalMs = 0, + Deserializer = deserializer + }; + } + + private sealed class StringDeserializer : IDeserializer + { + public bool ThrowOnNext { get; set; } + + public string Deserialize(ReadOnlyMemory data) + { + if (ThrowOnNext) + { + throw new InvalidOperationException("deserialize fail"); + } + + return Encoding.UTF8.GetString(data.Span); + } + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs index 234f5239b4..580d905f8f 100644 --- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs @@ -19,6 +19,7 @@ using Apache.Iggy.Contracts.Auth; using Apache.Iggy.Enums; using Apache.Iggy.Extensions; +using Apache.Iggy.IggyClient.Implementations; using Apache.Iggy.Tests.Utils; using Apache.Iggy.Tests.Utils.Groups; using Apache.Iggy.Tests.Utils.Messages; @@ -84,7 +85,8 @@ public void MapMessages_NoHeaders_ReturnsValidMessageResponses() msgTwoPayload.CopyTo(combinedPayload.AsSpan(16 + msgOnePayload.Length)); // Act - var responses = Mappers.BinaryMapper.MapMessages(combinedPayload); + var responses + = Mappers.BinaryMapper.MapRentedMessages(combinedPayload, TcpMessageStream.EmptyMemoryOwner.Instance); // Assert Assert.NotNull(responses); diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/MessageResponseConverterTests.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/MessageResponseConverterTests.cs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/MessageResponseConverterTests.cs @@ -0,0 +1 @@ + From e2c9c09c3c4bb1813c06bafdb8c8b56622ddc026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Zborek?= Date: Tue, 12 May 2026 23:10:41 +0200 Subject: [PATCH 2/4] refactor(csharp): format object initializers for improved readability in BinaryMapper --- foreign/csharp/Directory.Packages.props | 53 +- .../FetchMessagesTests.cs | 428 +-- .../IggyTypedConsumerTests.cs | 544 ++-- .../Iggy_SDK.Tests.Integration.csproj | 89 +- foreign/csharp/Iggy_SDK.sln | 2 +- .../Iggy_SDK/Consumers/DeserializedMessage.cs | 65 - .../Iggy_SDK/Consumers/IDeserializer.cs | 100 +- .../Iggy_SDK/Consumers/IggyConsumer.Rented.cs | 500 ++-- .../csharp/Iggy_SDK/Consumers/IggyConsumer.cs | 1096 +++---- .../Iggy_SDK/Consumers/IggyConsumerOfT.cs | 335 +-- .../Iggy_SDK/Consumers/ReceivedMessage.cs | 42 +- .../Consumers/ReceivedRentedMessage.cs | 252 +- .../Iggy_SDK/Contracts/MessageResponse.cs | 192 +- .../Contracts/PolledMessagesRental.cs | 133 +- .../Contracts/RentedMessageResponse.cs | 142 +- .../Encryption/AesMessageEncryptor.cs | 257 +- .../Iggy_SDK/Encryption/IMessageEncryptor.cs | 80 +- .../Extensions/IggyClientExtension.cs | 148 +- .../Iggy_SDK/IggyClient/IIggyConsumer.cs | 170 +- .../Implementations/HttpMessageStream.cs | 1724 +++++------ .../Implementations/TcpMessageStream.cs | 2655 +++++++++-------- .../MessageResponseConverter.cs | 212 +- .../csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 67 +- .../ConsumerTests/RentedConsumerTests.cs | 966 +++--- .../ConsumerTests/RentedTypedConsumerTests.cs | 366 +-- .../MessageResponseConverterTests.cs | 1 - 26 files changed, 5269 insertions(+), 5350 deletions(-) delete mode 100644 foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/MapperTests/MessageResponseConverterTests.cs diff --git a/foreign/csharp/Directory.Packages.props b/foreign/csharp/Directory.Packages.props index cd888e5b8f..b2dee5c2a0 100644 --- a/foreign/csharp/Directory.Packages.props +++ b/foreign/csharp/Directory.Packages.props @@ -1,26 +1,27 @@ - - - true - false - - - - - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - + + + true + false + + + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs index 91768ada45..153e7cea6a 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs @@ -1,214 +1,214 @@ -// // Licensed to the Apache Software Foundation (ASF) under one -// // or more contributor license agreements. See the NOTICE file -// // distributed with this work for additional information -// // regarding copyright ownership. The ASF licenses this file -// // to you under the Apache License, Version 2.0 (the -// // "License"); you may not use this file except in compliance -// // with the License. You may obtain a copy of the License at -// // -// // http://www.apache.org/licenses/LICENSE-2.0 -// // -// // Unless required by applicable law or agreed to in writing, -// // software distributed under the License is distributed on an -// // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// // KIND, either express or implied. See the License for the -// // specific language governing permissions and limitations -// // under the License. - -using System.Text; -using Apache.Iggy.Contracts; -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Headers; -using Apache.Iggy.IggyClient; -using Apache.Iggy.Kinds; -using Apache.Iggy.Messages; -using Apache.Iggy.Tests.Integrations.Fixtures; -using Shouldly; -using Partitioning = Apache.Iggy.Kinds.Partitioning; - -namespace Apache.Iggy.Tests.Integrations; - -public class FetchMessagesTests -{ - private const int MessageCount = 20; - private const string TopicName = "topic"; - private const string HeadersTopicName = "headers-topic"; - - [ClassDataSource(Shared = SharedType.PerAssembly)] - public required IggyServerFixture Fixture { get; init; } - - private async Task<(IIggyClient client, string streamName)> CreateStreamWithMessages(Protocol protocol) - { - var client = await Fixture.CreateAuthenticatedClient(protocol); - - var streamName = $"fetch-msg-{Guid.NewGuid():N}"; - - await client.CreateStreamAsync(streamName); - await client.CreateTopicAsync(Identifier.String(streamName), TopicName, 1); - await client.CreateTopicAsync(Identifier.String(streamName), HeadersTopicName, 1); - - await client.SendMessagesAsync(Identifier.String(streamName), - Identifier.String(TopicName), Partitioning.None(), - CreateMessagesWithoutHeader(MessageCount)); - - await client.SendMessagesAsync(Identifier.String(streamName), - Identifier.String(HeadersTopicName), Partitioning.None(), - CreateMessagesWithHeader(MessageCount)); - - return (client, streamName); - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task PollMessages_WithNoHeaders_Should_PollMessages_Successfully(Protocol protocol) - { - var (client, streamName) = await CreateStreamWithMessages(protocol); - - var response = await client.PollMessagesAsync(new MessageFetchRequest - { - Count = 10, - AutoCommit = true, - Consumer = Consumer.New(1), - PartitionId = 0, - PollingStrategy = PollingStrategy.Next(), - StreamId = Identifier.String(streamName), - TopicId = Identifier.String(TopicName) - }); - - response.Messages.Count.ShouldBe(10); - response.PartitionId.ShouldBe(0); - response.CurrentOffset.ShouldBe(19u); - - foreach (var responseMessage in response.Messages) - { - responseMessage.UserHeaders.ShouldBeNull(); - responseMessage.Payload.ShouldNotBeNull(); - responseMessage.Payload.Length.ShouldBeGreaterThan(0); - } - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task PollMessages_InvalidTopic_Should_Throw_InvalidResponse(Protocol protocol) - { - var (client, streamName) = await CreateStreamWithMessages(protocol); - - var invalidFetchRequest = new MessageFetchRequest - { - Count = 10, - AutoCommit = true, - Consumer = Consumer.New(1), - PartitionId = 0, - PollingStrategy = PollingStrategy.Next(), - StreamId = Identifier.String(streamName), - TopicId = Identifier.Numeric(2137) - }; - - await Should.ThrowAsync(() => - client.PollMessagesAsync(invalidFetchRequest)); - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task PollMessages_WithHeaders_Should_PollMessages_Successfully(Protocol protocol) - { - var (client, streamName) = await CreateStreamWithMessages(protocol); - - var headersMessageFetchRequest = new MessageFetchRequest - { - Count = 10, - AutoCommit = true, - Consumer = Consumer.New(1), - PartitionId = 0, - PollingStrategy = PollingStrategy.Next(), - StreamId = Identifier.String(streamName), - TopicId = Identifier.String(HeadersTopicName) - }; - - var response = await client.PollMessagesAsync(headersMessageFetchRequest); - response.Messages.Count.ShouldBe(10); - response.PartitionId.ShouldBe(0); - response.CurrentOffset.ShouldBe(19u); - foreach (var responseMessage in response.Messages) - { - responseMessage.UserHeaders.ShouldNotBeNull(); - responseMessage.UserHeaders.Count.ShouldBe(2); - responseMessage.UserHeaders[HeaderKey.FromString("header1")].ToString().ShouldBe("value1"); - responseMessage.UserHeaders[HeaderKey.FromString("header2")].ToInt32().ShouldBeGreaterThan(0); - } - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task PollMessages_WithHeaders_Should_AutoParseUserHeadersFromRaw(Protocol protocol) - { - var (client, streamName) = await CreateStreamWithMessages(protocol); - - var response = await client.PollMessagesAsync(new MessageFetchRequest - { - Count = 1, - AutoCommit = true, - Consumer = Consumer.New(2), - PartitionId = 0, - PollingStrategy = PollingStrategy.First(), - StreamId = Identifier.String(streamName), - TopicId = Identifier.String(HeadersTopicName) - }); - - response.Messages.Count.ShouldBe(1); - var msg = response.Messages[0]; - - Dictionary? parsed = msg.UserHeaders; - parsed.ShouldNotBeNull(); - parsed!.Count.ShouldBe(2); - parsed[HeaderKey.FromString("header1")].ToString().ShouldBe("value1"); - parsed[HeaderKey.FromString("header2")].ToInt32().ShouldBe(14); - - Dictionary? second = msg.UserHeaders; - second.ShouldBeSameAs(parsed); - } - - private static Message[] CreateMessagesWithoutHeader(int count) - { - var messages = new List(); - for (var i = 0; i < count; i++) - { - var dummyJson = $$""" - { - "userId": {{i + 1}}, - "id": {{i + 1}}, - "title": "delete", - "completed": false - } - """; - messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson))); - } - - return messages.ToArray(); - } - - private static Message[] CreateMessagesWithHeader(int count) - { - var messages = new List(); - for (var i = 0; i < count; i++) - { - var dummyJson = $$""" - { - "userId": {{i + 1}}, - "id": {{i + 1}}, - "title": "delete", - "completed": false - } - """; - messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson), - new Dictionary - { - { HeaderKey.FromString("header1"), HeaderValue.FromString("value1") }, - { HeaderKey.FromString("header2"), HeaderValue.FromInt32(14 + i) } - })); - } - - return messages.ToArray(); - } -} +// // Licensed to the Apache Software Foundation (ASF) under one +// // or more contributor license agreements. See the NOTICE file +// // distributed with this work for additional information +// // regarding copyright ownership. The ASF licenses this file +// // to you under the Apache License, Version 2.0 (the +// // "License"); you may not use this file except in compliance +// // with the License. You may obtain a copy of the License at +// // +// // http://www.apache.org/licenses/LICENSE-2.0 +// // +// // Unless required by applicable law or agreed to in writing, +// // software distributed under the License is distributed on an +// // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// // KIND, either express or implied. See the License for the +// // specific language governing permissions and limitations +// // under the License. + +using System.Text; +using Apache.Iggy.Contracts; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; +using Apache.Iggy.Messages; +using Apache.Iggy.Tests.Integrations.Fixtures; +using Shouldly; +using Partitioning = Apache.Iggy.Kinds.Partitioning; + +namespace Apache.Iggy.Tests.Integrations; + +public class FetchMessagesTests +{ + private const int MessageCount = 20; + private const string TopicName = "topic"; + private const string HeadersTopicName = "headers-topic"; + + [ClassDataSource(Shared = SharedType.PerAssembly)] + public required IggyServerFixture Fixture { get; init; } + + private async Task<(IIggyClient client, string streamName)> CreateStreamWithMessages(Protocol protocol) + { + var client = await Fixture.CreateAuthenticatedClient(protocol); + + var streamName = $"fetch-msg-{Guid.NewGuid():N}"; + + await client.CreateStreamAsync(streamName); + await client.CreateTopicAsync(Identifier.String(streamName), TopicName, 1); + await client.CreateTopicAsync(Identifier.String(streamName), HeadersTopicName, 1); + + await client.SendMessagesAsync(Identifier.String(streamName), + Identifier.String(TopicName), Partitioning.None(), + CreateMessagesWithoutHeader(MessageCount)); + + await client.SendMessagesAsync(Identifier.String(streamName), + Identifier.String(HeadersTopicName), Partitioning.None(), + CreateMessagesWithHeader(MessageCount)); + + return (client, streamName); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task PollMessages_WithNoHeaders_Should_PollMessages_Successfully(Protocol protocol) + { + var (client, streamName) = await CreateStreamWithMessages(protocol); + + var response = await client.PollMessagesAsync(new MessageFetchRequest + { + Count = 10, + AutoCommit = true, + Consumer = Consumer.New(1), + PartitionId = 0, + PollingStrategy = PollingStrategy.Next(), + StreamId = Identifier.String(streamName), + TopicId = Identifier.String(TopicName) + }); + + response.Messages.Count.ShouldBe(10); + response.PartitionId.ShouldBe(0); + response.CurrentOffset.ShouldBe(19u); + + foreach (var responseMessage in response.Messages) + { + responseMessage.UserHeaders.ShouldBeNull(); + responseMessage.Payload.ShouldNotBeNull(); + responseMessage.Payload.Length.ShouldBeGreaterThan(0); + } + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task PollMessages_InvalidTopic_Should_Throw_InvalidResponse(Protocol protocol) + { + var (client, streamName) = await CreateStreamWithMessages(protocol); + + var invalidFetchRequest = new MessageFetchRequest + { + Count = 10, + AutoCommit = true, + Consumer = Consumer.New(1), + PartitionId = 0, + PollingStrategy = PollingStrategy.Next(), + StreamId = Identifier.String(streamName), + TopicId = Identifier.Numeric(2137) + }; + + await Should.ThrowAsync(() => + client.PollMessagesAsync(invalidFetchRequest)); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task PollMessages_WithHeaders_Should_PollMessages_Successfully(Protocol protocol) + { + var (client, streamName) = await CreateStreamWithMessages(protocol); + + var headersMessageFetchRequest = new MessageFetchRequest + { + Count = 10, + AutoCommit = true, + Consumer = Consumer.New(1), + PartitionId = 0, + PollingStrategy = PollingStrategy.Next(), + StreamId = Identifier.String(streamName), + TopicId = Identifier.String(HeadersTopicName) + }; + + var response = await client.PollMessagesAsync(headersMessageFetchRequest); + response.Messages.Count.ShouldBe(10); + response.PartitionId.ShouldBe(0); + response.CurrentOffset.ShouldBe(19u); + foreach (var responseMessage in response.Messages) + { + responseMessage.UserHeaders.ShouldNotBeNull(); + responseMessage.UserHeaders.Count.ShouldBe(2); + responseMessage.UserHeaders[HeaderKey.FromString("header1")].ToString().ShouldBe("value1"); + responseMessage.UserHeaders[HeaderKey.FromString("header2")].ToInt32().ShouldBeGreaterThan(0); + } + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task PollMessages_WithHeaders_Should_AutoParseUserHeadersFromRaw(Protocol protocol) + { + var (client, streamName) = await CreateStreamWithMessages(protocol); + + var response = await client.PollMessagesAsync(new MessageFetchRequest + { + Count = 1, + AutoCommit = true, + Consumer = Consumer.New(2), + PartitionId = 0, + PollingStrategy = PollingStrategy.First(), + StreamId = Identifier.String(streamName), + TopicId = Identifier.String(HeadersTopicName) + }); + + response.Messages.Count.ShouldBe(1); + var msg = response.Messages[0]; + + Dictionary? parsed = msg.UserHeaders; + parsed.ShouldNotBeNull(); + parsed!.Count.ShouldBe(2); + parsed[HeaderKey.FromString("header1")].ToString().ShouldBe("value1"); + parsed[HeaderKey.FromString("header2")].ToInt32().ShouldBe(14); + + Dictionary? second = msg.UserHeaders; + second.ShouldBeSameAs(parsed); + } + + private static Message[] CreateMessagesWithoutHeader(int count) + { + var messages = new List(); + for (var i = 0; i < count; i++) + { + var dummyJson = $$""" + { + "userId": {{i + 1}}, + "id": {{i + 1}}, + "title": "delete", + "completed": false + } + """; + messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson))); + } + + return messages.ToArray(); + } + + private static Message[] CreateMessagesWithHeader(int count) + { + var messages = new List(); + for (var i = 0; i < count; i++) + { + var dummyJson = $$""" + { + "userId": {{i + 1}}, + "id": {{i + 1}}, + "title": "delete", + "completed": false + } + """; + messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson), + new Dictionary + { + { HeaderKey.FromString("header1"), HeaderValue.FromString("value1") }, + { HeaderKey.FromString("header2"), HeaderValue.FromInt32(14 + i) } + })); + } + + return messages.ToArray(); + } +} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs index 8dd3160ffd..3861ae812f 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs @@ -1,272 +1,272 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Text; -using Apache.Iggy.Consumers; -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.IggyClient; -using Apache.Iggy.Kinds; -using Apache.Iggy.Messages; -using Apache.Iggy.Tests.Integrations.Fixtures; -using Microsoft.Extensions.Logging.Abstractions; -using Shouldly; -using Partitioning = Apache.Iggy.Kinds.Partitioning; - -namespace Apache.Iggy.Tests.Integrations; - -public class IggyTypedConsumerTests -{ - [ClassDataSource(Shared = SharedType.PerAssembly)] - public required IggyServerFixture Fixture { get; init; } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task ReceiveRentedDeserializedAsync_Should_YieldMessages_WithCorrectData(Protocol protocol) - { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); - - var testStream = await CreateTestStreamWithMessages(client, protocol); - - var consumerId = protocol == Protocol.Tcp ? 200 : 300; - IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), - AutoCommitMode.Disabled, new Utf8StringDeserializer()); - - await consumer.InitAsync(); - - var received = new List>(); - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); - - await foreach (DeserializedMessage msg in consumer.ReceiveRentedDeserializedAsync(cts.Token)) - { - msg.ShouldNotBeNull(); - msg.Status.ShouldBe(MessageStatus.Success); - msg.Data.ShouldNotBeNull(); - msg.Data.ShouldStartWith("Test message"); - msg.PartitionId.ShouldBe(1u); - msg.Error.ShouldBeNull(); - received.Add(msg); - if (received.Count >= 10) - { - break; - } - } - - received.Count.ShouldBeGreaterThanOrEqualTo(10); - await consumer.DisposeAsync(); - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task ReceiveRentedDeserializedAsync_WithoutInit_Should_Throw_ConsumerNotInitializedException( - Protocol protocol) - { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); - - var testStream = await CreateTestStreamWithMessages(client, protocol); - - var consumerId = protocol == Protocol.Tcp ? 201 : 301; - IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), - AutoCommitMode.Disabled, new Utf8StringDeserializer()); - - await Should.ThrowAsync(async () => - { - await foreach (DeserializedMessage _ in consumer.ReceiveRentedDeserializedAsync()) - { - } - }); - - await consumer.DisposeAsync(); - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task ReceiveRentedDeserializedAsync_WithAutoCommitAfterReceive_Should_StoreOffset(Protocol protocol) - { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); - - var testStream = await CreateTestStreamWithMessages(client, protocol); - - var consumerId = protocol == Protocol.Tcp ? 202 : 302; - IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), - AutoCommitMode.AfterReceive, new Utf8StringDeserializer(), - PollingStrategy.First()); - - await consumer.InitAsync(); - - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); - var count = 0; - - await foreach (DeserializedMessage _ in consumer.ReceiveRentedDeserializedAsync(cts.Token)) - { - count++; - if (count >= 5) - { - break; - } - } - - await consumer.DisposeAsync(); - - var offset = await client.GetOffsetAsync(Consumer.New(consumerId), - Identifier.String(testStream.StreamId), - Identifier.String(testStream.TopicId), - 1u); - - offset.ShouldNotBeNull(); - offset.StoredOffset.ShouldBe(3ul); - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task ReceiveRentedDeserializedAsync_WithFailingDeserializer_Should_YieldDeserializationFailed( - Protocol protocol) - { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); - - var testStream = await CreateTestStreamWithMessages(client, protocol); - - var consumerId = protocol == Protocol.Tcp ? 203 : 303; - IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), - AutoCommitMode.Disabled, new FailingDeserializer()); - - await consumer.InitAsync(); - - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); - - await foreach (DeserializedMessage msg in consumer.ReceiveRentedDeserializedAsync(cts.Token)) - { - msg.Status.ShouldBe(MessageStatus.DeserializationFailed); - msg.Data.ShouldBeNull(); - msg.Error.ShouldNotBeNull(); - msg.Error.ShouldBeOfType(); - break; - } - - await consumer.DisposeAsync(); - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task ReceiveRentedDeserializedAsync_Should_StopCleanly_OnCancellation(Protocol protocol) - { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); - - var testStream = await CreateTestStreamWithMessages(client, protocol); - - var consumerId = protocol == Protocol.Tcp ? 204 : 304; - IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), - AutoCommitMode.Disabled, new Utf8StringDeserializer()); - - await consumer.InitAsync(); - - using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); - - await Should.NotThrowAsync(async () => - { - try - { - await foreach (DeserializedMessage _ in consumer.ReceiveRentedDeserializedAsync(cts.Token)) - { - } - } - catch (OperationCanceledException) - { - } - }); - - await consumer.DisposeAsync(); - } - - private static IggyConsumer BuildTypedConsumer(IIggyClient client, - TestStreamInfo stream, - Consumer consumer, - AutoCommitMode autoCommitMode, - IDeserializer deserializer, - PollingStrategy? pollingStrategy = null) - { - var config = new IggyConsumerConfig - { - StreamId = Identifier.String(stream.StreamId), - TopicId = Identifier.String(stream.TopicId), - Consumer = consumer, - PollingStrategy = pollingStrategy ?? PollingStrategy.Next(), - BatchSize = 10, - PartitionId = 1, - AutoCommitMode = autoCommitMode, - AutoCommit = autoCommitMode != AutoCommitMode.Disabled, - PollingIntervalMs = 0, - Deserializer = deserializer - }; - return new IggyConsumer(client, config, NullLoggerFactory.Instance); - } - - private async Task CreateTestStreamWithMessages(IIggyClient client, Protocol protocol, - uint partitionsCount = 5, int messagesPerPartition = 100) - { - var streamId = $"typed_stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}"; - var topicId = "test_topic"; - - await client.CreateStreamAsync(streamId); - await client.CreateTopicAsync(Identifier.String(streamId), topicId, partitionsCount); - - for (uint partitionId = 0; partitionId < partitionsCount; partitionId++) - { - var messages = new List(); - for (var i = 0; i < messagesPerPartition; i++) - { - messages.Add(new Message(Guid.NewGuid(), - Encoding.UTF8.GetBytes($"Test message {i} for partition {partitionId}"))); - } - - await client.SendMessagesAsync(Identifier.String(streamId), - Identifier.String(topicId), - Partitioning.PartitionId((int)partitionId), - messages); - } - - return new TestStreamInfo(streamId, topicId, partitionsCount, messagesPerPartition); - } - - private record TestStreamInfo(string StreamId, string TopicId, uint PartitionsCount, int MessagesPerPartition); - - private sealed class Utf8StringDeserializer : IDeserializer - { - public string Deserialize(ReadOnlyMemory data) - { - return Encoding.UTF8.GetString(data.Span); - } - } - - private sealed class FailingDeserializer : IDeserializer - { - public string Deserialize(ReadOnlyMemory data) - { - throw new InvalidOperationException("Intentional deserialization failure"); - } - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text; +using Apache.Iggy.Consumers; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; +using Apache.Iggy.Messages; +using Apache.Iggy.Tests.Integrations.Fixtures; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Partitioning = Apache.Iggy.Kinds.Partitioning; + +namespace Apache.Iggy.Tests.Integrations; + +public class IggyTypedConsumerTests +{ + [ClassDataSource(Shared = SharedType.PerAssembly)] + public required IggyServerFixture Fixture { get; init; } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task ReceiveDeserializedAsync_Should_YieldMessages_WithCorrectData(Protocol protocol) + { + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); + + var testStream = await CreateTestStreamWithMessages(client, protocol); + + var consumerId = protocol == Protocol.Tcp ? 200 : 300; + IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), + AutoCommitMode.Disabled, new Utf8StringDeserializer()); + + await consumer.InitAsync(); + + var received = new List>(); + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + await foreach (ReceivedMessage msg in consumer.ReceiveDeserializedAsync(cts.Token)) + { + msg.ShouldNotBeNull(); + msg.Status.ShouldBe(MessageStatus.Success); + msg.Data.ShouldNotBeNull(); + msg.Data.ShouldStartWith("Test message"); + msg.PartitionId.ShouldBe(1u); + msg.Error.ShouldBeNull(); + received.Add(msg); + if (received.Count >= 10) + { + break; + } + } + + received.Count.ShouldBeGreaterThanOrEqualTo(10); + await consumer.DisposeAsync(); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task ReceiveDeserializedAsync_WithoutInit_Should_Throw_ConsumerNotInitializedException( + Protocol protocol) + { + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); + + var testStream = await CreateTestStreamWithMessages(client, protocol); + + var consumerId = protocol == Protocol.Tcp ? 201 : 301; + IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), + AutoCommitMode.Disabled, new Utf8StringDeserializer()); + + await Should.ThrowAsync(async () => + { + await foreach (ReceivedMessage _ in consumer.ReceiveDeserializedAsync()) + { + } + }); + + await consumer.DisposeAsync(); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task ReceiveDeserializedAsync_WithAutoCommitAfterReceive_Should_StoreOffset(Protocol protocol) + { + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); + + var testStream = await CreateTestStreamWithMessages(client, protocol); + + var consumerId = protocol == Protocol.Tcp ? 202 : 302; + IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), + AutoCommitMode.AfterReceive, new Utf8StringDeserializer(), + PollingStrategy.First()); + + await consumer.InitAsync(); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var count = 0; + + await foreach (ReceivedMessage _ in consumer.ReceiveDeserializedAsync(cts.Token)) + { + count++; + if (count >= 5) + { + break; + } + } + + await consumer.DisposeAsync(); + + var offset = await client.GetOffsetAsync(Consumer.New(consumerId), + Identifier.String(testStream.StreamId), + Identifier.String(testStream.TopicId), + 1u); + + offset.ShouldNotBeNull(); + offset.StoredOffset.ShouldBe(3ul); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task ReceiveDeserializedAsync_WithFailingDeserializer_Should_YieldDeserializationFailed( + Protocol protocol) + { + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); + + var testStream = await CreateTestStreamWithMessages(client, protocol); + + var consumerId = protocol == Protocol.Tcp ? 203 : 303; + IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), + AutoCommitMode.Disabled, new FailingDeserializer()); + + await consumer.InitAsync(); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + await foreach (ReceivedMessage msg in consumer.ReceiveDeserializedAsync(cts.Token)) + { + msg.Status.ShouldBe(MessageStatus.DeserializationFailed); + msg.Data.ShouldBeNull(); + msg.Error.ShouldNotBeNull(); + msg.Error.ShouldBeOfType(); + break; + } + + await consumer.DisposeAsync(); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task ReceiveDeserializedAsync_Should_StopCleanly_OnCancellation(Protocol protocol) + { + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); + + var testStream = await CreateTestStreamWithMessages(client, protocol); + + var consumerId = protocol == Protocol.Tcp ? 204 : 304; + IggyConsumer consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId), + AutoCommitMode.Disabled, new Utf8StringDeserializer()); + + await consumer.InitAsync(); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); + + await Should.NotThrowAsync(async () => + { + try + { + await foreach (ReceivedMessage _ in consumer.ReceiveDeserializedAsync(cts.Token)) + { + } + } + catch (OperationCanceledException) + { + } + }); + + await consumer.DisposeAsync(); + } + + private static IggyConsumer BuildTypedConsumer(IIggyClient client, + TestStreamInfo stream, + Consumer consumer, + AutoCommitMode autoCommitMode, + IDeserializer deserializer, + PollingStrategy? pollingStrategy = null) + { + var config = new IggyConsumerConfig + { + StreamId = Identifier.String(stream.StreamId), + TopicId = Identifier.String(stream.TopicId), + Consumer = consumer, + PollingStrategy = pollingStrategy ?? PollingStrategy.Next(), + BatchSize = 10, + PartitionId = 1, + AutoCommitMode = autoCommitMode, + AutoCommit = autoCommitMode != AutoCommitMode.Disabled, + PollingIntervalMs = 0, + Deserializer = deserializer + }; + return new IggyConsumer(client, config, NullLoggerFactory.Instance); + } + + private async Task CreateTestStreamWithMessages(IIggyClient client, Protocol protocol, + uint partitionsCount = 5, int messagesPerPartition = 100) + { + var streamId = $"typed_stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}"; + var topicId = "test_topic"; + + await client.CreateStreamAsync(streamId); + await client.CreateTopicAsync(Identifier.String(streamId), topicId, partitionsCount); + + for (uint partitionId = 0; partitionId < partitionsCount; partitionId++) + { + var messages = new List(); + for (var i = 0; i < messagesPerPartition; i++) + { + messages.Add(new Message(Guid.NewGuid(), + Encoding.UTF8.GetBytes($"Test message {i} for partition {partitionId}"))); + } + + await client.SendMessagesAsync(Identifier.String(streamId), + Identifier.String(topicId), + Partitioning.PartitionId((int)partitionId), + messages); + } + + return new TestStreamInfo(streamId, topicId, partitionsCount, messagesPerPartition); + } + + private record TestStreamInfo(string StreamId, string TopicId, uint PartitionsCount, int MessagesPerPartition); + + private sealed class Utf8StringDeserializer : IDeserializer + { + public string Deserialize(ReadOnlyMemory data) + { + return Encoding.UTF8.GetString(data.Span); + } + } + + private sealed class FailingDeserializer : IDeserializer + { + public string Deserialize(ReadOnlyMemory data) + { + throw new InvalidOperationException("Intentional deserialization failure"); + } + } +} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj b/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj index a0bc77def7..1b229b82c6 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj @@ -1,44 +1,45 @@ - - - - enable - enable - Exe - net8.0;net10.0 - Apache.Iggy.Tests.Integrations - Apache.Iggy.Tests.Integrations - true - - - - - - - - - - - - - - - - - Certs\iggy.pfx - Always - - - Certs\iggy_ca_cert.pem - Always - - - Certs\iggy_cert.pem - Always - - - Certs\iggy_key.pem - Always - - - - + + + + enable + enable + Exe + net8.0;net10.0 + Apache.Iggy.Tests.Integrations + Apache.Iggy.Tests.Integrations + true + + + + + + + + + + + + + + + + + + Certs\iggy.pfx + Always + + + Certs\iggy_ca_cert.pem + Always + + + Certs\iggy_cert.pem + Always + + + Certs\iggy_key.pem + Always + + + + diff --git a/foreign/csharp/Iggy_SDK.sln b/foreign/csharp/Iggy_SDK.sln index 5732f69782..4e1d81c01a 100644 --- a/foreign/csharp/Iggy_SDK.sln +++ b/foreign/csharp/Iggy_SDK.sln @@ -1,4 +1,4 @@ - + Microsoft Visual Studio Solution File, Format Version 12.00 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Iggy_SDK", "Iggy_SDK\Iggy_SDK.csproj", "{661540EB-81F9-492C-828B-CF787BEBD50B}" EndProject diff --git a/foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs b/foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs deleted file mode 100644 index b7e8cdbb83..0000000000 --- a/foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Headers; -using Apache.Iggy.Messages; - -namespace Apache.Iggy.Consumers; - -/// -/// Represents a message whose payload was deserialized directly from rented memory. The rented buffer has -/// already been returned to the pool by the time this message is yielded, so only the header, user headers, -/// and the deserialized are available; the raw payload bytes are not retained. -/// -/// The deserialized payload type. -public sealed class DeserializedMessage -{ - /// - /// Message header. - /// - public required MessageHeader Header { get; init; } - - /// - /// The deserialized payload. Null if is not . - /// - public T? Data { get; init; } - - /// - /// Parsed user headers, if present. - /// - public Dictionary? UserHeaders { get; init; } - - /// - /// The current offset of this message in the partition. - /// - public required ulong CurrentOffset { get; init; } - - /// - /// The partition ID from which this message was consumed. - /// - public uint PartitionId { get; init; } - - /// - /// The status of the message (Success, DecryptionFailed, DeserializationFailed). - /// - public MessageStatus Status { get; init; } = MessageStatus.Success; - - /// - /// The exception that occurred during processing, if any. - /// - public Exception? Error { get; init; } -} diff --git a/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs b/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs index e22372144d..041b75cb3c 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs @@ -1,50 +1,50 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -namespace Apache.Iggy.Consumers; - -/// -/// Interface for deserializing message payloads from byte arrays to type T. -/// -/// No type constraints are enforced on T to provide maximum flexibility. -/// Implementations are responsible for ensuring that the provided byte data can be properly deserialized to the -/// target type. -/// -/// -/// -/// The target type for deserialization. Can be any type - reference or value type, nullable or non-nullable. -/// The deserializer implementation must be able to produce instances of the specific type. -/// -/// -/// Implementations should throw appropriate exceptions (e.g., , -/// , or ) -/// if the provided data cannot be deserialized to type T. These exceptions will be caught and logged by -/// during message processing. -/// -public interface IDeserializer -{ - /// - /// Deserializes a read-only memory into an instance of type T. Callers may pass a byte[] directly - /// thanks to the implicit conversion to . - /// - /// - /// Read-only memory containing the serialized data. The implementation MUST NOT retain a reference to - /// the span after returning. - /// - /// An instance of type T representing the deserialized data. - T Deserialize(ReadOnlyMemory data); -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.Iggy.Consumers; + +/// +/// Interface for deserializing message payloads from byte arrays to type T. +/// +/// No type constraints are enforced on T to provide maximum flexibility. +/// Implementations are responsible for ensuring that the provided byte data can be properly deserialized to the +/// target type. +/// +/// +/// +/// The target type for deserialization. Can be any type - reference or value type, nullable or non-nullable. +/// The deserializer implementation must be able to produce instances of the specific type. +/// +/// +/// Implementations should throw appropriate exceptions (e.g., , +/// , or ) +/// if the provided data cannot be deserialized to type T. These exceptions will be caught and logged by +/// during message processing. +/// +public interface IDeserializer +{ + /// + /// Deserializes a read-only memory into an instance of type T. Callers may pass a byte[] directly + /// thanks to the implicit conversion to . + /// + /// + /// Read-only memory containing the serialized data. The implementation MUST NOT retain a reference to + /// the span after returning. + /// + /// An instance of type T representing the deserialized data. + T Deserialize(ReadOnlyMemory data); +} diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs index a18f4c784e..1644d2f225 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs @@ -1,250 +1,250 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Runtime.CompilerServices; -using System.Threading.Channels; -using Apache.Iggy.Contracts; -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Headers; -using Apache.Iggy.Kinds; -using Apache.Iggy.Mappers; -using Microsoft.Extensions.Logging; - -namespace Apache.Iggy.Consumers; - -public partial class IggyConsumer -{ - private readonly Channel _rentedChannel = Channel.CreateUnbounded(); - - /// - /// Receives messages asynchronously as an async stream of rented messages. Each yielded - /// shares its underlying pooled buffer with the other messages from the - /// same poll and MUST be disposed by the caller when processing is complete. The buffer is returned to the - /// pool once every message of its batch has been disposed. - /// - /// Cancellation token to stop receiving messages. - /// An async enumerable of rented messages. - /// Thrown when has not been called. - public async IAsyncEnumerable ReceiveRentedAsync( - [EnumeratorCancellation] CancellationToken ct = default) - { - if (!_isInitialized) - { - throw new ConsumerNotInitializedException(); - } - - do - { - if (!_rentedChannel.Reader.TryRead(out var message)) - { - await PollRentedMessagesAsync(ct); - continue; - } - - yield return message; - - if (_config.AutoCommitMode == AutoCommitMode.AfterReceive) - { - await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, false, ct); - } - } while (!ct.IsCancellationRequested); - } - - /// - /// Publishes a single rented message from a polled batch to the consumer channel. Called once per - /// message during . The caller acquires a reference on - /// before invocation; the channel reader is expected to release that - /// reference by disposing the produced . Override to redirect - /// rented batches to a different sink (e.g. typed deserialization) — overrides must ensure the - /// acquired reference is released exactly once on every path. - /// - /// Reference-counted handle around the polled batch. Caller has already acquired one reference. - /// - /// The rented message to publish. Payload and raw headers are slices of pooled memory tied to - /// . - /// - /// Partition the message was polled from. - /// Outcome of any prior processing (e.g. decryption). - /// Exception captured if is non-success; otherwise null. - /// Cancellation token. - protected virtual async Task PublishRentedAsync(RentedBatchHandle rental, - RentedMessageResponse message, - uint partitionId, - MessageStatus status, - Exception? error, - CancellationToken ct) - { - await _rentedChannel.Writer.WriteAsync(new ReceivedRentedMessage - { - Handle = rental, - Message = message, - CurrentOffset = message.Header.Offset, - PartitionId = partitionId, - Status = status, - Error = error - }, ct); - } - - /// - /// Polls a rented batch from the server and publishes it via . - /// Handles decryption, offset tracking, and auto-commit logic. Rental lifetime is managed via - /// shared by every produced message. - /// - protected async Task PollRentedMessagesAsync(CancellationToken ct) - { - if (!_joinedConsumerGroup) - { - LogConsumerGroupNotJoinedYetSkippingPolling(); - return; - } - - await _pollingSemaphore.WaitAsync(ct); - - PolledMessagesRental? rental = null; - RentedBatchHandle? batchHandle = null; - try - { - if (_config.PollingIntervalMs > 0) - { - await WaitBeforePollingAsync(ct); - } - - rental = await _client.PollMessagesRentedAsync(_config.StreamId, _config.TopicId, - _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize, - _config.AutoCommit, ct); - - if (rental.Messages.Count == 0) - { - if (_logger.IsEnabled(LogLevel.Debug)) - { - _logger.LogDebug("No messages received from poll for partition {PartitionId}", rental.PartitionId); - } - - return; - } - - var partitionId = (uint)rental.PartitionId; - - var hasLastOffset = _lastPolledOffset.TryGetValue(rental.PartitionId, out var lastPolledPartitionOffset); - - var currentOffset = 0ul; - - batchHandle = new RentedBatchHandle(rental); - var anyNewMessages = false; - foreach (var message in rental.Messages) - { - if (hasLastOffset && message.Header.Offset <= lastPolledPartitionOffset) - { - continue; - } - - var processedMessage = message; - var status = MessageStatus.Success; - Exception? error = null; - - if (_config.MessageEncryptor != null) - { - try - { - var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload.Span); - - Dictionary? decryptedHeaders = null; - if (!message.RawUserHeaders.IsEmpty) - { - var decryptedHeaderBytes = - _config.MessageEncryptor.Decrypt(message.RawUserHeaders.Span); - decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes); - } - - processedMessage = new RentedMessageResponse - { - Header = message.Header, - Payload = decryptedPayload, - RawUserHeaders = ReadOnlyMemory.Empty, - UserHeaders = decryptedHeaders - }; - } - catch (Exception ex) - { - LogFailedToDecryptMessage(ex, message.Header.Offset); - status = MessageStatus.DecryptionFailed; - error = ex; - } - } - - currentOffset = message.Header.Offset; - batchHandle.Acquire(); - try - { - await PublishRentedAsync(batchHandle, processedMessage, partitionId, status, error, ct); - } - catch - { - batchHandle.Release(); - throw; - } - - anyNewMessages = true; - } - - if (!anyNewMessages - && _config.AutoCommitMode != AutoCommitMode.Disabled) - { - if (_logger.IsEnabled(LogLevel.Debug)) - { - _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}", - lastPolledPartitionOffset, rental.PartitionId); - } - - await StoreOffsetAsync(lastPolledPartitionOffset, partitionId, false, ct); - } - - if (anyNewMessages) - { - _lastPolledOffset.AddOrUpdate(rental.PartitionId, currentOffset, (_, _) => currentOffset); - } - - if (_config.PollingStrategy.Kind == MessagePolling.Offset) - { - _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1); - } - } - catch (OperationCanceledException) when (ct.IsCancellationRequested) - { - throw; - } - catch (Exception ex) - { - LogFailedToPollMessages(ex); - _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages")); - } - finally - { - if (batchHandle is not null) - { - batchHandle.Release(); - } - else - { - rental?.Dispose(); - } - - _pollingSemaphore.Release(); - } - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Apache.Iggy.Contracts; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Headers; +using Apache.Iggy.Kinds; +using Apache.Iggy.Mappers; +using Microsoft.Extensions.Logging; + +namespace Apache.Iggy.Consumers; + +public partial class IggyConsumer +{ + private readonly Channel _rentedChannel = Channel.CreateUnbounded(); + + /// + /// Receives messages asynchronously as an async stream of rented messages. Each yielded + /// shares its underlying pooled buffer with the other messages from the + /// same poll and MUST be disposed by the caller when processing is complete. The buffer is returned to the + /// pool once every message of its batch has been disposed. + /// + /// Cancellation token to stop receiving messages. + /// An async enumerable of rented messages. + /// Thrown when has not been called. + public async IAsyncEnumerable ReceiveRentedAsync( + [EnumeratorCancellation] CancellationToken ct = default) + { + if (!_isInitialized) + { + throw new ConsumerNotInitializedException(); + } + + do + { + if (!_rentedChannel.Reader.TryRead(out var message)) + { + await PollRentedMessagesAsync(ct); + continue; + } + + yield return message; + + if (_config.AutoCommitMode == AutoCommitMode.AfterReceive) + { + await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, false, ct); + } + } while (!ct.IsCancellationRequested); + } + + /// + /// Publishes a single rented message from a polled batch to the consumer channel. Called once per + /// message during . The caller acquires a reference on + /// before invocation; the channel reader is expected to release that + /// reference by disposing the produced . Override to redirect + /// rented batches to a different sink (e.g. typed deserialization) — overrides must ensure the + /// acquired reference is released exactly once on every path. + /// + /// Reference-counted handle around the polled batch. Caller has already acquired one reference. + /// + /// The rented message to publish. Payload and raw headers are slices of pooled memory tied to + /// . + /// + /// Partition the message was polled from. + /// Outcome of any prior processing (e.g. decryption). + /// Exception captured if is non-success; otherwise null. + /// Cancellation token. + protected virtual async Task PublishRentedAsync(RentedBatchHandle rental, + RentedMessageResponse message, + uint partitionId, + MessageStatus status, + Exception? error, + CancellationToken ct) + { + await _rentedChannel.Writer.WriteAsync(new ReceivedRentedMessage + { + Handle = rental, + Message = message, + CurrentOffset = message.Header.Offset, + PartitionId = partitionId, + Status = status, + Error = error + }, ct); + } + + /// + /// Polls a rented batch from the server and publishes it via . + /// Handles decryption, offset tracking, and auto-commit logic. Rental lifetime is managed via + /// shared by every produced message. + /// + protected async Task PollRentedMessagesAsync(CancellationToken ct) + { + if (!_joinedConsumerGroup) + { + LogConsumerGroupNotJoinedYetSkippingPolling(); + return; + } + + await _pollingSemaphore.WaitAsync(ct); + + PolledMessagesRental? rental = null; + RentedBatchHandle? batchHandle = null; + try + { + if (_config.PollingIntervalMs > 0) + { + await WaitBeforePollingAsync(ct); + } + + rental = await _client.PollMessagesRentedAsync(_config.StreamId, _config.TopicId, + _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize, + _config.AutoCommit, ct); + + if (rental.Messages.Count == 0) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug("No messages received from poll for partition {PartitionId}", rental.PartitionId); + } + + return; + } + + var partitionId = (uint)rental.PartitionId; + + var hasLastOffset = _lastPolledOffset.TryGetValue(rental.PartitionId, out var lastPolledPartitionOffset); + + var currentOffset = 0ul; + + batchHandle = new RentedBatchHandle(rental); + var anyNewMessages = false; + foreach (var message in rental.Messages) + { + if (hasLastOffset && message.Header.Offset <= lastPolledPartitionOffset) + { + continue; + } + + var processedMessage = message; + var status = MessageStatus.Success; + Exception? error = null; + + if (_config.MessageEncryptor != null) + { + try + { + var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload.ToArray()); + + Dictionary? decryptedHeaders = null; + if (!message.RawUserHeaders.IsEmpty) + { + var decryptedHeaderBytes = + _config.MessageEncryptor.Decrypt(message.RawUserHeaders.ToArray()); + decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes); + } + + processedMessage = new RentedMessageResponse + { + Header = message.Header, + Payload = decryptedPayload, + RawUserHeaders = ReadOnlyMemory.Empty, + UserHeaders = decryptedHeaders + }; + } + catch (Exception ex) + { + LogFailedToDecryptMessage(ex, message.Header.Offset); + status = MessageStatus.DecryptionFailed; + error = ex; + } + } + + currentOffset = message.Header.Offset; + batchHandle.Acquire(); + try + { + await PublishRentedAsync(batchHandle, processedMessage, partitionId, status, error, ct); + } + catch + { + batchHandle.Release(); + throw; + } + + anyNewMessages = true; + } + + if (!anyNewMessages + && _config.AutoCommitMode != AutoCommitMode.Disabled) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}", + lastPolledPartitionOffset, rental.PartitionId); + } + + await StoreOffsetAsync(lastPolledPartitionOffset, partitionId, false, ct); + } + + if (anyNewMessages) + { + _lastPolledOffset.AddOrUpdate(rental.PartitionId, currentOffset, (_, _) => currentOffset); + } + + if (_config.PollingStrategy.Kind == MessagePolling.Offset) + { + _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1); + } + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + LogFailedToPollMessages(ex); + _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages")); + } + finally + { + if (batchHandle is not null) + { + batchHandle.Release(); + } + else + { + rental?.Dispose(); + } + + _pollingSemaphore.Release(); + } + } +} diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs index 47349f9cf4..493e552339 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs @@ -1,545 +1,551 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Collections.Concurrent; -using System.Runtime.CompilerServices; -using System.Threading.Channels; -using Apache.Iggy.Contracts; -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Headers; -using Apache.Iggy.IggyClient; -using Apache.Iggy.Kinds; -using Apache.Iggy.Mappers; -using Apache.Iggy.Utils; -using Microsoft.Extensions.Logging; - -namespace Apache.Iggy.Consumers; - -/// -/// High-level consumer for receiving messages from Iggy streams. -/// Provides automatic polling, offset management, and consumer group support. -/// -public partial class IggyConsumer : IAsyncDisposable -{ - private readonly Channel _channel; - private readonly IIggyClient _client; - private readonly IggyConsumerConfig _config; - private readonly SemaphoreSlim _connectionStateSemaphore = new(1, 1); - private readonly EventAggregator _consumerErrorEvents; - private readonly ConcurrentDictionary _lastPolledOffset = new(); - private readonly ILogger _logger; - private readonly SemaphoreSlim _pollingSemaphore = new(1, 1); - private string? _consumerGroupName; - private int _disposeState; - private volatile bool _isInitialized; - private volatile bool _joinedConsumerGroup; - private long _lastPolledAtMs; - - /// Whether this consumer has been initialized via . - protected bool IsInitialized => _isInitialized; - - /// - /// Initializes a new instance of the class - /// - /// The Iggy client for server communication - /// Consumer configuration settings - /// Logger for creating loggers - public IggyConsumer(IIggyClient client, IggyConsumerConfig config, ILoggerFactory loggerFactory) - { - _client = client; - _config = config; - _logger = loggerFactory.CreateLogger(); - - _channel = Channel.CreateUnbounded(); - _consumerErrorEvents = new EventAggregator(loggerFactory); - } - - /// - /// Disposes the consumer, leaving consumer groups and logging out if applicable - /// - public async ValueTask DisposeAsync() - { - if (Interlocked.Exchange(ref _disposeState, 1) == 1) - { - return; - } - - if (_isInitialized) - { - _client.UnsubscribeConnectionEvents(OnClientConnectionStateChangedAsync); - } - - if (!string.IsNullOrEmpty(_consumerGroupName) && _isInitialized) - { - try - { - await _client.LeaveConsumerGroupAsync(_config.StreamId, _config.TopicId, - Identifier.String(_consumerGroupName)); - - LogLeftConsumerGroup(_consumerGroupName); - } - catch (Exception e) - { - LogFailedToLeaveConsumerGroup(e, _consumerGroupName); - } - } - - if (_config.CreateIggyClient && _isInitialized) - { - try - { - await _client.LogoutUserAsync(); - _client.Dispose(); - } - catch (Exception e) - { - LogFailedToLogoutOrDispose(e); - } - } - - _consumerErrorEvents.Clear(); - _pollingSemaphore.Dispose(); - _connectionStateSemaphore.Dispose(); - } - - /// - /// Initializes the consumer by logging in (if needed) and setting up consumer groups - /// - /// Cancellation token - /// Thrown when consumer group name is invalid - /// Thrown when consumer group doesn't exist and auto-creation is disabled - public async Task InitAsync(CancellationToken ct = default) - { - if (_isInitialized) - { - return; - } - - await _connectionStateSemaphore.WaitAsync(ct); - try - { - if (_isInitialized) - { - return; - } - - if (_config.Consumer.Type == ConsumerType.ConsumerGroup && _config.PartitionId != null) - { - LogPartitionIdIsIgnoredWhenConsumerTypeIsConsumerGroup(); - _config.PartitionId = null; - } - - await _client.ConnectAsync(ct); - - if (_config.CreateIggyClient) - { - await _client.LoginUserAsync(_config.Login, _config.Password, ct); - } - - await InitializeConsumerGroupAsync(ct); - - _client.SubscribeConnectionEvents(OnClientConnectionStateChangedAsync); - - _isInitialized = true; - } - finally - { - _connectionStateSemaphore.Release(); - } - } - - /// - /// Receives messages asynchronously from the consumer as an async stream. - /// Messages are automatically polled from the server and buffered in a bounded channel. - /// - /// Cancellation token to stop receiving messages - /// An async enumerable of received messages - /// Thrown when InitAsync has not been called - public async IAsyncEnumerable ReceiveAsync([EnumeratorCancellation] CancellationToken ct = default) - { - if (!_isInitialized) - { - throw new ConsumerNotInitializedException(); - } - - do - { - if (!_channel.Reader.TryRead(out var message)) - { - await PollMessagesAsync(ct); - continue; - } - - yield return message; - - if (_config.AutoCommitMode == AutoCommitMode.AfterReceive) - { - await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, false, ct); - } - } while (!ct.IsCancellationRequested); - } - - /// - /// Manually stores the consumer offset for a specific partition. - /// Use this when auto-commit is disabled or when you need manual offset control. - /// - /// The offset to store - /// The partition ID - /// - /// Cancellation token - public async Task StoreOffsetAsync(ulong offset, uint partitionId, bool resetLastPooled = false, - CancellationToken ct = default) - { - await _client.StoreOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, offset, partitionId, ct); - - if (resetLastPooled) - { - _lastPolledOffset[(int)partitionId] = offset; - } - } - - /// - /// Deletes the stored consumer offset for a specific partition. - /// The next poll will start from the beginning or based on the polling strategy. - /// - /// The partition ID - /// Cancellation token - public async Task DeleteOffsetAsync(uint partitionId, CancellationToken ct = default) - { - await _client.DeleteOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, partitionId, ct); - } - - /// - /// Event raised when an error occurs during polling. - /// - /// Callback method - public void SubscribeToErrorEvents(Func callback) - { - _consumerErrorEvents.Subscribe(callback); - } - - /// - /// Unsubscribe from error events - /// - /// - public void UnsubscribeFromErrorEvents(Func callback) - { - _consumerErrorEvents.Unsubscribe(callback); - } - - /// - /// Initializes consumer group if configured, creating and joining as needed - /// - private async Task InitializeConsumerGroupAsync(CancellationToken ct = default) - { - if (_joinedConsumerGroup) - { - return; - } - - if (_config.Consumer.Type == ConsumerType.Consumer) - { - _joinedConsumerGroup = true; - return; - } - - _consumerGroupName = _config.Consumer.ConsumerId.Kind == IdKind.String - ? _config.Consumer.ConsumerId.GetString() - : _config.ConsumerGroupName; - - if (string.IsNullOrEmpty(_consumerGroupName)) - { - throw new InvalidConsumerGroupNameException("Consumer group name is empty or null."); - } - - try - { - var existingGroup = await _client.GetConsumerGroupByIdAsync(_config.StreamId, _config.TopicId, - Identifier.String(_consumerGroupName), ct); - - if (existingGroup == null && _config.CreateConsumerGroupIfNotExists) - { - LogCreatingConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId); - - var createdGroup = await TryCreateConsumerGroupAsync(_consumerGroupName, ct); - - if (createdGroup) - { - LogConsumerGroupCreated(_consumerGroupName); - } - } - else if (existingGroup == null) - { - throw new ConsumerGroupNotFoundException(_consumerGroupName); - } - - if (_config.JoinConsumerGroup) - { - LogJoiningConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId); - - await _client.JoinConsumerGroupAsync(_config.StreamId, _config.TopicId, - Identifier.String(_consumerGroupName), ct); - - _joinedConsumerGroup = true; - LogConsumerGroupJoined(_consumerGroupName); - } - } - catch (Exception ex) - { - LogFailedToInitializeConsumerGroup(ex, _consumerGroupName); - throw; - } - } - - /// - /// Attempts to create a consumer group, handling the case where it already exists - /// - /// True if the group was created or already exists, false on error - private async Task TryCreateConsumerGroupAsync(string groupName, CancellationToken ct) - { - try - { - await _client.CreateConsumerGroupAsync(_config.StreamId, _config.TopicId, - groupName, ct); - } - catch (IggyInvalidStatusCodeException ex) - { - // 5004 - Consumer group already exists TODO: refactor errors - if (ex.StatusCode != 5004) - { - LogFailedToCreateConsumerGroup(ex, groupName); - return false; - } - - return true; - } - - return true; - } - - /// - /// Polls messages from the server and writes them to the internal channel. - /// Handles decryption, offset tracking, and auto-commit logic. - /// Uses semaphore to ensure single concurrent polling operation. - /// - private async Task PollMessagesAsync(CancellationToken ct) - { - if (!_joinedConsumerGroup) - { - LogConsumerGroupNotJoinedYetSkippingPolling(); - return; - } - - await _pollingSemaphore.WaitAsync(ct); - - try - { - if (_config.PollingIntervalMs > 0) - { - await WaitBeforePollingAsync(ct); - } - - var messages = await _client.PollMessagesAsync(_config.StreamId, _config.TopicId, - _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize, - _config.AutoCommit, ct); - - var receiveMessages = messages.Messages.Count > 0; - - if (_lastPolledOffset.TryGetValue(messages.PartitionId, out var lastPolledPartitionOffset)) - { - messages.Messages = messages.Messages.Where(x => x.Header.Offset > lastPolledPartitionOffset).ToList(); - } - - if (messages.Messages.Count == 0 - && receiveMessages - && _config.AutoCommitMode != AutoCommitMode.Disabled) - { - _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}", - lastPolledPartitionOffset, messages.PartitionId); - await StoreOffsetAsync(lastPolledPartitionOffset, (uint)messages.PartitionId, false, ct); - } - - if (messages.Messages.Count == 0) - { - return; - } - - var currentOffset = 0ul; - foreach (var message in messages.Messages) - { - var processedMessage = message; - var status = MessageStatus.Success; - Exception? error = null; - - if (_config.MessageEncryptor != null) - { - try - { - var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload); - - Dictionary? decryptedHeaders = null; - if (message.RawUserHeaders is { Length: > 0 }) - { - var decryptedHeaderBytes = _config.MessageEncryptor.Decrypt(message.RawUserHeaders); - decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes); - } - - processedMessage = new MessageResponse - { - Header = message.Header, - Payload = decryptedPayload, - UserHeaders = decryptedHeaders - }; - } - catch (Exception ex) - { - LogFailedToDecryptMessage(ex, message.Header.Offset); - status = MessageStatus.DecryptionFailed; - error = ex; - } - } - - var receivedMessage = new ReceivedMessage - { - Message = processedMessage, - CurrentOffset = processedMessage.Header.Offset, - PartitionId = (uint)messages.PartitionId, - Status = status, - Error = error - }; - - await _channel.Writer.WriteAsync(receivedMessage, ct); - currentOffset = receivedMessage.CurrentOffset; - } - - _lastPolledOffset.AddOrUpdate(messages.PartitionId, currentOffset, - (_, _) => currentOffset); - - if (_config.PollingStrategy.Kind == MessagePolling.Offset) - { - _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1); - } - } - catch (Exception ex) - { - LogFailedToPollMessages(ex); - _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages")); - } - finally - { - _pollingSemaphore.Release(); - } - } - - /// - /// Implements polling interval throttling to avoid excessive server requests. - /// Uses monotonic time tracking to ensure proper intervals even with clock adjustments. - /// - private async Task WaitBeforePollingAsync(CancellationToken ct) - { - var intervalMs = _config.PollingIntervalMs; - if (intervalMs <= 0) - { - return; - } - - var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - var lastPolledAtMs = Interlocked.Read(ref _lastPolledAtMs); - - if (nowMs < lastPolledAtMs) - { - LogMonotonicTimeWentBackwards(nowMs, lastPolledAtMs); - await Task.Delay(intervalMs, ct); - Interlocked.Exchange(ref _lastPolledAtMs, nowMs); - return; - } - - var elapsedMs = nowMs - lastPolledAtMs; - if (elapsedMs >= intervalMs) - { - LogNoNeedToWaitBeforePolling(nowMs, lastPolledAtMs, elapsedMs); - Interlocked.Exchange(ref _lastPolledAtMs, nowMs); - return; - } - - var remainingMs = intervalMs - elapsedMs; - LogWaitingBeforePolling(remainingMs); - - if (remainingMs > 0) - { - await Task.Delay((int)remainingMs, ct); - } - - Interlocked.Exchange(ref _lastPolledAtMs, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); - } - - /// - /// Handles connection state changes from the client. - /// - /// Event object - private async Task OnClientConnectionStateChangedAsync(ConnectionStateChangedEventArgs e) - { - LogConnectionStateChanged(e.PreviousState, e.CurrentState); - - await _connectionStateSemaphore.WaitAsync(); - try - { - if (e.CurrentState == ConnectionState.Disconnected) - { - _joinedConsumerGroup = false; - } - - if (e.CurrentState != ConnectionState.Authenticated - || e.PreviousState == ConnectionState.Authenticated - || _joinedConsumerGroup) - { - return; - } - - await RejoinConsumerGroupOnReconnectionAsync(); - } - finally - { - _connectionStateSemaphore.Release(); - } - } - - /// - /// Asynchronously rejoins the consumer group after a client reconnection. - /// This restores the consumer group membership that was lost during the connection failure. - /// - private async Task RejoinConsumerGroupOnReconnectionAsync() - { - if (string.IsNullOrEmpty(_consumerGroupName)) - { - LogConsumerGroupNameIsEmptySkippingRejoiningConsumerGroup(); - return; - } - - try - { - await InitializeConsumerGroupAsync(); - } - catch (Exception ex) - { - LogFailedToRejoinConsumerGroup(ex, _consumerGroupName); - _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, - "Failed to rejoin consumer group after reconnection")); - } - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Apache.Iggy.Contracts; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; +using Apache.Iggy.Mappers; +using Apache.Iggy.Utils; +using Microsoft.Extensions.Logging; + +namespace Apache.Iggy.Consumers; + +/// +/// High-level consumer for receiving messages from Iggy streams. +/// Provides automatic polling, offset management, and consumer group support. +/// +public partial class IggyConsumer : IAsyncDisposable +{ + private readonly Channel _channel; + private readonly IIggyClient _client; + private readonly IggyConsumerConfig _config; + private readonly SemaphoreSlim _connectionStateSemaphore = new(1, 1); + private readonly EventAggregator _consumerErrorEvents; + private readonly ConcurrentDictionary _lastPolledOffset = new(); + private readonly ILogger _logger; + private readonly SemaphoreSlim _pollingSemaphore = new(1, 1); + private string? _consumerGroupName; + private int _disposeState; + private volatile bool _isInitialized; + private volatile bool _joinedConsumerGroup; + private long _lastPolledAtMs; + + /// Whether this consumer has been initialized via . + protected bool IsInitialized => _isInitialized; + + /// + /// Initializes a new instance of the class + /// + /// The Iggy client for server communication + /// Consumer configuration settings + /// Logger for creating loggers + public IggyConsumer(IIggyClient client, IggyConsumerConfig config, ILoggerFactory loggerFactory) + { + _client = client; + _config = config; + _logger = loggerFactory.CreateLogger(); + + _channel = Channel.CreateBounded(new BoundedChannelOptions((int)config.BatchSize * 2)); + _consumerErrorEvents = new EventAggregator(loggerFactory); + } + + /// + /// Disposes the consumer, leaving consumer groups and logging out if applicable + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposeState, 1) == 1) + { + return; + } + + if (_isInitialized) + { + _client.UnsubscribeConnectionEvents(OnClientConnectionStateChangedAsync); + } + + if (!string.IsNullOrEmpty(_consumerGroupName) && _isInitialized) + { + try + { + await _client.LeaveConsumerGroupAsync(_config.StreamId, _config.TopicId, + Identifier.String(_consumerGroupName)); + + LogLeftConsumerGroup(_consumerGroupName); + } + catch (Exception e) + { + LogFailedToLeaveConsumerGroup(e, _consumerGroupName); + } + } + + if (_config.CreateIggyClient && _isInitialized) + { + try + { + await _client.LogoutUserAsync(); + _client.Dispose(); + } + catch (Exception e) + { + LogFailedToLogoutOrDispose(e); + } + } + + _consumerErrorEvents.Clear(); + _pollingSemaphore.Dispose(); + _connectionStateSemaphore.Dispose(); + } + + /// + /// Initializes the consumer by logging in (if needed) and setting up consumer groups + /// + /// Cancellation token + /// Thrown when consumer group name is invalid + /// Thrown when consumer group doesn't exist and auto-creation is disabled + public async Task InitAsync(CancellationToken ct = default) + { + if (_isInitialized) + { + return; + } + + await _connectionStateSemaphore.WaitAsync(ct); + try + { + if (_isInitialized) + { + return; + } + + if (_config.Consumer.Type == ConsumerType.ConsumerGroup && _config.PartitionId != null) + { + LogPartitionIdIsIgnoredWhenConsumerTypeIsConsumerGroup(); + _config.PartitionId = null; + } + + await _client.ConnectAsync(ct); + + if (_config.CreateIggyClient) + { + await _client.LoginUserAsync(_config.Login, _config.Password, ct); + } + + await InitializeConsumerGroupAsync(ct); + + _client.SubscribeConnectionEvents(OnClientConnectionStateChangedAsync); + + _isInitialized = true; + } + finally + { + _connectionStateSemaphore.Release(); + } + } + + /// + /// Receives messages asynchronously from the consumer as an async stream. + /// Messages are automatically polled from the server and buffered in a bounded channel. + /// + /// Cancellation token to stop receiving messages + /// An async enumerable of received messages + /// Thrown when InitAsync has not been called + public async IAsyncEnumerable ReceiveAsync([EnumeratorCancellation] CancellationToken ct = default) + { + if (!_isInitialized) + { + throw new ConsumerNotInitializedException(); + } + + do + { + if (!_channel.Reader.TryRead(out var message)) + { + await PollMessagesAsync(ct); + continue; + } + + yield return message; + + if (_config.AutoCommitMode == AutoCommitMode.AfterReceive) + { + await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, false, ct); + } + } while (!ct.IsCancellationRequested); + } + + /// + /// Manually stores the consumer offset for a specific partition. + /// Use this when auto-commit is disabled or when you need manual offset control. + /// + /// The offset to store + /// The partition ID + /// + /// Cancellation token + public async Task StoreOffsetAsync(ulong offset, uint partitionId, bool resetLastPooled = false, + CancellationToken ct = default) + { + await _client.StoreOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, offset, partitionId, ct); + + if (resetLastPooled) + { + _lastPolledOffset[(int)partitionId] = offset; + } + } + + /// + /// Deletes the stored consumer offset for a specific partition. + /// The next poll will start from the beginning or based on the polling strategy. + /// + /// The partition ID + /// Cancellation token + public async Task DeleteOffsetAsync(uint partitionId, CancellationToken ct = default) + { + await _client.DeleteOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, partitionId, ct); + } + + /// + /// Event raised when an error occurs during polling. + /// + /// Callback method + public void SubscribeToErrorEvents(Func callback) + { + _consumerErrorEvents.Subscribe(callback); + } + + /// + /// Unsubscribe from error events + /// + /// + public void UnsubscribeFromErrorEvents(Func callback) + { + _consumerErrorEvents.Unsubscribe(callback); + } + + /// + /// Initializes consumer group if configured, creating and joining as needed + /// + private async Task InitializeConsumerGroupAsync(CancellationToken ct = default) + { + if (_joinedConsumerGroup) + { + return; + } + + if (_config.Consumer.Type == ConsumerType.Consumer) + { + _joinedConsumerGroup = true; + return; + } + + _consumerGroupName = _config.Consumer.ConsumerId.Kind == IdKind.String + ? _config.Consumer.ConsumerId.GetString() + : _config.ConsumerGroupName; + + if (string.IsNullOrEmpty(_consumerGroupName)) + { + throw new InvalidConsumerGroupNameException("Consumer group name is empty or null."); + } + + try + { + var existingGroup = await _client.GetConsumerGroupByIdAsync(_config.StreamId, _config.TopicId, + Identifier.String(_consumerGroupName), ct); + + if (existingGroup == null && _config.CreateConsumerGroupIfNotExists) + { + LogCreatingConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId); + + var createdGroup = await TryCreateConsumerGroupAsync(_consumerGroupName, ct); + + if (createdGroup) + { + LogConsumerGroupCreated(_consumerGroupName); + } + } + else if (existingGroup == null) + { + throw new ConsumerGroupNotFoundException(_consumerGroupName); + } + + if (_config.JoinConsumerGroup) + { + LogJoiningConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId); + + await _client.JoinConsumerGroupAsync(_config.StreamId, _config.TopicId, + Identifier.String(_consumerGroupName), ct); + + _joinedConsumerGroup = true; + LogConsumerGroupJoined(_consumerGroupName); + } + } + catch (Exception ex) + { + LogFailedToInitializeConsumerGroup(ex, _consumerGroupName); + throw; + } + } + + /// + /// Attempts to create a consumer group, handling the case where it already exists + /// + /// True if the group was created or already exists, false on error + private async Task TryCreateConsumerGroupAsync(string groupName, CancellationToken ct) + { + try + { + await _client.CreateConsumerGroupAsync(_config.StreamId, _config.TopicId, + groupName, ct); + } + catch (IggyInvalidStatusCodeException ex) + { + // 5004 - Consumer group already exists TODO: refactor errors + if (ex.StatusCode != 5004) + { + LogFailedToCreateConsumerGroup(ex, groupName); + return false; + } + + return true; + } + + return true; + } + + /// + /// Polls messages from the server and writes them to the internal channel. + /// Handles decryption, offset tracking, and auto-commit logic. + /// Uses semaphore to ensure single concurrent polling operation. + /// + private async Task PollMessagesAsync(CancellationToken ct) + { + if (!_joinedConsumerGroup) + { + LogConsumerGroupNotJoinedYetSkippingPolling(); + return; + } + + await _pollingSemaphore.WaitAsync(ct); + + try + { + if (_config.PollingIntervalMs > 0) + { + await WaitBeforePollingAsync(ct); + } + + var messages = await _client.PollMessagesAsync(_config.StreamId, _config.TopicId, + _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize, + _config.AutoCommit, ct); + + if (messages.Messages.Count == 0) + { + return; + } + + var hasLastOffset = _lastPolledOffset.TryGetValue(messages.PartitionId, + out var lastPolledPartitionOffset); + + var currentOffset = 0ul; + var anyNewMessages = false; + foreach (var message in messages.Messages) + { + if (hasLastOffset && message.Header.Offset <= lastPolledPartitionOffset) + { + continue; + } + + var processedMessage = message; + var status = MessageStatus.Success; + Exception? error = null; + + if (_config.MessageEncryptor != null) + { + try + { + var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload); + + Dictionary? decryptedHeaders = null; + if (message.RawUserHeaders is { Length: > 0 }) + { + var decryptedHeaderBytes = _config.MessageEncryptor.Decrypt(message.RawUserHeaders); + decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes); + } + + processedMessage = new MessageResponse + { + Header = message.Header, + Payload = decryptedPayload, + UserHeaders = decryptedHeaders + }; + } + catch (Exception ex) + { + LogFailedToDecryptMessage(ex, message.Header.Offset); + status = MessageStatus.DecryptionFailed; + error = ex; + } + } + + var receivedMessage = new ReceivedMessage + { + Message = processedMessage, + CurrentOffset = processedMessage.Header.Offset, + PartitionId = (uint)messages.PartitionId, + Status = status, + Error = error + }; + + await _channel.Writer.WriteAsync(receivedMessage, ct); + currentOffset = receivedMessage.CurrentOffset; + anyNewMessages = true; + } + + if (!anyNewMessages) + { + if (_config.AutoCommitMode != AutoCommitMode.Disabled) + { + _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}", + lastPolledPartitionOffset, messages.PartitionId); + await StoreOffsetAsync(lastPolledPartitionOffset, (uint)messages.PartitionId, false, ct); + } + + return; + } + + _lastPolledOffset.AddOrUpdate(messages.PartitionId, currentOffset, + (_, _) => currentOffset); + + if (_config.PollingStrategy.Kind == MessagePolling.Offset) + { + _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1); + } + } + catch (Exception ex) + { + LogFailedToPollMessages(ex); + _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages")); + } + finally + { + _pollingSemaphore.Release(); + } + } + + /// + /// Implements polling interval throttling to avoid excessive server requests. + /// Uses monotonic time tracking to ensure proper intervals even with clock adjustments. + /// + private async Task WaitBeforePollingAsync(CancellationToken ct) + { + var intervalMs = _config.PollingIntervalMs; + if (intervalMs <= 0) + { + return; + } + + var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var lastPolledAtMs = Interlocked.Read(ref _lastPolledAtMs); + + if (nowMs < lastPolledAtMs) + { + LogMonotonicTimeWentBackwards(nowMs, lastPolledAtMs); + await Task.Delay(intervalMs, ct); + Interlocked.Exchange(ref _lastPolledAtMs, nowMs); + return; + } + + var elapsedMs = nowMs - lastPolledAtMs; + if (elapsedMs >= intervalMs) + { + LogNoNeedToWaitBeforePolling(nowMs, lastPolledAtMs, elapsedMs); + Interlocked.Exchange(ref _lastPolledAtMs, nowMs); + return; + } + + var remainingMs = intervalMs - elapsedMs; + LogWaitingBeforePolling(remainingMs); + + if (remainingMs > 0) + { + await Task.Delay((int)remainingMs, ct); + } + + Interlocked.Exchange(ref _lastPolledAtMs, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + } + + /// + /// Handles connection state changes from the client. + /// + /// Event object + private async Task OnClientConnectionStateChangedAsync(ConnectionStateChangedEventArgs e) + { + LogConnectionStateChanged(e.PreviousState, e.CurrentState); + + await _connectionStateSemaphore.WaitAsync(); + try + { + if (e.CurrentState == ConnectionState.Disconnected) + { + _joinedConsumerGroup = false; + } + + if (e.CurrentState != ConnectionState.Authenticated + || e.PreviousState == ConnectionState.Authenticated + || _joinedConsumerGroup) + { + return; + } + + await RejoinConsumerGroupOnReconnectionAsync(); + } + finally + { + _connectionStateSemaphore.Release(); + } + } + + /// + /// Asynchronously rejoins the consumer group after a client reconnection. + /// This restores the consumer group membership that was lost during the connection failure. + /// + private async Task RejoinConsumerGroupOnReconnectionAsync() + { + if (string.IsNullOrEmpty(_consumerGroupName)) + { + LogConsumerGroupNameIsEmptySkippingRejoiningConsumerGroup(); + return; + } + + try + { + await InitializeConsumerGroupAsync(); + } + catch (Exception ex) + { + LogFailedToRejoinConsumerGroup(ex, _consumerGroupName); + _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, + "Failed to rejoin consumer group after reconnection")); + } + } +} diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs index b8ea65b622..5f9126169e 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs @@ -1,193 +1,142 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Runtime.CompilerServices; -using System.Threading.Channels; -using Apache.Iggy.Contracts; -using Apache.Iggy.Exceptions; -using Apache.Iggy.IggyClient; -using Microsoft.Extensions.Logging; - -namespace Apache.Iggy.Consumers; - -/// -/// Typed consumer that automatically deserializes message payloads to type T. -/// Extends with deserialization capabilities. -/// -/// The type to deserialize message payloads to -public class IggyConsumer : IggyConsumer -{ - private readonly Channel> _deserializedChannel = - Channel.CreateUnbounded>(); - - private readonly IggyConsumerConfig _typedConfig; - private readonly ILogger> _typedLogger; - - /// - /// Initializes a new instance of the typed class - /// - /// The Iggy client for server communication - /// Typed consumer configuration including deserializer - /// Logger instance for diagnostic output - public IggyConsumer(IIggyClient client, IggyConsumerConfig config, ILoggerFactory logger) : base(client, config, - logger) - { - _typedConfig = config; - _typedLogger = logger.CreateLogger>(); - } - - /// - /// Receives and deserializes messages from the consumer - /// - /// Cancellation token - /// Async enumerable of deserialized messages with status - public async IAsyncEnumerable> ReceiveDeserializedAsync( - [EnumeratorCancellation] CancellationToken ct = default) - { - await foreach (var message in ReceiveAsync(ct)) - { - if (message.Status != MessageStatus.Success) - { - yield return new ReceivedMessage - { - Data = default, - Message = message.Message, - CurrentOffset = message.CurrentOffset, - PartitionId = message.PartitionId, - Status = message.Status, - Error = message.Error - }; - continue; - } - - T? deserializedPayload = default; - Exception? deserializationError = null; - var status = MessageStatus.Success; - - try - { - deserializedPayload = Deserialize(message.Message.Payload); - } - catch (Exception ex) - { - _typedLogger.LogError(ex, "Failed to deserialize message at offset {Offset}", message.CurrentOffset); - status = MessageStatus.DeserializationFailed; - deserializationError = ex; - } - - yield return new ReceivedMessage - { - Data = deserializedPayload, - Message = message.Message, - CurrentOffset = message.CurrentOffset, - PartitionId = message.PartitionId, - Status = status, - Error = deserializationError - }; - } - } - - /// - /// Receives and deserializes messages via the rented poll path. Each polled batch is deserialized - /// in full before any message is yielded — the rented buffer is returned immediately after - /// deserialization, independently of how fast the caller iterates. The caller does not need to - /// dispose anything. - /// - /// Cancellation token. - /// Async enumerable of deserialized messages with status. - public async IAsyncEnumerable> ReceiveRentedDeserializedAsync( - [EnumeratorCancellation] CancellationToken ct = default) - { - if (!IsInitialized) - { - throw new ConsumerNotInitializedException(); - } - - do - { - if (!_deserializedChannel.Reader.TryRead(out DeserializedMessage? message)) - { - await PollRentedMessagesAsync(ct); - continue; - } - - yield return message; - - if (_typedConfig.AutoCommitMode == AutoCommitMode.AfterReceive) - { - await StoreOffsetAsync(message.Header.Offset, message.PartitionId, false, ct); - } - } while (!ct.IsCancellationRequested); - } - - /// - /// Overrides the base batch-publishing step: instead of routing rented messages through the base - /// class channel, deserializes the entire batch immediately (releasing all rented buffer refs via - /// ) and writes the deserialized results to - /// . Auto-commit is also handled here since the base-class - /// yield path is bypassed. - /// - protected override async Task PublishRentedAsync(RentedBatchHandle rental, RentedMessageResponse message, - uint partitionId, MessageStatus status, - Exception? error, CancellationToken ct) - { - T? data = default; - var deserError = status != MessageStatus.Success ? error : null; - var msgStatus = status; - - if (status == MessageStatus.Success) - { - try - { - data = Deserialize(message.Payload); - } - catch (Exception ex) - { - _typedLogger.LogError(ex, "Failed to deserialize message at offset {Offset}", - message.Header.Offset); - msgStatus = MessageStatus.DeserializationFailed; - deserError = ex; - } - } - - var deserialized = new DeserializedMessage - { - Data = data, - Header = message.Header, - UserHeaders = message.UserHeaders, - CurrentOffset = message.Header.Offset, - PartitionId = partitionId, - Status = msgStatus, - Error = deserError - }; - - await _deserializedChannel.Writer.WriteAsync(deserialized, ct); - - rental.Release(); - } - - /// - /// Deserializes a message payload from a span using the configured deserializer. Zero-copy when the - /// deserializer overrides the span overload; otherwise falls back to a one-time array copy. - /// - /// The payload memory to deserialize. - /// The deserialized object of type T. - public T Deserialize(ReadOnlyMemory payload) - { - return _typedConfig.Deserializer.Deserialize(payload); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Apache.Iggy.Contracts; +using Apache.Iggy.Exceptions; +using Apache.Iggy.IggyClient; +using Microsoft.Extensions.Logging; + +namespace Apache.Iggy.Consumers; + +/// +/// Typed consumer that automatically deserializes message payloads to type T. +/// Extends with deserialization capabilities. +/// +/// The type to deserialize message payloads to +public class IggyConsumer : IggyConsumer +{ + private readonly Channel> _deserializedChannel = + Channel.CreateUnbounded>(); + + private readonly IggyConsumerConfig _typedConfig; + private readonly ILogger> _typedLogger; + + /// + /// Initializes a new instance of the typed class + /// + /// The Iggy client for server communication + /// Typed consumer configuration including deserializer + /// Logger instance for diagnostic output + public IggyConsumer(IIggyClient client, IggyConsumerConfig config, ILoggerFactory logger) : base(client, config, + logger) + { + _typedConfig = config; + _typedLogger = logger.CreateLogger>(); + } + + /// + /// Receives and deserializes messages via the rented poll path. Each polled batch is deserialized + /// in full before any message is yielded — the rented buffer is returned immediately after + /// deserialization, independently of how fast the caller iterates. The caller does not need to + /// dispose anything. + /// + /// Cancellation token. + /// Async enumerable of deserialized messages with status. + public async IAsyncEnumerable> ReceiveDeserializedAsync( + [EnumeratorCancellation] CancellationToken ct = default) + { + if (!IsInitialized) + { + throw new ConsumerNotInitializedException(); + } + + do + { + if (!_deserializedChannel.Reader.TryRead(out ReceivedMessage? message)) + { + await PollRentedMessagesAsync(ct); + continue; + } + + yield return message; + + if (_typedConfig.AutoCommitMode == AutoCommitMode.AfterReceive) + { + await StoreOffsetAsync(message.Header.Offset, message.PartitionId, false, ct); + } + } while (!ct.IsCancellationRequested); + } + + /// + /// Overrides the base batch-publishing step: instead of routing rented messages through the base + /// class channel, deserializes the entire batch immediately (releasing all rented buffer refs via + /// ) and writes the deserialized results to + /// . Auto-commit is also handled here since the base-class + /// yield path is bypassed. + /// + protected override async Task PublishRentedAsync(RentedBatchHandle rental, RentedMessageResponse message, + uint partitionId, MessageStatus status, + Exception? error, CancellationToken ct) + { + T? data = default; + var deserError = status != MessageStatus.Success ? error : null; + var msgStatus = status; + + if (status == MessageStatus.Success) + { + try + { + data = Deserialize(message.Payload); + } + catch (Exception ex) + { + _typedLogger.LogError(ex, "Failed to deserialize message at offset {Offset}", + message.Header.Offset); + msgStatus = MessageStatus.DeserializationFailed; + deserError = ex; + } + } + + var deserialized = new ReceivedMessage + { + Data = data, + Header = message.Header, + UserHeaders = message.UserHeaders, + CurrentOffset = message.Header.Offset, + PartitionId = partitionId, + Status = msgStatus, + Error = deserError + }; + + await _deserializedChannel.Writer.WriteAsync(deserialized, ct); + + rental.Release(); + } + + /// + /// Deserializes a message payload from a span using the configured deserializer. Zero-copy when the + /// deserializer overrides the span overload; otherwise falls back to a one-time array copy. + /// + /// The payload memory to deserialize. + /// The deserialized object of type T. + public T Deserialize(ReadOnlyMemory payload) + { + return _typedConfig.Deserializer.Deserialize(payload); + } +} diff --git a/foreign/csharp/Iggy_SDK/Consumers/ReceivedMessage.cs b/foreign/csharp/Iggy_SDK/Consumers/ReceivedMessage.cs index 9c5160cc2d..24213821c0 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/ReceivedMessage.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/ReceivedMessage.cs @@ -16,19 +16,53 @@ // under the License. using Apache.Iggy.Contracts; +using Apache.Iggy.Headers; +using Apache.Iggy.Messages; namespace Apache.Iggy.Consumers; /// -/// Represents a message received from the Iggy consumer with a deserialized payload of type T +/// Represents a message whose payload was deserialized directly from rented memory. The rented buffer has +/// already been returned to the pool by the time this message is yielded, so only the header, user headers, +/// and the deserialized are available; the raw payload bytes are not retained. /// -/// The type of the deserialized message payload -public class ReceivedMessage : ReceivedMessage +/// The deserialized payload type. +public sealed class ReceivedMessage { /// - /// The deserialized message payload. Will be null if deserialization failed. + /// Message header. + /// + public required MessageHeader Header { get; init; } + + /// + /// The deserialized payload. Null if is not . /// public T? Data { get; init; } + + /// + /// Parsed user headers, if present. + /// + public Dictionary? UserHeaders { get; init; } + + /// + /// The current offset of this message in the partition. + /// + public required ulong CurrentOffset { get; init; } + + /// + /// The partition ID from which this message was consumed. + /// + public uint PartitionId { get; init; } + + /// + /// The status of the message (Success, DecryptionFailed, DeserializationFailed). + /// + public MessageStatus Status { get; init; } = MessageStatus.Success; + + /// + /// The exception that occurred during processing, if any. + /// + public Exception? Error { get; init; } } /// diff --git a/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs b/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs index ac0f92e6c1..60ac5ac7b1 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs @@ -1,126 +1,126 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Contracts; - -namespace Apache.Iggy.Consumers; - -/// -/// Represents a message received from the Iggy consumer whose payload and raw headers are backed by rented memory -/// shared across a polled batch. The caller SHOULD dispose the message once processing is complete for deterministic -/// release of the underlying pool buffer. If the caller forgets, the buffer is still returned to the pool when the -/// owning becomes unreachable and its finalizer runs - non-deterministic but safe. -/// -public sealed class ReceivedRentedMessage : IDisposable -{ - private int _disposed; - internal RentedBatchHandle? Handle { get; init; } - - /// - /// The underlying rented message response containing headers and rented payload memory. - /// - public required RentedMessageResponse Message { get; init; } - - /// - /// The current offset of this message in the partition. - /// - public required ulong CurrentOffset { get; init; } - - /// - /// The partition ID from which this message was consumed. - /// - public uint PartitionId { get; init; } - - /// - /// The status of the message (Success, DecryptionFailed). - /// - public MessageStatus Status { get; init; } = MessageStatus.Success; - - /// - /// The exception that occurred during processing, if any. - /// - public Exception? Error { get; init; } - - /// - /// Releases this message's reference on the underlying rental. When the final message of a batch is disposed, - /// the rented buffer is returned to the pool and any payload/raw header slices are invalidated. - /// - public void Dispose() - { - if (Interlocked.Exchange(ref _disposed, 1) == 1) - { - return; - } - - Handle?.Release(); - } -} - -/// -/// Reference-counted handle around a . Shared by all -/// instances produced from one poll; the rental is disposed -/// when the reference count drops to zero. -/// -public sealed class RentedBatchHandle : IDisposable -{ - private readonly PolledMessagesRental _rental; - private int _refCount = 1; - - /// - /// Creates a new handle with a single self-reference held by the constructing producer. The producer - /// must call before each publish and release the self-reference (via - /// or ) once it has finished producing. - /// - /// The polled messages rental whose lifetime this handle manages. - public RentedBatchHandle(PolledMessagesRental rental) - { - _rental = rental; - } - - /// - /// Releases one reference on the underlying rental. Equivalent to . - /// - public void Dispose() - { - Release(); - } - - /// - /// Acquires an additional reference. Must be balanced by a matching . - /// - public void Acquire() - { - Interlocked.Increment(ref _refCount); - } - - /// - /// Decrements the reference count. When the count reaches zero, the underlying rental is disposed - /// and its pool buffer returned. - /// - public void Release() - { - var remaining = Interlocked.Decrement(ref _refCount); - if (remaining == 0) - { - _rental.Dispose(); - } - else if (remaining < 0) - { - throw new InvalidOperationException("RentedBatchHandle released more times than acquired."); - } - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Contracts; + +namespace Apache.Iggy.Consumers; + +/// +/// Represents a message received from the Iggy consumer whose payload and raw headers are backed by rented memory +/// shared across a polled batch. The caller SHOULD dispose the message once processing is complete for deterministic +/// release of the underlying pool buffer. If the caller forgets, the buffer is still returned to the pool when the +/// owning becomes unreachable and its finalizer runs - non-deterministic but safe. +/// +public sealed class ReceivedRentedMessage : IDisposable +{ + private int _disposed; + internal RentedBatchHandle? Handle { get; init; } + + /// + /// The underlying rented message response containing headers and rented payload memory. + /// + public required RentedMessageResponse Message { get; init; } + + /// + /// The current offset of this message in the partition. + /// + public required ulong CurrentOffset { get; init; } + + /// + /// The partition ID from which this message was consumed. + /// + public uint PartitionId { get; init; } + + /// + /// The status of the message (Success, DecryptionFailed). + /// + public MessageStatus Status { get; init; } = MessageStatus.Success; + + /// + /// The exception that occurred during processing, if any. + /// + public Exception? Error { get; init; } + + /// + /// Releases this message's reference on the underlying rental. When the final message of a batch is disposed, + /// the rented buffer is returned to the pool and any payload/raw header slices are invalidated. + /// + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + Handle?.Release(); + } +} + +/// +/// Reference-counted handle around a . Shared by all +/// instances produced from one poll; the rental is disposed +/// when the reference count drops to zero. +/// +public sealed class RentedBatchHandle : IDisposable +{ + private readonly PolledMessagesRental _rental; + private int _refCount = 1; + + /// + /// Creates a new handle with a single self-reference held by the constructing producer. The producer + /// must call before each publish and release the self-reference (via + /// or ) once it has finished producing. + /// + /// The polled messages rental whose lifetime this handle manages. + public RentedBatchHandle(PolledMessagesRental rental) + { + _rental = rental; + } + + /// + /// Releases one reference on the underlying rental. Equivalent to . + /// + public void Dispose() + { + Release(); + } + + /// + /// Acquires an additional reference. Must be balanced by a matching . + /// + public void Acquire() + { + Interlocked.Increment(ref _refCount); + } + + /// + /// Decrements the reference count. When the count reaches zero, the underlying rental is disposed + /// and its pool buffer returned. + /// + public void Release() + { + var remaining = Interlocked.Decrement(ref _refCount); + if (remaining == 0) + { + _rental.Dispose(); + } + else if (remaining < 0) + { + throw new InvalidOperationException("RentedBatchHandle released more times than acquired."); + } + } +} diff --git a/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs index 7895e7921d..f29ca79fa8 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs @@ -1,96 +1,96 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Text.Json.Serialization; -using Apache.Iggy.Headers; -using Apache.Iggy.JsonConverters; -using Apache.Iggy.Mappers; -using Apache.Iggy.Messages; - -namespace Apache.Iggy.Contracts; - -/// -/// Response from the server containing a message payload. -/// -[JsonConverter(typeof(MessageResponseConverter))] -public sealed class MessageResponse -{ - private byte[]? _rawUserHeaders; - private Dictionary? _userHeaders; - private bool _userHeadersInitialized; - - /// - /// Message header. - /// - public required MessageHeader Header { get; set; } - - /// - /// Message payload. - /// - public required byte[] Payload { get; set; } = []; - - /// - /// Headers defined by the user. - /// - public Dictionary? UserHeaders - { - get - { - if (!_userHeadersInitialized) - { - _userHeaders = _rawUserHeaders is { Length: > 0 } - ? BinaryMapper.TryMapHeaders(_rawUserHeaders) - : null; - _userHeadersInitialized = true; - } - - return _userHeaders; - } - set - { - _userHeaders = value; - _userHeadersInitialized = true; - } - } - - /// - /// Raw user header bytes before deserialization. - /// Used internally for decrypting encrypted headers. - /// - [JsonIgnore] - internal byte[]? RawUserHeaders - { - get => _rawUserHeaders; - set - { - _rawUserHeaders = value; - _userHeaders = null; - _userHeadersInitialized = false; - } - } - - internal void ParseUserHeaders() - { - if (!_userHeadersInitialized) - { - _userHeaders = _rawUserHeaders is { Length: > 0 } - ? BinaryMapper.TryMapHeaders(_rawUserHeaders) - : null; - _userHeadersInitialized = true; - } - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text.Json.Serialization; +using Apache.Iggy.Headers; +using Apache.Iggy.JsonConverters; +using Apache.Iggy.Mappers; +using Apache.Iggy.Messages; + +namespace Apache.Iggy.Contracts; + +/// +/// Response from the server containing a message payload. +/// +[JsonConverter(typeof(MessageResponseConverter))] +public sealed class MessageResponse +{ + private byte[]? _rawUserHeaders; + private Dictionary? _userHeaders; + private bool _userHeadersInitialized; + + /// + /// Message header. + /// + public required MessageHeader Header { get; set; } + + /// + /// Message payload. + /// + public required byte[] Payload { get; set; } = []; + + /// + /// Headers defined by the user. + /// + public Dictionary? UserHeaders + { + get + { + if (!_userHeadersInitialized) + { + _userHeaders = _rawUserHeaders is { Length: > 0 } + ? BinaryMapper.TryMapHeaders(_rawUserHeaders) + : null; + _userHeadersInitialized = true; + } + + return _userHeaders; + } + set + { + _userHeaders = value; + _userHeadersInitialized = true; + } + } + + /// + /// Raw user header bytes before deserialization. + /// Used internally for decrypting encrypted headers. + /// + [JsonIgnore] + internal byte[]? RawUserHeaders + { + get => _rawUserHeaders; + set + { + _rawUserHeaders = value; + _userHeaders = null; + _userHeadersInitialized = false; + } + } + + internal void ParseUserHeaders() + { + if (!_userHeadersInitialized) + { + _userHeaders = _rawUserHeaders is { Length: > 0 } + ? BinaryMapper.TryMapHeaders(_rawUserHeaders) + : null; + _userHeadersInitialized = true; + } + } +} diff --git a/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs b/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs index 04a82834bf..72bd6f6bfc 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs @@ -1,71 +1,62 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers; - -namespace Apache.Iggy.Contracts; - -/// -/// Represents a rented poll result whose payload and raw header memory remain valid until disposed. -/// -public sealed class PolledMessagesRental : IDisposable -{ - private readonly IMemoryOwner _owner; - private int _disposed; - - /// - /// Partition identifier for the messages. - /// - public required int PartitionId { get; init; } - - /// - /// Current offset for the partition. - /// - public required ulong CurrentOffset { get; init; } - - /// - /// Rented messages. - /// - public required IReadOnlyList Messages { get; init; } - - internal PolledMessagesRental(IMemoryOwner owner) - { - _owner = owner; - } - - /// - /// Disposes the rental and returns the underlying buffer to the pool. - /// - public void Dispose() - { - if (Interlocked.Exchange(ref _disposed, 1) != 0) - { - return; - } - - _owner.Dispose(); - GC.SuppressFinalize(this); - } - - /// - /// Finalizer fallback that returns the buffer to the pool if the caller forgot to dispose. - /// - ~PolledMessagesRental() - { - Dispose(); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers; + +namespace Apache.Iggy.Contracts; + +/// +/// Represents a rented poll result whose payload and raw header memory remain valid until disposed. +/// +public sealed class PolledMessagesRental : IDisposable +{ + private readonly IMemoryOwner _owner; + private int _disposed; + + /// + /// Partition identifier for the messages. + /// + public required int PartitionId { get; init; } + + /// + /// Current offset for the partition. + /// + public required ulong CurrentOffset { get; init; } + + /// + /// Rented messages. + /// + public required IReadOnlyList Messages { get; init; } + + internal PolledMessagesRental(IMemoryOwner owner) + { + _owner = owner; + } + + /// + /// Disposes the rental and returns the underlying buffer to the pool. + /// + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + _owner.Dispose(); + } +} diff --git a/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs index 43c629380e..f4db2413af 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs @@ -1,71 +1,71 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Headers; -using Apache.Iggy.Mappers; -using Apache.Iggy.Messages; - -namespace Apache.Iggy.Contracts; - -/// -/// Response containing rented payload and raw header memory. -/// The payload and raw headers are only valid while the owning is alive. -/// -public sealed class RentedMessageResponse -{ - private Dictionary? _userHeaders; - private bool _userHeadersInitialized; - - /// - /// Message header. - /// - public required MessageHeader Header { get; init; } - - /// - /// Message payload backed by rented memory. - /// - public required ReadOnlyMemory Payload { get; init; } - - /// - /// Raw user header bytes backed by rented memory. - /// - public ReadOnlyMemory RawUserHeaders { get; init; } - - /// - /// Parsed user headers. Parsed lazily and cached on first access. - /// - public Dictionary? UserHeaders - { - get - { - if (!_userHeadersInitialized) - { - _userHeaders = RawUserHeaders.IsEmpty - ? null - : BinaryMapper.TryMapHeaders(RawUserHeaders.Span); - _userHeadersInitialized = true; - } - - return _userHeaders; - } - init - { - _userHeaders = value; - _userHeadersInitialized = true; - } - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Headers; +using Apache.Iggy.Mappers; +using Apache.Iggy.Messages; + +namespace Apache.Iggy.Contracts; + +/// +/// Response containing rented payload and raw header memory. +/// The payload and raw headers are only valid while the owning is alive. +/// +public sealed class RentedMessageResponse +{ + private Dictionary? _userHeaders; + private bool _userHeadersInitialized; + + /// + /// Message header. + /// + public required MessageHeader Header { get; init; } + + /// + /// Message payload backed by rented memory. + /// + public required ReadOnlyMemory Payload { get; init; } + + /// + /// Raw user header bytes backed by rented memory. + /// + public ReadOnlyMemory RawUserHeaders { get; init; } + + /// + /// Parsed user headers. Parsed lazily and cached on first access. + /// + public Dictionary? UserHeaders + { + get + { + if (!_userHeadersInitialized) + { + _userHeaders = RawUserHeaders.IsEmpty + ? null + : BinaryMapper.TryMapHeaders(RawUserHeaders.Span); + _userHeadersInitialized = true; + } + + return _userHeaders; + } + init + { + _userHeaders = value; + _userHeadersInitialized = true; + } + } +} diff --git a/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs b/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs index 6aaf4d8224..8bd8ec2a6f 100644 --- a/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs +++ b/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs @@ -1,129 +1,128 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Security.Cryptography; - -namespace Apache.Iggy.Encryption; - -/// -/// AES-256-GCM based message encryptor for secure message encryption/decryption. -/// Uses AES-GCM (Galois/Counter Mode) which provides both confidentiality and authenticity. -/// -public sealed class AesMessageEncryptor : IMessageEncryptor -{ - private const int NonceSize = 12; // 96 bits - recommended for GCM - private const int TagSize = 16; // 128 bits authentication tag - private readonly byte[] _key; - - /// - /// Creates a new AES message encryptor with the specified key. - /// - /// The encryption key. Must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256 respectively. - /// Thrown when key length is invalid - public AesMessageEncryptor(byte[] key) - { - if (key.Length != 16 && key.Length != 24 && key.Length != 32) - { - throw new ArgumentException("Key must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256", - nameof(key)); - } - - _key = key; - } - - /// - /// Encrypts the provided plain data using AES-GCM. - /// Format: [12-byte nonce][encrypted data][16-byte authentication tag] - /// - /// The plain data to encrypt - /// The encrypted data with nonce and tag - public byte[] Encrypt(Span plainData) - { - using var aesGcm = new AesGcm(_key, TagSize); - - var nonce = new byte[NonceSize]; - RandomNumberGenerator.Fill(nonce); - - var ciphertext = new byte[plainData.Length]; - var tag = new byte[TagSize]; - - aesGcm.Encrypt(nonce, plainData, ciphertext, tag); - - // Combine: nonce + ciphertext + tag - var result = new byte[NonceSize + ciphertext.Length + TagSize]; - Buffer.BlockCopy(nonce, 0, result, 0, NonceSize); - Buffer.BlockCopy(ciphertext, 0, result, NonceSize, ciphertext.Length); - Buffer.BlockCopy(tag, 0, result, NonceSize + ciphertext.Length, TagSize); - - return result; - } - - /// - /// Decrypts the provided encrypted data using AES-GCM. - /// Expected format: [12-byte nonce][encrypted data][16-byte authentication tag] - /// - /// The encrypted data with nonce and tag - /// The decrypted plain data - /// Thrown when encrypted data format is invalid - /// Thrown when decryption or authentication fails - public byte[] Decrypt(ReadOnlySpan encryptedData) - { - if (encryptedData.Length < NonceSize + TagSize) - { - throw new ArgumentException("Encrypted data is too short to contain nonce and tag", nameof(encryptedData)); - } - - using var aesGcm = new AesGcm(_key, TagSize); - - // Extract nonce, ciphertext, and tag - Span nonce = stackalloc byte[NonceSize]; - Span tag = stackalloc byte[TagSize]; - var ciphertextLength = encryptedData.Length - NonceSize - TagSize; - Span ciphertext = stackalloc byte[ciphertextLength]; - - encryptedData[..NonceSize].CopyTo(nonce); - encryptedData.Slice(NonceSize, ciphertextLength).CopyTo(ciphertext); - encryptedData.Slice(NonceSize + ciphertextLength, TagSize).CopyTo(tag); - - var plaintext = new byte[ciphertextLength]; - aesGcm.Decrypt(nonce, ciphertext, tag, plaintext); - - return plaintext; - } - - /// - /// Creates a new AES-256 message encryptor with the specified base64-encoded key. - /// - /// The base64-encoded encryption key - /// A new AesMessageEncryptor instance - public static AesMessageEncryptor FromBase64Key(string base64Key) - { - return new AesMessageEncryptor(Convert.FromBase64String(base64Key)); - } - - /// - /// Generates a new random AES-256 key. - /// - /// A 32-byte random key suitable for AES-256 - public static byte[] GenerateKey() - { - var key = new byte[32]; - using var rng = RandomNumberGenerator.Create(); - rng.GetBytes(key); - return key; - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Security.Cryptography; + +namespace Apache.Iggy.Encryption; + +/// +/// AES-256-GCM based message encryptor for secure message encryption/decryption. +/// Uses AES-GCM (Galois/Counter Mode) which provides both confidentiality and authenticity. +/// +public sealed class AesMessageEncryptor : IMessageEncryptor +{ + private readonly byte[] _key; + private const int NonceSize = 12; // 96 bits - recommended for GCM + private const int TagSize = 16; // 128 bits authentication tag + + /// + /// Creates a new AES message encryptor with the specified key. + /// + /// The encryption key. Must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256 respectively. + /// Thrown when key length is invalid + public AesMessageEncryptor(byte[] key) + { + if (key.Length != 16 && key.Length != 24 && key.Length != 32) + { + throw new ArgumentException("Key must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256", nameof(key)); + } + + _key = key; + } + + /// + /// Creates a new AES-256 message encryptor with the specified base64-encoded key. + /// + /// The base64-encoded encryption key + /// A new AesMessageEncryptor instance + public static AesMessageEncryptor FromBase64Key(string base64Key) + { + return new AesMessageEncryptor(Convert.FromBase64String(base64Key)); + } + + /// + /// Generates a new random AES-256 key. + /// + /// A 32-byte random key suitable for AES-256 + public static byte[] GenerateKey() + { + var key = new byte[32]; + using var rng = RandomNumberGenerator.Create(); + rng.GetBytes(key); + return key; + } + + /// + /// Encrypts the provided plain data using AES-GCM. + /// Format: [12-byte nonce][encrypted data][16-byte authentication tag] + /// + /// The plain data to encrypt + /// The encrypted data with nonce and tag + public byte[] Encrypt(byte[] plainData) + { + using var aesGcm = new AesGcm(_key, TagSize); + + var nonce = new byte[NonceSize]; + RandomNumberGenerator.Fill(nonce); + + var ciphertext = new byte[plainData.Length]; + var tag = new byte[TagSize]; + + aesGcm.Encrypt(nonce, plainData, ciphertext, tag); + + // Combine: nonce + ciphertext + tag + var result = new byte[NonceSize + ciphertext.Length + TagSize]; + Buffer.BlockCopy(nonce, 0, result, 0, NonceSize); + Buffer.BlockCopy(ciphertext, 0, result, NonceSize, ciphertext.Length); + Buffer.BlockCopy(tag, 0, result, NonceSize + ciphertext.Length, TagSize); + + return result; + } + + /// + /// Decrypts the provided encrypted data using AES-GCM. + /// Expected format: [12-byte nonce][encrypted data][16-byte authentication tag] + /// + /// The encrypted data with nonce and tag + /// The decrypted plain data + /// Thrown when encrypted data format is invalid + /// Thrown when decryption or authentication fails + public byte[] Decrypt(byte[] encryptedData) + { + if (encryptedData.Length < NonceSize + TagSize) + { + throw new ArgumentException("Encrypted data is too short to contain nonce and tag", nameof(encryptedData)); + } + + using var aesGcm = new AesGcm(_key, TagSize); + + // Extract nonce, ciphertext, and tag + var nonce = new byte[NonceSize]; + var tag = new byte[TagSize]; + var ciphertextLength = encryptedData.Length - NonceSize - TagSize; + var ciphertext = new byte[ciphertextLength]; + + Buffer.BlockCopy(encryptedData, 0, nonce, 0, NonceSize); + Buffer.BlockCopy(encryptedData, NonceSize, ciphertext, 0, ciphertextLength); + Buffer.BlockCopy(encryptedData, NonceSize + ciphertextLength, tag, 0, TagSize); + + var plaintext = new byte[ciphertextLength]; + aesGcm.Decrypt(nonce, ciphertext, tag, plaintext); + + return plaintext; + } +} diff --git a/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs b/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs index f6111f89d2..fbb99c6262 100644 --- a/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs +++ b/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs @@ -1,40 +1,40 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -namespace Apache.Iggy.Encryption; - -/// -/// Interface for encrypting and decrypting message payloads in Iggy messaging system. -/// Implementations of this interface can be used with IggyPublisher and IggyConsumer -/// to provide end-to-end encryption of message data. -/// -public interface IMessageEncryptor -{ - /// - /// Encrypts the provided plain data. - /// - /// The plain data to encrypt - /// The encrypted data - byte[] Encrypt(Span plainData); - - /// - /// Decrypts the provided encrypted data. - /// - /// The encrypted data to decrypt - /// The decrypted plain data - byte[] Decrypt(ReadOnlySpan encryptedData); -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.Iggy.Encryption; + +/// +/// Interface for encrypting and decrypting message payloads in Iggy messaging system. +/// Implementations of this interface can be used with IggyPublisher and IggyConsumer +/// to provide end-to-end encryption of message data. +/// +public interface IMessageEncryptor +{ + /// + /// Encrypts the provided plain data. + /// + /// The plain data to encrypt + /// The encrypted data + byte[] Encrypt(byte[] plainData); + + /// + /// Decrypts the provided encrypted data. + /// + /// The encrypted data to decrypt + /// The decrypted plain data + byte[] Decrypt(byte[] encryptedData); +} diff --git a/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs b/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs index c585e1fb50..ffa5afc16f 100644 --- a/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs +++ b/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs @@ -1,74 +1,74 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Consumers; -using Apache.Iggy.IggyClient; -using Apache.Iggy.Kinds; -using Apache.Iggy.Publishers; - -namespace Apache.Iggy.Extensions; - -/// -/// Extension methods for -/// -public static class IggyClientExtension -{ - /// - /// Creates a new from for the specified stream and - /// topic. - /// - /// Existing iggy client - /// Stream identifier from which to consume - /// Topic identifier from which to consume - /// Consumer - /// Iggy consumer builder - public static IggyConsumerBuilder CreateConsumerBuilder(this IIggyClient client, Identifier streamId, - Identifier topicId, Consumer consumer) - { - return IggyConsumerBuilder.Create(client, streamId, topicId, consumer); - } - - /// - /// Creates a new from for the specified stream and - /// topic. - /// - /// Existing iggy client - /// Stream identifier from which to consume - /// Topic identifier from which to consume - /// Consumer - /// Optional deserializer - /// Iggy consumer builder - public static IggyConsumerBuilder CreateConsumerBuilder(this IIggyClient client, Identifier streamId, - Identifier topicId, Consumer consumer, IDeserializer deserializer) - { - return IggyConsumerBuilder.Create(client, streamId, topicId, consumer, deserializer); - } - - /// - /// Creates a new from for the specified stream and - /// topic. - /// - /// Existing iggy client - /// Stream identifier to publish to - /// >Topic identifier to publish to - /// Iggy publisher builder - public static IggyPublisherBuilder CreatePublisherBuilder(this IIggyClient client, Identifier streamId, - Identifier topicId) - { - return IggyPublisherBuilder.Create(client, streamId, topicId); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Consumers; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; +using Apache.Iggy.Publishers; + +namespace Apache.Iggy.Extensions; + +/// +/// Extension methods for +/// +public static class IggyClientExtension +{ + /// + /// Creates a new from for the specified stream and + /// topic. + /// + /// Existing iggy client + /// Stream identifier from which to consume + /// Topic identifier from which to consume + /// Consumer + /// Iggy consumer builder + public static IggyConsumerBuilder CreateConsumerBuilder(this IIggyClient client, Identifier streamId, + Identifier topicId, Consumer consumer) + { + return IggyConsumerBuilder.Create(client, streamId, topicId, consumer); + } + + /// + /// Creates a new from for the specified stream and + /// topic. + /// + /// Existing iggy client + /// Stream identifier from which to consume + /// Topic identifier from which to consume + /// Consumer + /// Optional deserializer + /// Iggy consumer builder + public static IggyConsumerBuilder CreateConsumerBuilder(this IIggyClient client, Identifier streamId, + Identifier topicId, Consumer consumer, IDeserializer deserializer) + { + return IggyConsumerBuilder.Create(client, streamId, topicId, consumer, deserializer); + } + + /// + /// Creates a new from for the specified stream and + /// topic. + /// + /// Existing iggy client + /// Stream identifier to publish to + /// >Topic identifier to publish to + /// Iggy publisher builder + public static IggyPublisherBuilder CreatePublisherBuilder(this IIggyClient client, Identifier streamId, + Identifier topicId) + { + return IggyPublisherBuilder.Create(client, streamId, topicId); + } +} diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs index 1898bdbd35..418fd5a93f 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs @@ -1,85 +1,85 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Contracts; -using Apache.Iggy.Kinds; - -namespace Apache.Iggy.IggyClient; - -/// -/// Defines methods for consuming messages from topics in an Iggy client. -/// -public interface IIggyConsumer -{ - /// - /// Polls messages from a specified topic and partition with a given polling strategy. - /// - /// - /// This method retrieves messages from a topic based on the specified consumer and polling strategy. - /// The polling strategy determines where to start reading messages (e.g., from a specific offset, latest, earliest). - /// If a partition ID is not specified, messages can be consumed from any partition. - /// - /// The identifier of the stream containing the topic (numeric ID or name). - /// The identifier of the topic to consume from (numeric ID or name). - /// The specific partition to consume from, or null to consume from any partition. - /// The consumer identifier (group ID or member ID). - /// The strategy for determining where to start reading messages. - /// The maximum number of messages to retrieve. - /// If true, automatically commit the offset after polling. - /// The cancellation token to cancel the operation. - /// A task that represents the asynchronous operation and returns the polled messages. - Task PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId, - Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit, - CancellationToken token = default); - - /// - /// Polls messages from a specified topic and partition while renting the payload buffers from a shared pool - /// instead of copying them into byte arrays. - /// - /// - /// The returned rental must be disposed when the caller is done reading the payload and raw header memory. - /// Payload and raw header slices are invalidated once the rental is disposed. - /// - Task PollMessagesRentedAsync(Identifier streamId, Identifier topicId, uint? partitionId, - Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit, - CancellationToken token = default); - - /// - /// Polls messages from a specified topic using a pre-constructed request. - /// - /// - /// This is a convenience method that wraps the full PollMessagesAsync method using a request object. - /// - /// The message fetch request containing all polling parameters. - /// The cancellation token to cancel the operation. - /// A task that represents the asynchronous operation and returns the polled messages. - Task PollMessagesAsync(MessageFetchRequest request, CancellationToken token = default) - { - return PollMessagesAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer, - request.PollingStrategy, request.Count, request.AutoCommit, token); - } - - /// - /// Polls messages from a specified topic using a pre-constructed request while renting the payload buffers - /// from a shared pool. - /// - Task PollMessagesRentedAsync(MessageFetchRequest request, CancellationToken token = default) - { - return PollMessagesRentedAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer, - request.PollingStrategy, request.Count, request.AutoCommit, token); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Contracts; +using Apache.Iggy.Kinds; + +namespace Apache.Iggy.IggyClient; + +/// +/// Defines methods for consuming messages from topics in an Iggy client. +/// +public interface IIggyConsumer +{ + /// + /// Polls messages from a specified topic and partition with a given polling strategy. + /// + /// + /// This method retrieves messages from a topic based on the specified consumer and polling strategy. + /// The polling strategy determines where to start reading messages (e.g., from a specific offset, latest, earliest). + /// If a partition ID is not specified, messages can be consumed from any partition. + /// + /// The identifier of the stream containing the topic (numeric ID or name). + /// The identifier of the topic to consume from (numeric ID or name). + /// The specific partition to consume from, or null to consume from any partition. + /// The consumer identifier (group ID or member ID). + /// The strategy for determining where to start reading messages. + /// The maximum number of messages to retrieve. + /// If true, automatically commit the offset after polling. + /// The cancellation token to cancel the operation. + /// A task that represents the asynchronous operation and returns the polled messages. + Task PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId, + Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit, + CancellationToken token = default); + + /// + /// Polls messages from a specified topic and partition while renting the payload buffers from a shared pool + /// instead of copying them into byte arrays. + /// + /// + /// The returned rental must be disposed when the caller is done reading the payload and raw header memory. + /// Payload and raw header slices are invalidated once the rental is disposed. + /// + Task PollMessagesRentedAsync(Identifier streamId, Identifier topicId, uint? partitionId, + Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit, + CancellationToken token = default); + + /// + /// Polls messages from a specified topic using a pre-constructed request. + /// + /// + /// This is a convenience method that wraps the full PollMessagesAsync method using a request object. + /// + /// The message fetch request containing all polling parameters. + /// The cancellation token to cancel the operation. + /// A task that represents the asynchronous operation and returns the polled messages. + Task PollMessagesAsync(MessageFetchRequest request, CancellationToken token = default) + { + return PollMessagesAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer, + request.PollingStrategy, request.Count, request.AutoCommit, token); + } + + /// + /// Polls messages from a specified topic using a pre-constructed request while renting the payload buffers + /// from a shared pool. + /// + Task PollMessagesRentedAsync(MessageFetchRequest request, CancellationToken token = default) + { + return PollMessagesRentedAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer, + request.PollingStrategy, request.Count, request.AutoCommit, token); + } +} diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs index e7e7e77a76..9b1ce3905a 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs @@ -1,862 +1,862 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Net; -using System.Net.Http.Headers; -using System.Net.Http.Json; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using Apache.Iggy.Contracts; -using Apache.Iggy.Contracts.Auth; -using Apache.Iggy.Contracts.Http; -using Apache.Iggy.Contracts.Http.Auth; -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Kinds; -using Apache.Iggy.Mappers; -using Apache.Iggy.Messages; -using Apache.Iggy.StringHandlers; -using Apache.Iggy.Utils; -using Partitioning = Apache.Iggy.Kinds.Partitioning; - -namespace Apache.Iggy.IggyClient.Implementations; - -/// -/// Implementation of that uses to communicate with the server. -/// -public class HttpMessageStream : IIggyClient -{ - private const string Context = "csharp-sdk"; - - private readonly HttpClient _httpClient; - - //TODO - create mechanism for refreshing jwt token - //TODO - replace the HttpClient with IHttpClientFactory, when implementing support for ASP.NET Core DI - //TODO - the error handling pattern is pretty ugly, look into moving it into an extension method - private readonly JsonSerializerOptions _jsonSerializerOptions; - - internal HttpMessageStream(HttpClient httpClient) - { - _httpClient = httpClient; - - _jsonSerializerOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, - Converters = { new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) } - }; - } - - /// - public async Task CreateStreamAsync(string name, CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new CreateStreamRequest(name), _jsonSerializerOptions); - - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response = await _httpClient.PostAsync("/streams", data, token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - - return null; - } - - /// - public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default) - { - var response = await _httpClient.DeleteAsync($"/streams/{streamId}/purge", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default) - { - var response = await _httpClient.DeleteAsync($"/streams/{streamId}", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task GetStreamByIdAsync(Identifier streamId, CancellationToken token = default) - { - var response = await _httpClient.GetAsync($"/streams/{streamId}", token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - - return null; - } - - /// - public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new UpdateStreamRequest(name), _jsonSerializerOptions); - - var data = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PutAsync($"/streams/{streamId}", data, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task> GetStreamsAsync(CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/streams", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, - token) - ?? Array.Empty(); - } - - await HandleResponseAsync(response); - return Array.Empty(); - } - - /// - public async Task CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null, - TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new CreateTopicRequest - { - Name = name, - CompressionAlgorithm = compressionAlgorithm, - MaxTopicSize = maxTopicSize, - MessageExpiry = DurationHelpers.ToDuration(messageExpiry), - PartitionsCount = partitionsCount, - ReplicationFactor = replicationFactor - }, _jsonSerializerOptions); - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response = await _httpClient.PostAsync($"/streams/{streamId}/topics", data, token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - - return null; - } - - /// - public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, - ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null, - CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new UpdateTopicRequest(name, compressionAlgorithm, maxTopicSize, - DurationHelpers.ToDuration(messageExpiry), - replicationFactor), - _jsonSerializerOptions); - var data = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}", data, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) - { - var response = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) - { - return _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/purge", token) - .ContinueWith(async response => - { - if (!response.Result.IsSuccessStatusCode) - { - await HandleResponseAsync(response.Result); - } - }, token); - } - - /// - public async Task> GetTopicsAsync(Identifier streamId, - CancellationToken token = default) - { - var response = await _httpClient.GetAsync($"/streams/{streamId}/topics", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, token) - ?? Array.Empty(); - } - - await HandleResponseAsync(response); - return Array.Empty(); - } - - /// - public async Task GetTopicByIdAsync(Identifier streamId, Identifier topicId, - CancellationToken token = default) - { - var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}", token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - - return null; - } - - /// - public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, - IList messages, - CancellationToken token = default) - { - var request = new MessageSendRequest - { - StreamId = streamId, - TopicId = topicId, - Partitioning = partitioning, - Messages = messages - }; - var json = JsonSerializer.Serialize(request, _jsonSerializerOptions); - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response = await _httpClient.PostAsync($"/streams/{request.StreamId}/topics/{request.TopicId}/messages", - data, - token); - - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, - CancellationToken token = default) - { - var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages/flush/{partitionId}/{fsync}"); - - var response = await _httpClient.GetAsync(url, token); - - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response, true); - } - } - - /// - public async Task PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId, - Consumer consumer, - PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) - { - var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty; - var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages?consumer_id={consumer.ConsumerId}" + - $"{partitionIdParam}&kind={pollingStrategy.Kind}&value={pollingStrategy.Value}&count={count}&auto_commit={autoCommit}"); - - var response = await _httpClient.GetAsync(url, token); - if (response.IsSuccessStatusCode) - { - var pollMessages = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token) - ?? PolledMessages.Empty; - - return pollMessages; - } - - await HandleResponseAsync(response, true); - return PolledMessages.Empty; - } - - /// - public async Task PollMessagesRentedAsync(Identifier streamId, Identifier topicId, - uint? partitionId, - Consumer consumer, - PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) - { - var messages = await PollMessagesAsync(streamId, topicId, partitionId, consumer, pollingStrategy, count, - autoCommit, token); - return BinaryMapper.ToRentedMessages(messages); - } - - /// - public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset, - uint? partitionId, CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new StoreOffsetRequest(consumer, partitionId, offset), - _jsonSerializerOptions); - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response - = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}/consumer-offsets", data, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, - uint? partitionId, CancellationToken token = default) - { - var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty; - var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/" + - $"consumer-offsets?consumer_id={consumer.ConsumerId}{partitionIdParam}", - token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId, - CancellationToken token = default) - { - var partitionIdParam = partitionId.HasValue ? $"?partition_id={partitionId.Value}" : string.Empty; - var response = await _httpClient.DeleteAsync( - $"/streams/{streamId}/topics/{topicId}/consumer-offsets/{consumer}{partitionIdParam}", token); - await HandleResponseAsync(response); - } - - /// - public async Task> GetConsumerGroupsAsync(Identifier streamId, - Identifier topicId, CancellationToken token = default) - { - var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync>( - _jsonSerializerOptions, token) - ?? Array.Empty(); - } - - await HandleResponseAsync(response); - return Array.Empty(); - } - - /// - public async Task GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId, - Identifier groupId, CancellationToken token = default) - { - var response - = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task CreateConsumerGroupAsync(Identifier streamId, Identifier topicId, - string name, CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new CreateConsumerGroupRequest(name), _jsonSerializerOptions); - - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response - = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", data, token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, - CancellationToken token = default) - { - var response - = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token); - await HandleResponseAsync(response); - } - - /// - /// This method is only supported in TCP protocol - /// - /// - /// - /// - public Task GetMeAsync(CancellationToken token = default) - { - throw new FeatureUnavailableException(); - } - - /// - public async Task GetStatsAsync(CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/stats", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task GetClusterMetadataAsync(CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/cluster/metadata", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - - return null; - } - - /// - public async Task PingAsync(CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/ping", token); - - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task GetSnapshotAsync(SnapshotCompression compression, - IList snapshotTypes, CancellationToken token = default) - { - // Rust serde uses default derive (PascalCase) for these enums, not snake_case. - // We use .ToString() to produce PascalCase names matching Rust's serde expectations. - var request = new - { - compression = compression.ToString(), - snapshot_types = snapshotTypes.Select(t => t.ToString()).ToList() - }; - var json = JsonSerializer.Serialize(request, _jsonSerializerOptions); - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response = await _httpClient.PostAsync("/snapshot", data, token); - - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadAsByteArrayAsync(token); - } - - await HandleResponseAsync(response); - return []; - } - - /// - public Task ConnectAsync(CancellationToken token = default) - { - return Task.CompletedTask; - } - - /// - public async Task> GetClientsAsync(CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/clients", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, - token) - ?? Array.Empty(); - } - - await HandleResponseAsync(response); - return Array.Empty(); - } - - /// - public async Task GetClientByIdAsync(uint clientId, CancellationToken token = default) - { - var response = await _httpClient.GetAsync($"/clients/{clientId}", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - /// This method is only supported in TCP protocol - /// - /// The identifier of the stream containing the topic (numeric ID or name). - /// The identifier of the topic (numeric ID or name). - /// The identifier of the consumer group to join (numeric ID or name). - /// The cancellation token to cancel the operation. - /// A task representing the asynchronous operation. - /// - [Obsolete("This method is only supported in TCP protocol", true)] - public Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, - CancellationToken token = default) - { - throw new FeatureUnavailableException(); - } - - /// - /// This method is only supported in TCP protocol - /// - /// The identifier of the stream containing the topic (numeric ID or name). - /// The identifier of the topic (numeric ID or name). - /// The identifier of the consumer group to leave (numeric ID or name). - /// The cancellation token to cancel the operation. - /// A task representing the asynchronous operation. - /// - [Obsolete("This method is only supported in TCP protocol", true)] - public Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, - CancellationToken token = default) - { - throw new FeatureUnavailableException(); - } - - /// - public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, - CancellationToken token = default) - { - var response - = await _httpClient.DeleteAsync( - $"/streams/{streamId}/topics/{topicId}/partitions?partitions_count={partitionsCount}", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - /// This method is only supported in TCP protocol - /// - /// The identifier of the stream containing the topic (numeric ID or name). - /// The identifier of the topic containing the partition (numeric ID or name). - /// The unique partition ID. - /// The number of segments to delete. - /// The cancellation token to cancel the operation. - /// A task representing the asynchronous operation. - /// - public Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId, - uint segmentsCount, CancellationToken token = default) - { - throw new FeatureUnavailableException(); - } - - /// - public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, - CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new CreatePartitionsRequest(partitionsCount), _jsonSerializerOptions); - - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/partitions", data, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task GetUserAsync(Identifier userId, CancellationToken token = default) - { - //TODO - this doesn't work prob needs a custom json serializer - var response = await _httpClient.GetAsync($"/users/{userId}", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task> GetUsersAsync(CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/users", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, token) - ?? Array.Empty(); - } - - await HandleResponseAsync(response); - return Array.Empty(); - } - - /// - public async Task CreateUserAsync(string userName, string password, UserStatus status, - Permissions? permissions = null, CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new CreateUserRequest(userName, password, status, permissions), - _jsonSerializerOptions); - - var content = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PostAsync("/users", content, token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default) - { - var response = await _httpClient.DeleteAsync($"/users/{userId}", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null, - CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new UpdateUserRequest(userName, status), _jsonSerializerOptions); - var content = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PutAsync($"/users/{userId}", content, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null, - CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new UpdateUserPermissionsRequest(permissions), _jsonSerializerOptions); - var content = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PutAsync($"/users/{userId}/permissions", content, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword, - CancellationToken token = default) - { - var json = JsonSerializer.Serialize(new ChangePasswordRequest(currentPassword, newPassword), - _jsonSerializerOptions); - var content = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PutAsync($"/users/{userId}/password", content, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task LoginUserAsync(string userName, string password, CancellationToken token = default) - { - // TODO: Add binary protocol version - var json = JsonSerializer.Serialize(new LoginUserRequest(userName, password, SdkVersion.Value, Context), - _jsonSerializerOptions); - - var data = new StringContent(json, Encoding.UTF8, "application/json"); - - var response = await _httpClient.PostAsync("users/login", data, token); - if (response.IsSuccessStatusCode) - { - var authResponse = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - var jwtToken = authResponse!.AccessToken?.Token; - if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token)) - { - _httpClient.DefaultRequestHeaders.Authorization = - new AuthenticationHeaderValue("Bearer", jwtToken); - } - else - { - throw new Exception("The JWT token is missing."); - } - - return authResponse; - } - - await HandleResponseAsync(response); - return null; - } - - /// - public async Task LogoutUserAsync(CancellationToken token = default) - { - var response = await _httpClient.DeleteAsync("users/logout", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - - _httpClient.DefaultRequestHeaders.Authorization = null; - } - - /// - public async Task> GetPersonalAccessTokensAsync( - CancellationToken token = default) - { - var response = await _httpClient.GetAsync("/personal-access-tokens", token); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync>( - _jsonSerializerOptions, token) - ?? Array.Empty(); - } - - await HandleResponseAsync(response); - return Array.Empty(); - } - - /// - public async Task CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null, - CancellationToken token = default) - { - var json = JsonSerializer.Serialize( - new CreatePersonalAccessTokenRequest(name, DurationHelpers.ToDuration(expiry)), _jsonSerializerOptions); - - var content = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PostAsync("/personal-access-tokens", content, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); - } - - /// - public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default) - { - var response = await _httpClient.DeleteAsync($"/personal-access-tokens/{name}", token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } - } - - /// - public async Task LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default) - { - var json = JsonSerializer.Serialize(new LoginWithPersonalAccessTokenRequest(token), _jsonSerializerOptions); - - var content = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PostAsync("/personal-access-tokens/login", content, ct); - if (response.IsSuccessStatusCode) - { - var authResponse = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, ct); - var jwtToken = authResponse!.AccessToken?.Token; - if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token)) - { - _httpClient.DefaultRequestHeaders.Authorization = - new AuthenticationHeaderValue("Bearer", jwtToken); - } - else - { - throw new Exception("The JWT token is missing."); - } - - return authResponse; - } - - await HandleResponseAsync(response); - - return null; - } - - /// - /// Dispose the client. - /// - public void Dispose() - { - } - - /// - public void SubscribeConnectionEvents(Func callback) - { - } - - /// - public void UnsubscribeConnectionEvents(Func callback) - { - } - - /// - public string GetCurrentAddress() - { - return _httpClient.BaseAddress?.ToString() ?? string.Empty; - } - - private static async Task HandleResponseAsync(HttpResponseMessage response, bool shouldThrowOnGetNotFound = false) - { - if ((int)response.StatusCode > 300 - && (int)response.StatusCode < 500 - && !(response.RequestMessage!.Method == HttpMethod.Get && response.StatusCode == HttpStatusCode.NotFound && - !shouldThrowOnGetNotFound)) - { - var err = await response.Content.ReadAsStringAsync(); - var errorModel = JsonSerializer.Deserialize(err); - throw new IggyInvalidStatusCodeException(errorModel?.Id ?? -1, err); - } - - if (response.StatusCode == HttpStatusCode.InternalServerError) - { - throw new Exception("Internal server error"); - } - } - - private static string CreateUrl(ref MessageRequestInterpolationHandler message) - { - return message.ToString(); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Apache.Iggy.Contracts; +using Apache.Iggy.Contracts.Auth; +using Apache.Iggy.Contracts.Http; +using Apache.Iggy.Contracts.Http.Auth; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Kinds; +using Apache.Iggy.Mappers; +using Apache.Iggy.Messages; +using Apache.Iggy.StringHandlers; +using Apache.Iggy.Utils; +using Partitioning = Apache.Iggy.Kinds.Partitioning; + +namespace Apache.Iggy.IggyClient.Implementations; + +/// +/// Implementation of that uses to communicate with the server. +/// +public class HttpMessageStream : IIggyClient +{ + private const string Context = "csharp-sdk"; + + private readonly HttpClient _httpClient; + + //TODO - create mechanism for refreshing jwt token + //TODO - replace the HttpClient with IHttpClientFactory, when implementing support for ASP.NET Core DI + //TODO - the error handling pattern is pretty ugly, look into moving it into an extension method + private readonly JsonSerializerOptions _jsonSerializerOptions; + + internal HttpMessageStream(HttpClient httpClient) + { + _httpClient = httpClient; + + _jsonSerializerOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) } + }; + } + + /// + public async Task CreateStreamAsync(string name, CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new CreateStreamRequest(name), _jsonSerializerOptions); + + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync("/streams", data, token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + + return null; + } + + /// + public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default) + { + var response = await _httpClient.DeleteAsync($"/streams/{streamId}/purge", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default) + { + var response = await _httpClient.DeleteAsync($"/streams/{streamId}", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task GetStreamByIdAsync(Identifier streamId, CancellationToken token = default) + { + var response = await _httpClient.GetAsync($"/streams/{streamId}", token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + + return null; + } + + /// + public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new UpdateStreamRequest(name), _jsonSerializerOptions); + + var data = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PutAsync($"/streams/{streamId}", data, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task> GetStreamsAsync(CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/streams", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, + token) + ?? Array.Empty(); + } + + await HandleResponseAsync(response); + return Array.Empty(); + } + + /// + public async Task CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, + CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null, + TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new CreateTopicRequest + { + Name = name, + CompressionAlgorithm = compressionAlgorithm, + MaxTopicSize = maxTopicSize, + MessageExpiry = DurationHelpers.ToDuration(messageExpiry), + PartitionsCount = partitionsCount, + ReplicationFactor = replicationFactor + }, _jsonSerializerOptions); + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/streams/{streamId}/topics", data, token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + + return null; + } + + /// + public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, + CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, + ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null, + CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new UpdateTopicRequest(name, compressionAlgorithm, maxTopicSize, + DurationHelpers.ToDuration(messageExpiry), + replicationFactor), + _jsonSerializerOptions); + var data = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}", data, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) + { + var response = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) + { + return _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/purge", token) + .ContinueWith(async response => + { + if (!response.Result.IsSuccessStatusCode) + { + await HandleResponseAsync(response.Result); + } + }, token); + } + + /// + public async Task> GetTopicsAsync(Identifier streamId, + CancellationToken token = default) + { + var response = await _httpClient.GetAsync($"/streams/{streamId}/topics", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, token) + ?? Array.Empty(); + } + + await HandleResponseAsync(response); + return Array.Empty(); + } + + /// + public async Task GetTopicByIdAsync(Identifier streamId, Identifier topicId, + CancellationToken token = default) + { + var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}", token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + + return null; + } + + /// + public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, + IList messages, + CancellationToken token = default) + { + var request = new MessageSendRequest + { + StreamId = streamId, + TopicId = topicId, + Partitioning = partitioning, + Messages = messages + }; + var json = JsonSerializer.Serialize(request, _jsonSerializerOptions); + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/streams/{request.StreamId}/topics/{request.TopicId}/messages", + data, + token); + + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, + CancellationToken token = default) + { + var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages/flush/{partitionId}/{fsync}"); + + var response = await _httpClient.GetAsync(url, token); + + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response, true); + } + } + + /// + public async Task PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId, + Consumer consumer, + PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) + { + var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty; + var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages?consumer_id={consumer.ConsumerId}" + + $"{partitionIdParam}&kind={pollingStrategy.Kind}&value={pollingStrategy.Value}&count={count}&auto_commit={autoCommit}"); + + var response = await _httpClient.GetAsync(url, token); + if (response.IsSuccessStatusCode) + { + var pollMessages = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token) + ?? PolledMessages.Empty; + + return pollMessages; + } + + await HandleResponseAsync(response, true); + return PolledMessages.Empty; + } + + /// + public async Task PollMessagesRentedAsync(Identifier streamId, Identifier topicId, + uint? partitionId, + Consumer consumer, + PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) + { + var messages = await PollMessagesAsync(streamId, topicId, partitionId, consumer, pollingStrategy, count, + autoCommit, token); + return BinaryMapper.ToRentedMessages(messages); + } + + /// + public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset, + uint? partitionId, CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new StoreOffsetRequest(consumer, partitionId, offset), + _jsonSerializerOptions); + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response + = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}/consumer-offsets", data, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, + uint? partitionId, CancellationToken token = default) + { + var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty; + var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/" + + $"consumer-offsets?consumer_id={consumer.ConsumerId}{partitionIdParam}", + token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId, + CancellationToken token = default) + { + var partitionIdParam = partitionId.HasValue ? $"?partition_id={partitionId.Value}" : string.Empty; + var response = await _httpClient.DeleteAsync( + $"/streams/{streamId}/topics/{topicId}/consumer-offsets/{consumer}{partitionIdParam}", token); + await HandleResponseAsync(response); + } + + /// + public async Task> GetConsumerGroupsAsync(Identifier streamId, + Identifier topicId, CancellationToken token = default) + { + var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync>( + _jsonSerializerOptions, token) + ?? Array.Empty(); + } + + await HandleResponseAsync(response); + return Array.Empty(); + } + + /// + public async Task GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId, + Identifier groupId, CancellationToken token = default) + { + var response + = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task CreateConsumerGroupAsync(Identifier streamId, Identifier topicId, + string name, CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new CreateConsumerGroupRequest(name), _jsonSerializerOptions); + + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response + = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", data, token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, + CancellationToken token = default) + { + var response + = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token); + await HandleResponseAsync(response); + } + + /// + /// This method is only supported in TCP protocol + /// + /// + /// + /// + public Task GetMeAsync(CancellationToken token = default) + { + throw new FeatureUnavailableException(); + } + + /// + public async Task GetStatsAsync(CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/stats", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task GetClusterMetadataAsync(CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/cluster/metadata", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + + return null; + } + + /// + public async Task PingAsync(CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/ping", token); + + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task GetSnapshotAsync(SnapshotCompression compression, + IList snapshotTypes, CancellationToken token = default) + { + // Rust serde uses default derive (PascalCase) for these enums, not snake_case. + // We use .ToString() to produce PascalCase names matching Rust's serde expectations. + var request = new + { + compression = compression.ToString(), + snapshot_types = snapshotTypes.Select(t => t.ToString()).ToList() + }; + var json = JsonSerializer.Serialize(request, _jsonSerializerOptions); + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync("/snapshot", data, token); + + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadAsByteArrayAsync(token); + } + + await HandleResponseAsync(response); + return []; + } + + /// + public Task ConnectAsync(CancellationToken token = default) + { + return Task.CompletedTask; + } + + /// + public async Task> GetClientsAsync(CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/clients", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, + token) + ?? Array.Empty(); + } + + await HandleResponseAsync(response); + return Array.Empty(); + } + + /// + public async Task GetClientByIdAsync(uint clientId, CancellationToken token = default) + { + var response = await _httpClient.GetAsync($"/clients/{clientId}", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + /// This method is only supported in TCP protocol + /// + /// The identifier of the stream containing the topic (numeric ID or name). + /// The identifier of the topic (numeric ID or name). + /// The identifier of the consumer group to join (numeric ID or name). + /// The cancellation token to cancel the operation. + /// A task representing the asynchronous operation. + /// + [Obsolete("This method is only supported in TCP protocol", true)] + public Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, + CancellationToken token = default) + { + throw new FeatureUnavailableException(); + } + + /// + /// This method is only supported in TCP protocol + /// + /// The identifier of the stream containing the topic (numeric ID or name). + /// The identifier of the topic (numeric ID or name). + /// The identifier of the consumer group to leave (numeric ID or name). + /// The cancellation token to cancel the operation. + /// A task representing the asynchronous operation. + /// + [Obsolete("This method is only supported in TCP protocol", true)] + public Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, + CancellationToken token = default) + { + throw new FeatureUnavailableException(); + } + + /// + public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, + CancellationToken token = default) + { + var response + = await _httpClient.DeleteAsync( + $"/streams/{streamId}/topics/{topicId}/partitions?partitions_count={partitionsCount}", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + /// This method is only supported in TCP protocol + /// + /// The identifier of the stream containing the topic (numeric ID or name). + /// The identifier of the topic containing the partition (numeric ID or name). + /// The unique partition ID. + /// The number of segments to delete. + /// The cancellation token to cancel the operation. + /// A task representing the asynchronous operation. + /// + public Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId, + uint segmentsCount, CancellationToken token = default) + { + throw new FeatureUnavailableException(); + } + + /// + public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, + CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new CreatePartitionsRequest(partitionsCount), _jsonSerializerOptions); + + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/partitions", data, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task GetUserAsync(Identifier userId, CancellationToken token = default) + { + //TODO - this doesn't work prob needs a custom json serializer + var response = await _httpClient.GetAsync($"/users/{userId}", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task> GetUsersAsync(CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/users", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, token) + ?? Array.Empty(); + } + + await HandleResponseAsync(response); + return Array.Empty(); + } + + /// + public async Task CreateUserAsync(string userName, string password, UserStatus status, + Permissions? permissions = null, CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new CreateUserRequest(userName, password, status, permissions), + _jsonSerializerOptions); + + var content = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PostAsync("/users", content, token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default) + { + var response = await _httpClient.DeleteAsync($"/users/{userId}", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null, + CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new UpdateUserRequest(userName, status), _jsonSerializerOptions); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PutAsync($"/users/{userId}", content, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null, + CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new UpdateUserPermissionsRequest(permissions), _jsonSerializerOptions); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PutAsync($"/users/{userId}/permissions", content, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword, + CancellationToken token = default) + { + var json = JsonSerializer.Serialize(new ChangePasswordRequest(currentPassword, newPassword), + _jsonSerializerOptions); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PutAsync($"/users/{userId}/password", content, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task LoginUserAsync(string userName, string password, CancellationToken token = default) + { + // TODO: Add binary protocol version + var json = JsonSerializer.Serialize(new LoginUserRequest(userName, password, SdkVersion.Value, Context), + _jsonSerializerOptions); + + var data = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync("users/login", data, token); + if (response.IsSuccessStatusCode) + { + var authResponse = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + var jwtToken = authResponse!.AccessToken?.Token; + if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token)) + { + _httpClient.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("Bearer", jwtToken); + } + else + { + throw new Exception("The JWT token is missing."); + } + + return authResponse; + } + + await HandleResponseAsync(response); + return null; + } + + /// + public async Task LogoutUserAsync(CancellationToken token = default) + { + var response = await _httpClient.DeleteAsync("users/logout", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + + _httpClient.DefaultRequestHeaders.Authorization = null; + } + + /// + public async Task> GetPersonalAccessTokensAsync( + CancellationToken token = default) + { + var response = await _httpClient.GetAsync("/personal-access-tokens", token); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync>( + _jsonSerializerOptions, token) + ?? Array.Empty(); + } + + await HandleResponseAsync(response); + return Array.Empty(); + } + + /// + public async Task CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null, + CancellationToken token = default) + { + var json = JsonSerializer.Serialize( + new CreatePersonalAccessTokenRequest(name, DurationHelpers.ToDuration(expiry)), _jsonSerializerOptions); + + var content = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PostAsync("/personal-access-tokens", content, token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token); + } + + /// + public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default) + { + var response = await _httpClient.DeleteAsync($"/personal-access-tokens/{name}", token); + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response); + } + } + + /// + public async Task LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default) + { + var json = JsonSerializer.Serialize(new LoginWithPersonalAccessTokenRequest(token), _jsonSerializerOptions); + + var content = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PostAsync("/personal-access-tokens/login", content, ct); + if (response.IsSuccessStatusCode) + { + var authResponse = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, ct); + var jwtToken = authResponse!.AccessToken?.Token; + if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token)) + { + _httpClient.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("Bearer", jwtToken); + } + else + { + throw new Exception("The JWT token is missing."); + } + + return authResponse; + } + + await HandleResponseAsync(response); + + return null; + } + + /// + /// Dispose the client. + /// + public void Dispose() + { + } + + /// + public void SubscribeConnectionEvents(Func callback) + { + } + + /// + public void UnsubscribeConnectionEvents(Func callback) + { + } + + /// + public string GetCurrentAddress() + { + return _httpClient.BaseAddress?.ToString() ?? string.Empty; + } + + private static async Task HandleResponseAsync(HttpResponseMessage response, bool shouldThrowOnGetNotFound = false) + { + if ((int)response.StatusCode > 300 + && (int)response.StatusCode < 500 + && !(response.RequestMessage!.Method == HttpMethod.Get && response.StatusCode == HttpStatusCode.NotFound && + !shouldThrowOnGetNotFound)) + { + var err = await response.Content.ReadAsStringAsync(); + var errorModel = JsonSerializer.Deserialize(err); + throw new IggyInvalidStatusCodeException(errorModel?.Id ?? -1, err); + } + + if (response.StatusCode == HttpStatusCode.InternalServerError) + { + throw new Exception("Internal server error"); + } + } + + private static string CreateUrl(ref MessageRequestInterpolationHandler message) + { + return message.ToString(); + } +} diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs index 72d0d9b511..a804561aae 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs @@ -1,1317 +1,1338 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers; -using System.Buffers.Binary; -using System.Net.Security; -using System.Net.Sockets; -using System.Security.Cryptography.X509Certificates; -using System.Text; -using Apache.Iggy.Configuration; -using Apache.Iggy.ConnectionStream; -using Apache.Iggy.Contracts; -using Apache.Iggy.Contracts.Auth; -using Apache.Iggy.Contracts.Tcp; -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Kinds; -using Apache.Iggy.Mappers; -using Apache.Iggy.Messages; -using Apache.Iggy.Utils; -using Microsoft.Extensions.Logging; -using Partitioning = Apache.Iggy.Kinds.Partitioning; - -namespace Apache.Iggy.IggyClient.Implementations; - -/// -/// A TCP client for interacting with the Iggy server. -/// -public sealed class TcpMessageStream : IIggyClient -{ - private readonly IggyClientConfigurator _configuration; - private readonly EventAggregator _connectionEvents; - private readonly SemaphoreSlim _connectionSemaphore; - private readonly ILogger _logger; - private readonly byte[] _responseHeaderBuffer = new byte[BufferSizes.EXPECTED_RESPONSE_SIZE]; - private readonly SemaphoreSlim _sendingSemaphore; - private string _currentAddress = string.Empty; - private X509Certificate2Collection _customCaStore = []; - private bool _isConnecting; - private DateTimeOffset _lastConnectionTime; - private ConnectionState _state = ConnectionState.Disconnected; - private TcpConnectionStream _stream = null!; - - internal TcpMessageStream(IggyClientConfigurator configuration, ILoggerFactory loggerFactory) - { - _configuration = configuration; - _logger = loggerFactory.CreateLogger(); - _sendingSemaphore = new SemaphoreSlim(1, 1); - _connectionSemaphore = new SemaphoreSlim(1, 1); - _lastConnectionTime = DateTimeOffset.MinValue; - _connectionEvents = new EventAggregator(loggerFactory); - } - - /// - /// Fired whenever the connection state changes. - /// - //public event EventHandler? OnConnectionStateChanged; - public void Dispose() - { - _stream?.Close(); - _stream?.Dispose(); - _sendingSemaphore.Dispose(); - _connectionSemaphore.Dispose(); - _connectionEvents.Clear(); - } - - /// - public void SubscribeConnectionEvents(Func callback) - { - _connectionEvents.Subscribe(callback); - } - - /// - public void UnsubscribeConnectionEvents(Func callback) - { - _connectionEvents.Unsubscribe(callback); - } - - /// - public string GetCurrentAddress() - { - return _currentAddress; - } - - /// - public async Task CreateStreamAsync(string name, CancellationToken token = default) - { - var message = TcpContracts.CreateStream(name); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_STREAM_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - throw new InvalidResponseException("Received empty response while trying to create stream."); - } - - return BinaryMapper.MapStream(responseBuffer.Memory.Span); - } - - /// - public async Task GetStreamByIdAsync(Identifier streamId, CancellationToken token = default) - { - var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAM_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - return BinaryMapper.MapStream(responseBuffer.Memory.Span); - } - - /// - public async Task> GetStreamsAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAMS_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return []; - } - - return BinaryMapper.MapStreams(responseBuffer.Memory.Span); - } - - /// - public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default) - { - var message = TcpContracts.UpdateStream(streamId, name); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_STREAM_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default) - { - var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_STREAM_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default) - { - var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_STREAM_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task> GetTopicsAsync(Identifier streamId, - CancellationToken token = default) - { - var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPICS_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return []; - } - - return BinaryMapper.MapTopics(responseBuffer.Memory.Span); - } - - /// - public async Task GetTopicByIdAsync(Identifier streamId, Identifier topicId, - CancellationToken token = default) - { - var message = TcpContracts.GetTopicById(streamId, topicId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPIC_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - return BinaryMapper.MapTopic(responseBuffer.Memory.Span); - } - - /// - public async Task CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null, - TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default) - { - var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry); - var message = TcpContracts.CreateTopic(streamId, name, partitionsCount, compressionAlgorithm, - replicationFactor, messageExpiryValue, maxTopicSize); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_TOPIC_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - return BinaryMapper.MapTopic(responseBuffer.Memory.Span); - } - - /// - public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, - ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null, - CancellationToken token = default) - { - var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry); - var message = TcpContracts.UpdateTopic(streamId, topicId, name, compressionAlgorithm, maxTopicSize, - messageExpiryValue, replicationFactor); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_TOPIC_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) - { - var message = TcpContracts.DeleteTopic(streamId, topicId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_TOPIC_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) - { - var message = TcpContracts.PurgeTopic(streamId, topicId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_TOPIC_CODE); - - await SendAckAsync(payload, token); - } - - - /// - public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, - IList messages, CancellationToken token = default) - { - var metadataLength = 2 + streamId.Length + 2 + topicId.Length - + 2 + partitioning.Length + 4 + 4; - var messageBufferSize = TcpMessageStreamHelpers.CalculateMessageBytesCount(messages) - + metadataLength; - var payloadBufferSize = messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH; - - IMemoryOwner messageBuffer = MemoryPool.Shared.Rent(messageBufferSize); - IMemoryOwner payloadBuffer = MemoryPool.Shared.Rent(payloadBufferSize); - try - { - TcpContracts.CreateMessage(messageBuffer.Memory.Span[..messageBufferSize], streamId, - topicId, partitioning, messages); - - TcpMessageStreamHelpers.CreatePayload(payloadBuffer.Memory.Span[..payloadBufferSize], - messageBuffer.Memory.Span[..messageBufferSize], CommandCodes.SEND_MESSAGES_CODE); - - await SendAckAsync(payloadBuffer.Memory[..payloadBufferSize], token); - } - finally - { - messageBuffer.Dispose(); - payloadBuffer.Dispose(); - } - } - - /// - public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, - CancellationToken token = default) - { - var message = TcpContracts.FlushUnsavedBuffer(streamId, topicId, partitionId, fsync); - - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.FLUSH_UNSAVED_BUFFER_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId, - Consumer consumer, - PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) - { - using var rental = await PollMessagesRentedAsync(streamId, topicId, partitionId, consumer, pollingStrategy, - count, autoCommit, token); - return BinaryMapper.MaterializeMessages(rental); - } - - /// - public async Task PollMessagesRentedAsync(Identifier streamId, Identifier topicId, - uint? partitionId, - Consumer consumer, - PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) - { - var messageBufferSize = CalculateMessageBufferSize(streamId, topicId, consumer); - var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize); - var payload = ArrayPool.Shared.Rent(payloadBufferSize); - - try - { - TcpContracts.GetMessages(payload.AsSpan().Slice(8, messageBufferSize), consumer, streamId, - topicId, pollingStrategy, count, autoCommit, partitionId); - BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[..4], messageBufferSize + 4); - BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[4..8], CommandCodes.POLL_MESSAGES_CODE); - - IMemoryOwner responseBuffer - = await SendWithResponseAsync(payload.AsMemory(0, payloadBufferSize), token); - return BinaryMapper.MapRentedMessages(responseBuffer); - } - finally - { - ArrayPool.Shared.Return(payload); - } - } - - /// - public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset, - uint? partitionId, CancellationToken token = default) - { - var message = TcpContracts.UpdateOffset(streamId, topicId, consumer, offset, partitionId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.STORE_CONSUMER_OFFSET_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, - uint? partitionId, CancellationToken token = default) - { - var message = TcpContracts.GetOffset(streamId, topicId, consumer, partitionId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_OFFSET_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - return BinaryMapper.MapOffsets(responseBuffer.Memory.Span); - } - - /// - public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId, - CancellationToken token = default) - { - var message = TcpContracts.DeleteOffset(streamId, topicId, consumer, partitionId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_OFFSET_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task> GetConsumerGroupsAsync(Identifier streamId, - Identifier topicId, - CancellationToken token = default) - { - var message = TcpContracts.GetGroups(streamId, topicId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUPS_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return []; - } - - return BinaryMapper.MapConsumerGroups(responseBuffer.Memory.Span); - } - - /// - public async Task GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId, - Identifier groupId, CancellationToken token = default) - { - var message = TcpContracts.GetGroup(streamId, topicId, groupId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUP_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - return BinaryMapper.MapConsumerGroup(responseBuffer.Memory.Span); - } - - /// - public async Task CreateConsumerGroupAsync(Identifier streamId, Identifier topicId, - string name, CancellationToken token = default) - { - var message = TcpContracts.CreateGroup(streamId, topicId, name); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_CONSUMER_GROUP_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - return BinaryMapper.MapConsumerGroup(responseBuffer.Memory.Span); - } - - /// - public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, - CancellationToken token = default) - { - var message = TcpContracts.DeleteGroup(streamId, topicId, groupId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_GROUP_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, - CancellationToken token = default) - { - var message = TcpContracts.JoinGroup(streamId, topicId, groupId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.JOIN_CONSUMER_GROUP_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, - CancellationToken token = default) - { - var message = TcpContracts.LeaveGroup(streamId, topicId, groupId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LEAVE_CONSUMER_GROUP_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, - CancellationToken token = default) - { - var message = TcpContracts.DeletePartitions(streamId, topicId, partitionsCount); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PARTITIONS_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, - CancellationToken token = default) - { - var message = TcpContracts.CreatePartitions(streamId, topicId, partitionsCount); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PARTITIONS_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId, - uint segmentsCount, CancellationToken token = default) - { - var message = TcpContracts.DeleteSegments(streamId, topicId, partitionId, segmentsCount); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_SEGMENTS_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task GetMeAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_ME_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - return BinaryMapper.MapClient(responseBuffer.Memory.Span); - } - - /// - public async Task GetStatsAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STATS_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - return BinaryMapper.MapStats(responseBuffer.Memory.Span); - } - - /// - public async Task GetClusterMetadataAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLUSTER_METADATA_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - return BinaryMapper.MapClusterMetadata(responseBuffer.Memory.Span); - } - - /// - public async Task PingAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PING_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task GetSnapshotAsync(SnapshotCompression compression, - IList snapshotTypes, CancellationToken token = default) - { - var message = TcpContracts.GetSnapshot(compression, snapshotTypes); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_SNAPSHOT_CODE); - - using IMemoryOwner result = await SendWithResponseAsync(payload, token); - - return result.Memory.Span.ToArray(); - } - - /// - public async Task ConnectAsync(CancellationToken token = default) - { - if (_state is ConnectionState.Connected - or ConnectionState.Authenticating - or ConnectionState.Authenticated) - { - _logger.LogWarning("Connection is already connected"); - return; - } - - if (_lastConnectionTime != DateTimeOffset.MinValue) - { - await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token); - } - - SetConnectionStateAsync(ConnectionState.Connecting); - _isConnecting = true; - try - { - await TryEstablishConnectionAsync(token); - } - finally - { - _isConnecting = false; - } - } - - /// - public async Task> GetClientsAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENTS_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return []; - } - - return BinaryMapper.MapClients(responseBuffer.Memory.Span); - } - - /// - public async Task GetClientByIdAsync(uint clientId, CancellationToken token = default) - { - var message = TcpContracts.GetClient(clientId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENT_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - return BinaryMapper.MapClient(responseBuffer.Memory.Span); - } - - /// - public async Task GetUserAsync(Identifier userId, CancellationToken token = default) - { - var message = TcpContracts.GetUser(userId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USER_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - return BinaryMapper.MapUser(responseBuffer.Memory.Span); - } - - /// - public async Task> GetUsersAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USERS_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return []; - } - - return BinaryMapper.MapUsers(responseBuffer.Memory.Span); - } - - /// - public async Task CreateUserAsync(string userName, string password, UserStatus status, - Permissions? permissions = null, CancellationToken token = default) - { - var message = TcpContracts.CreateUser(userName, password, status, permissions); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_USER_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - return BinaryMapper.MapUser(responseBuffer.Memory.Span); - } - - /// - public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default) - { - var message = TcpContracts.DeleteUser(userId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_USER_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null, - CancellationToken token = default) - { - var message = TcpContracts.UpdateUser(userId, userName, status); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_USER_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null, - CancellationToken token = default) - { - var message = TcpContracts.UpdatePermissions(userId, permissions); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_PERMISSIONS_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword, - CancellationToken token = default) - { - var message = TcpContracts.ChangePassword(userId, currentPassword, newPassword); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CHANGE_PASSWORD_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task LoginUserAsync(string userName, string password, CancellationToken token = default) - { - if (_state == ConnectionState.Disconnected) - { - throw new NotConnectedException(); - } - - // TODO: Add binary protocol version - var message = TcpContracts.LoginUser(userName, password, SdkVersion.Value, "csharp-sdk"); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_USER_CODE); - - SetConnectionStateAsync(ConnectionState.Authenticating); - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..responseBuffer.Memory.Length]); - SetConnectionStateAsync(ConnectionState.Authenticated); - - if (await RedirectAsync(token)) - { - await ConnectAsync(token); - return await LoginUserAsync(userName, password, token); - } - - var authResponse = new AuthResponse(userId, null); - return authResponse; - } - - /// - public async Task LogoutUserAsync(CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGOUT_USER_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task> GetPersonalAccessTokensAsync( - CancellationToken token = default) - { - var message = Array.Empty(); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_PERSONAL_ACCESS_TOKENS_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return []; - } - - return BinaryMapper.MapPersonalAccessTokens(responseBuffer.Memory.Span); - } - - /// - public async Task CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null, - CancellationToken token = default) - { - var message = TcpContracts.CreatePersonalAccessToken(name, DurationHelpers.ToDuration(expiry)); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - return BinaryMapper.MapRawPersonalAccessToken(responseBuffer.Memory.Span); - } - - /// - public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default) - { - var message = TcpContracts.DeletePersonalRequestToken(name); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PERSONAL_ACCESS_TOKEN_CODE); - - await SendAckAsync(payload, token); - } - - /// - public async Task LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default) - { - var message = TcpContracts.LoginWithPersonalAccessToken(token); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE); - - SetConnectionStateAsync(ConnectionState.Authenticating); - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, ct); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..4]); - - SetConnectionStateAsync(ConnectionState.Authenticated); - - if (await RedirectAsync(ct)) - { - await ConnectAsync(ct); - return await LoginWithPersonalAccessTokenAsync(token, ct); - } - - return new AuthResponse(userId, null); - } - - private async Task TryEstablishConnectionAsync(CancellationToken token) - { - var retryCount = 0; - var delay = _configuration.ReconnectionSettings.InitialDelay; - do - { - // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract - _stream?.Close(); - _stream?.Dispose(); - - if (string.IsNullOrEmpty(_currentAddress)) - { - _currentAddress = _configuration.BaseAddress; - } - - var urlPortSplitter = _currentAddress.Split(":"); - if (urlPortSplitter.Length > 2) - { - throw new InvalidBaseAddressException(); - } - - try - { - var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - socket.SendBufferSize = _configuration.SendBufferSize; - socket.ReceiveBufferSize = _configuration.ReceiveBufferSize; - - await socket.ConnectAsync(urlPortSplitter[0], int.Parse(urlPortSplitter[1]), token); - - socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true); - socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 5); - - SetConnectionStateAsync(ConnectionState.Connected); - _lastConnectionTime = DateTimeOffset.UtcNow; - - _stream = _configuration.TlsSettings.Enabled switch - { - true => await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings), - false => new TcpConnectionStream(new NetworkStream(socket, true)) - }; - - if (_configuration.AutoLoginSettings.Enabled) - { - _logger.LogInformation("Auto login enabled. Trying to login with credentials: {Username}", - _configuration.AutoLoginSettings.Username); - await LoginUserAsync(_configuration.AutoLoginSettings.Username, - _configuration.AutoLoginSettings.Password, token); - } - - break; - } - catch (Exception e) - { - _logger.LogError(e, "Failed to connect"); - - if (!_configuration.ReconnectionSettings.Enabled || - (_configuration.ReconnectionSettings.MaxRetries > 0 && - retryCount >= _configuration.ReconnectionSettings.MaxRetries)) - { - SetConnectionStateAsync(ConnectionState.Disconnected); - throw; - } - - retryCount++; - if (_configuration.ReconnectionSettings.UseExponentialBackoff) - { - delay *= _configuration.ReconnectionSettings.BackoffMultiplier; - - if (delay > _configuration.ReconnectionSettings.MaxDelay) - { - delay = _configuration.ReconnectionSettings.MaxDelay; - } - } - - if (_logger.IsEnabled(LogLevel.Information)) - { - _logger.LogInformation("Retrying connection attempt {RetryCount} with delay {Delay}", retryCount, - delay); - } - - await Task.Delay(delay, token); - } - } while (true); - } - - private async Task GetCurrentLeaderNodeAsync(CancellationToken token) - { - try - { - var clusterMetadata = await GetClusterMetadataAsync(token); - if (clusterMetadata == null) - { - return null; - } - - // Single-node cluster (clustering disabled) - no redirection needed - if (clusterMetadata.Nodes.Count() == 1) - { - return null; - } - - var leaderNode = clusterMetadata.Nodes.FirstOrDefault(x => x.Role == ClusterNodeRole.Leader); - if (leaderNode == null) - { - throw new MissingLeaderException(); - } - - return leaderNode; - } - // todo: change after error refactoring, error code 5 is for feature not supported - catch (IggyInvalidStatusCodeException e) when (e.StatusCode == 5) - { - return null; - } - } - - private async Task CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings) - { - ValidateCertificatePath(tlsSettings.CertificatePath); - - _customCaStore = new X509Certificate2Collection(); - _customCaStore.ImportFromPemFile(tlsSettings.CertificatePath); - var stream = new NetworkStream(socket, true); - var sslStream = new SslStream(stream, false, RemoteCertificateValidationCallback); - - await sslStream.AuthenticateAsClientAsync(tlsSettings.Hostname); - - return new TcpConnectionStream(sslStream); - } - - private async Task SendAckAsync(ReadOnlyMemory payload, CancellationToken token = default) - { - using IMemoryOwner _ = await SendWithResponseAsync(payload, token); - } - - private async Task> SendWithResponseAsync(ReadOnlyMemory payload, - CancellationToken token = default) - { - try - { - return await SendRawAsync(payload, token); - } - catch (Exception e) when (IsConnectionException(e) && !_isConnecting) - { - _logger.LogWarning("Connection lost"); - if (!_configuration.ReconnectionSettings.Enabled) - { - _logger.LogWarning("Reconnection is disabled"); - SetConnectionStateAsync(ConnectionState.Disconnected); - throw; - } - - return await HandleReconnectionAsync(payload, token); - } - } - - private async Task> HandleReconnectionAsync(ReadOnlyMemory payload, - CancellationToken token) - { - var currentTime = DateTimeOffset.UtcNow; - await _connectionSemaphore.WaitAsync(token); - - try - { - if (_state is ConnectionState.Connected or ConnectionState.Authenticated - && _lastConnectionTime > currentTime) - { - _logger.LogInformation("Connection already established, sending payload"); - return await SendRawAsync(payload, token); - } - - SetConnectionStateAsync(ConnectionState.Disconnected); - _logger.LogInformation("Reconnecting to the server"); - await ConnectAsync(token); - - _logger.LogInformation("Reconnected to the server"); - - await Task.Delay(_configuration.ReconnectionSettings.WaitAfterReconnect, token); - - return await SendRawAsync(payload, token); - } - finally - { - _connectionSemaphore.Release(); - } - } - - private async Task> SendRawAsync(ReadOnlyMemory payload, CancellationToken token) - { - if (_state is ConnectionState.Disconnected or ConnectionState.Connecting) - { - throw new NotConnectedException(); - } - - await _sendingSemaphore.WaitAsync(token); - - try - { - await _stream.SendAsync(payload, token); - await _stream.FlushAsync(token); - - // Read the 8-byte header (4 bytes status + 4 bytes length) - var totalRead = 0; - while (totalRead < BufferSizes.EXPECTED_RESPONSE_SIZE) - { - var readBytes - = await _stream.ReadAsync( - _responseHeaderBuffer.AsMemory(totalRead, BufferSizes.EXPECTED_RESPONSE_SIZE - totalRead), - token); - if (readBytes == 0) - { - throw new IggyZeroBytesException(); - } - - totalRead += readBytes; - } - - var response = TcpMessageStreamHelpers.GetResponseLengthAndStatus(_responseHeaderBuffer); - - if (response.Status != 0) - { - if (response.Length == 0) - { - throw new IggyInvalidStatusCodeException(response.Status, - $"Invalid response status code: {response.Status}"); - } - - var errorBuffer = new byte[response.Length]; - totalRead = 0; - while (totalRead < response.Length) - { - var readBytes - = await _stream.ReadAsync(errorBuffer.AsMemory(totalRead, response.Length - totalRead), token); - if (readBytes == 0) - { - throw new IggyZeroBytesException(); - } - - totalRead += readBytes; - } - - throw new InvalidResponseException(Encoding.UTF8.GetString(errorBuffer)); - } - - if (response.Length == 0) - { - return EmptyMemoryOwner.Instance; - } - - var responseBuffer = ArrayPoolHelper.Rent(response.Length); - try - { - totalRead = 0; - while (totalRead < response.Length) - { - var readBytes - = await _stream.ReadAsync(responseBuffer.Memory.Slice(totalRead, response.Length - totalRead), - token); - - if (readBytes == 0) - { - throw new IggyZeroBytesException(); - } - - totalRead += readBytes; - } - } - catch - { - responseBuffer.Dispose(); - throw; - } - - return responseBuffer; - } - finally - { - _sendingSemaphore.Release(); - } - } - - private static bool IsConnectionException(Exception ex) - { - return ex is IggyZeroBytesException or - NotConnectedException or - SocketException or - IOException or - ObjectDisposedException; - } - - private static int CalculatePayloadBufferSize(int messageBufferSize) - { - return messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH; - } - - private static int CalculateMessageBufferSize(Identifier streamId, Identifier topicId, Consumer consumer) - { - // Original: 14 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.Id.Length - // Added 1 byte for partition flag - return 15 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.ConsumerId.Length; - } - - /// - /// Sets the connection state and fires the OnConnectionStateChanged event. - /// - /// The new connection state - private void SetConnectionStateAsync(ConnectionState newState) - { - if (_state == newState) - { - return; - } - - var previousState = _state; - _state = newState; - - _logger.LogInformation("Connection state changed: {PreviousState} -> {CurrentState}", previousState, newState); - _connectionEvents.Publish(new ConnectionStateChangedEventArgs(previousState, newState)); - } - - private void ValidateCertificatePath(string tlsCertificatePath) - { - if (string.IsNullOrEmpty(tlsCertificatePath) - || !File.Exists(tlsCertificatePath)) - { - throw new InvalidCertificatePathException(tlsCertificatePath); - } - } - - private bool RemoteCertificateValidationCallback(object sender, X509Certificate? certificate, X509Chain? chain, - SslPolicyErrors sslPolicyErrors) - { - if (sslPolicyErrors == SslPolicyErrors.None) - { - return true; - } - - if (certificate is null) - { - return false; - } - - if (certificate is not X509Certificate2 serverCert) - { - serverCert = new X509Certificate2(certificate); - } - - if (_customCaStore.Any(ca => ca.Thumbprint == serverCert.Thumbprint)) - { - if (DateTime.UtcNow <= serverCert.NotAfter && DateTime.UtcNow >= serverCert.NotBefore) - { - return true; - } - - _logger.LogError( - "Server certificate matches trusted key but is expired. Valid from {NotBefore} to {NotAfter}", - serverCert.NotBefore, serverCert.NotAfter); - return false; - } - - - using var customChain = new X509Chain(); - customChain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; - customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; - foreach (var ca in _customCaStore) - { - customChain.ChainPolicy.CustomTrustStore.Add(ca); - customChain.ChainPolicy.ExtraStore.Add(ca); - } - - customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; - - if (customChain.Build(new X509Certificate2(certificate))) - { - if (!sslPolicyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch)) - { - return true; - } - - _logger.LogError("Custom CA chain is valid, but hostname does not match"); - return false; - } - - foreach (var chainStatus in customChain.ChainStatus) - { - _logger.LogWarning("Certificate validation failed: {ChainStatus} - {StatusInformation}", chainStatus.Status, - chainStatus.StatusInformation); - } - - return false; - } - - private async Task RedirectAsync(CancellationToken token) - { - var currentLeaderNode = await GetCurrentLeaderNodeAsync(token); - if (currentLeaderNode == null) - { - return false; - } - - var leaderAddress = $"{currentLeaderNode.Ip}:{currentLeaderNode.Endpoints.Tcp}"; - if (leaderAddress == _currentAddress) - { - return false; - } - - _currentAddress = leaderAddress; - - _logger.LogInformation("Leader address changed. Trying to reconnect to {Address}", - leaderAddress); - - _stream.Close(); - SetConnectionStateAsync(ConnectionState.Disconnected); - return true; - } - - internal sealed class EmptyMemoryOwner : IMemoryOwner - { - public static readonly EmptyMemoryOwner Instance = new(); - - private EmptyMemoryOwner() - { - } - - public Memory Memory => Memory.Empty; - - public void Dispose() - { - } - } -} - -internal static class ArrayPoolHelper -{ - public static SlicedMemoryOwner Rent(int minimumLength) - { - return new SlicedMemoryOwner(minimumLength); - } - - internal sealed class SlicedMemoryOwner(int minimumLength) : IMemoryOwner - { - private readonly byte[] _value = ArrayPool.Shared.Rent(minimumLength); - private int _disposed; - - public void Dispose() - { - if (Interlocked.Exchange(ref _disposed, 1) != 0) - { - return; - } - - ArrayPool.Shared.Return(_value); - } - - public Memory Memory => _value.AsMemory()[..minimumLength]; - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers; +using System.Buffers.Binary; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using Apache.Iggy.Configuration; +using Apache.Iggy.ConnectionStream; +using Apache.Iggy.Contracts; +using Apache.Iggy.Contracts.Auth; +using Apache.Iggy.Contracts.Tcp; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Kinds; +using Apache.Iggy.Mappers; +using Apache.Iggy.Messages; +using Apache.Iggy.Utils; +using Microsoft.Extensions.Logging; +using Partitioning = Apache.Iggy.Kinds.Partitioning; + +namespace Apache.Iggy.IggyClient.Implementations; + +/// +/// A TCP client for interacting with the Iggy server. +/// +public sealed class TcpMessageStream : IIggyClient +{ + private readonly IggyClientConfigurator _configuration; + private readonly EventAggregator _connectionEvents; + private readonly SemaphoreSlim _connectionSemaphore; + private readonly ILogger _logger; + private readonly byte[] _responseHeaderBuffer = new byte[BufferSizes.EXPECTED_RESPONSE_SIZE]; + private readonly SemaphoreSlim _sendingSemaphore; + private string _currentAddress = string.Empty; + private X509Certificate2Collection _customCaStore = []; + private bool _isConnecting; + private DateTimeOffset _lastConnectionTime; + private ConnectionState _state = ConnectionState.Disconnected; + private TcpConnectionStream _stream = null!; + + internal TcpMessageStream(IggyClientConfigurator configuration, ILoggerFactory loggerFactory) + { + _configuration = configuration; + _logger = loggerFactory.CreateLogger(); + _sendingSemaphore = new SemaphoreSlim(1, 1); + _connectionSemaphore = new SemaphoreSlim(1, 1); + _lastConnectionTime = DateTimeOffset.MinValue; + _connectionEvents = new EventAggregator(loggerFactory); + } + + /// + /// Fired whenever the connection state changes. + /// + //public event EventHandler? OnConnectionStateChanged; + public void Dispose() + { + _stream?.Close(); + _stream?.Dispose(); + _sendingSemaphore.Dispose(); + _connectionSemaphore.Dispose(); + _connectionEvents.Clear(); + } + + /// + public void SubscribeConnectionEvents(Func callback) + { + _connectionEvents.Subscribe(callback); + } + + /// + public void UnsubscribeConnectionEvents(Func callback) + { + _connectionEvents.Unsubscribe(callback); + } + + /// + public string GetCurrentAddress() + { + return _currentAddress; + } + + /// + public async Task CreateStreamAsync(string name, CancellationToken token = default) + { + var message = TcpContracts.CreateStream(name); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_STREAM_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + throw new InvalidResponseException("Received empty response while trying to create stream."); + } + + return BinaryMapper.MapStream(responseBuffer.Memory.Span); + } + + /// + public async Task GetStreamByIdAsync(Identifier streamId, CancellationToken token = default) + { + var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAM_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapStream(responseBuffer.Memory.Span); + } + + /// + public async Task> GetStreamsAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAMS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return []; + } + + return BinaryMapper.MapStreams(responseBuffer.Memory.Span); + } + + /// + public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default) + { + var message = TcpContracts.UpdateStream(streamId, name); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_STREAM_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default) + { + var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_STREAM_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default) + { + var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_STREAM_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task> GetTopicsAsync(Identifier streamId, + CancellationToken token = default) + { + var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPICS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return []; + } + + return BinaryMapper.MapTopics(responseBuffer.Memory.Span); + } + + /// + public async Task GetTopicByIdAsync(Identifier streamId, Identifier topicId, + CancellationToken token = default) + { + var message = TcpContracts.GetTopicById(streamId, topicId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPIC_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapTopic(responseBuffer.Memory.Span); + } + + /// + public async Task CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, + CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null, + TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default) + { + var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry); + var message = TcpContracts.CreateTopic(streamId, name, partitionsCount, compressionAlgorithm, + replicationFactor, messageExpiryValue, maxTopicSize); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_TOPIC_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapTopic(responseBuffer.Memory.Span); + } + + /// + public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, + CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, + ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null, + CancellationToken token = default) + { + var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry); + var message = TcpContracts.UpdateTopic(streamId, topicId, name, compressionAlgorithm, maxTopicSize, + messageExpiryValue, replicationFactor); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_TOPIC_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) + { + var message = TcpContracts.DeleteTopic(streamId, topicId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_TOPIC_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default) + { + var message = TcpContracts.PurgeTopic(streamId, topicId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_TOPIC_CODE); + + await SendAckAsync(payload, token); + } + + + /// + public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, + IList messages, CancellationToken token = default) + { + var metadataLength = 2 + streamId.Length + 2 + topicId.Length + + 2 + partitioning.Length + 4 + 4; + var messageBufferSize = TcpMessageStreamHelpers.CalculateMessageBytesCount(messages) + + metadataLength; + var payloadBufferSize = messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH; + + IMemoryOwner messageBuffer = MemoryPool.Shared.Rent(messageBufferSize); + IMemoryOwner payloadBuffer = MemoryPool.Shared.Rent(payloadBufferSize); + try + { + TcpContracts.CreateMessage(messageBuffer.Memory.Span[..messageBufferSize], streamId, + topicId, partitioning, messages); + + TcpMessageStreamHelpers.CreatePayload(payloadBuffer.Memory.Span[..payloadBufferSize], + messageBuffer.Memory.Span[..messageBufferSize], CommandCodes.SEND_MESSAGES_CODE); + + await SendAckAsync(payloadBuffer.Memory[..payloadBufferSize], token); + } + finally + { + messageBuffer.Dispose(); + payloadBuffer.Dispose(); + } + } + + /// + public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, + CancellationToken token = default) + { + var message = TcpContracts.FlushUnsavedBuffer(streamId, topicId, partitionId, fsync); + + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.FLUSH_UNSAVED_BUFFER_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId, + Consumer consumer, + PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) + { + using var rental = await PollMessagesRentedAsync(streamId, topicId, partitionId, consumer, pollingStrategy, + count, autoCommit, token); + return BinaryMapper.MaterializeMessages(rental); + } + + /// + public async Task PollMessagesRentedAsync(Identifier streamId, Identifier topicId, + uint? partitionId, + Consumer consumer, + PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) + { + var messageBufferSize = CalculateMessageBufferSize(streamId, topicId, consumer); + var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize); + var payload = ArrayPool.Shared.Rent(payloadBufferSize); + IMemoryOwner? responseBuffer = null; + + try + { + TcpContracts.GetMessages(payload.AsSpan().Slice(8, messageBufferSize), consumer, streamId, + topicId, pollingStrategy, count, autoCommit, partitionId); + BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[..4], messageBufferSize + 4); + BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[4..8], CommandCodes.POLL_MESSAGES_CODE); + + responseBuffer = await SendWithResponseAsync(payload.AsMemory(0, payloadBufferSize), token); + return BinaryMapper.MapRentedMessages(responseBuffer.Memory, responseBuffer); + } + catch + { + responseBuffer?.Dispose(); + throw; + } + finally + { + ArrayPool.Shared.Return(payload); + } + } + + /// + public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset, + uint? partitionId, CancellationToken token = default) + { + var message = TcpContracts.UpdateOffset(streamId, topicId, consumer, offset, partitionId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.STORE_CONSUMER_OFFSET_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, + uint? partitionId, CancellationToken token = default) + { + var message = TcpContracts.GetOffset(streamId, topicId, consumer, partitionId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_OFFSET_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapOffsets(responseBuffer.Memory.Span); + } + + /// + public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId, + CancellationToken token = default) + { + var message = TcpContracts.DeleteOffset(streamId, topicId, consumer, partitionId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_OFFSET_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task> GetConsumerGroupsAsync(Identifier streamId, + Identifier topicId, + CancellationToken token = default) + { + var message = TcpContracts.GetGroups(streamId, topicId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUPS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return []; + } + + return BinaryMapper.MapConsumerGroups(responseBuffer.Memory.Span); + } + + /// + public async Task GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId, + Identifier groupId, CancellationToken token = default) + { + var message = TcpContracts.GetGroup(streamId, topicId, groupId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUP_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapConsumerGroup(responseBuffer.Memory.Span); + } + + /// + public async Task CreateConsumerGroupAsync(Identifier streamId, Identifier topicId, + string name, CancellationToken token = default) + { + var message = TcpContracts.CreateGroup(streamId, topicId, name); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_CONSUMER_GROUP_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapConsumerGroup(responseBuffer.Memory.Span); + } + + /// + public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, + CancellationToken token = default) + { + var message = TcpContracts.DeleteGroup(streamId, topicId, groupId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_GROUP_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, + CancellationToken token = default) + { + var message = TcpContracts.JoinGroup(streamId, topicId, groupId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.JOIN_CONSUMER_GROUP_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId, + CancellationToken token = default) + { + var message = TcpContracts.LeaveGroup(streamId, topicId, groupId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LEAVE_CONSUMER_GROUP_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, + CancellationToken token = default) + { + var message = TcpContracts.DeletePartitions(streamId, topicId, partitionsCount); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PARTITIONS_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount, + CancellationToken token = default) + { + var message = TcpContracts.CreatePartitions(streamId, topicId, partitionsCount); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PARTITIONS_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId, + uint segmentsCount, CancellationToken token = default) + { + var message = TcpContracts.DeleteSegments(streamId, topicId, partitionId, segmentsCount); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_SEGMENTS_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task GetMeAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_ME_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapClient(responseBuffer.Memory.Span); + } + + /// + public async Task GetStatsAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STATS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapStats(responseBuffer.Memory.Span); + } + + /// + public async Task GetClusterMetadataAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLUSTER_METADATA_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapClusterMetadata(responseBuffer.Memory.Span); + } + + /// + public async Task PingAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PING_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task GetSnapshotAsync(SnapshotCompression compression, + IList snapshotTypes, CancellationToken token = default) + { + var message = TcpContracts.GetSnapshot(compression, snapshotTypes); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_SNAPSHOT_CODE); + + using IMemoryOwner result = await SendWithResponseAsync(payload, token); + + return result.Memory.Span.ToArray(); + } + + /// + public async Task ConnectAsync(CancellationToken token = default) + { + if (_state is ConnectionState.Connected + or ConnectionState.Authenticating + or ConnectionState.Authenticated) + { + _logger.LogWarning("Connection is already connected"); + return; + } + + if (_lastConnectionTime != DateTimeOffset.MinValue) + { + await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token); + } + + SetConnectionStateAsync(ConnectionState.Connecting); + _isConnecting = true; + try + { + await TryEstablishConnectionAsync(token); + } + finally + { + _isConnecting = false; + } + } + + /// + public async Task> GetClientsAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENTS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return []; + } + + return BinaryMapper.MapClients(responseBuffer.Memory.Span); + } + + /// + public async Task GetClientByIdAsync(uint clientId, CancellationToken token = default) + { + var message = TcpContracts.GetClient(clientId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENT_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapClient(responseBuffer.Memory.Span); + } + + /// + public async Task GetUserAsync(Identifier userId, CancellationToken token = default) + { + var message = TcpContracts.GetUser(userId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USER_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapUser(responseBuffer.Memory.Span); + } + + /// + public async Task> GetUsersAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USERS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return []; + } + + return BinaryMapper.MapUsers(responseBuffer.Memory.Span); + } + + /// + public async Task CreateUserAsync(string userName, string password, UserStatus status, + Permissions? permissions = null, CancellationToken token = default) + { + var message = TcpContracts.CreateUser(userName, password, status, permissions); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_USER_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapUser(responseBuffer.Memory.Span); + } + + /// + public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default) + { + var message = TcpContracts.DeleteUser(userId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_USER_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null, + CancellationToken token = default) + { + var message = TcpContracts.UpdateUser(userId, userName, status); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_USER_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null, + CancellationToken token = default) + { + var message = TcpContracts.UpdatePermissions(userId, permissions); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_PERMISSIONS_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword, + CancellationToken token = default) + { + var message = TcpContracts.ChangePassword(userId, currentPassword, newPassword); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CHANGE_PASSWORD_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task LoginUserAsync(string userName, string password, CancellationToken token = default) + { + if (_state == ConnectionState.Disconnected) + { + throw new NotConnectedException(); + } + + // TODO: Add binary protocol version + var message = TcpContracts.LoginUser(userName, password, SdkVersion.Value, "csharp-sdk"); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_USER_CODE); + + SetConnectionStateAsync(ConnectionState.Authenticating); + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..responseBuffer.Memory.Length]); + SetConnectionStateAsync(ConnectionState.Authenticated); + + if (await RedirectAsync(token)) + { + await ConnectAsync(token); + return await LoginUserAsync(userName, password, token); + } + + var authResponse = new AuthResponse(userId, null); + return authResponse; + } + + /// + public async Task LogoutUserAsync(CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGOUT_USER_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task> GetPersonalAccessTokensAsync( + CancellationToken token = default) + { + var message = Array.Empty(); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_PERSONAL_ACCESS_TOKENS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return []; + } + + return BinaryMapper.MapPersonalAccessTokens(responseBuffer.Memory.Span); + } + + /// + public async Task CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null, + CancellationToken token = default) + { + var message = TcpContracts.CreatePersonalAccessToken(name, DurationHelpers.ToDuration(expiry)); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + return BinaryMapper.MapRawPersonalAccessToken(responseBuffer.Memory.Span); + } + + /// + public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default) + { + var message = TcpContracts.DeletePersonalRequestToken(name); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PERSONAL_ACCESS_TOKEN_CODE); + + await SendAckAsync(payload, token); + } + + /// + public async Task LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default) + { + var message = TcpContracts.LoginWithPersonalAccessToken(token); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE); + + SetConnectionStateAsync(ConnectionState.Authenticating); + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, ct); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..4]); + + SetConnectionStateAsync(ConnectionState.Authenticated); + + if (await RedirectAsync(ct)) + { + await ConnectAsync(ct); + return await LoginWithPersonalAccessTokenAsync(token, ct); + } + + return new AuthResponse(userId, null); + } + + private async Task TryEstablishConnectionAsync(CancellationToken token) + { + var retryCount = 0; + var delay = _configuration.ReconnectionSettings.InitialDelay; + do + { + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract + _stream?.Close(); + _stream?.Dispose(); + + if (string.IsNullOrEmpty(_currentAddress)) + { + _currentAddress = _configuration.BaseAddress; + } + + var urlPortSplitter = _currentAddress.Split(":"); + if (urlPortSplitter.Length > 2) + { + throw new InvalidBaseAddressException(); + } + + try + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + socket.SendBufferSize = _configuration.SendBufferSize; + socket.ReceiveBufferSize = _configuration.ReceiveBufferSize; + + await socket.ConnectAsync(urlPortSplitter[0], int.Parse(urlPortSplitter[1]), token); + + socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true); + socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 5); + + SetConnectionStateAsync(ConnectionState.Connected); + _lastConnectionTime = DateTimeOffset.UtcNow; + + _stream = _configuration.TlsSettings.Enabled switch + { + true => await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings), + false => new TcpConnectionStream(new NetworkStream(socket, true)) + }; + + if (_configuration.AutoLoginSettings.Enabled) + { + _logger.LogInformation("Auto login enabled. Trying to login with credentials: {Username}", + _configuration.AutoLoginSettings.Username); + await LoginUserAsync(_configuration.AutoLoginSettings.Username, + _configuration.AutoLoginSettings.Password, token); + } + + break; + } + catch (Exception e) + { + _logger.LogError(e, "Failed to connect"); + + if (!_configuration.ReconnectionSettings.Enabled || + (_configuration.ReconnectionSettings.MaxRetries > 0 && + retryCount >= _configuration.ReconnectionSettings.MaxRetries)) + { + SetConnectionStateAsync(ConnectionState.Disconnected); + throw; + } + + retryCount++; + if (_configuration.ReconnectionSettings.UseExponentialBackoff) + { + delay *= _configuration.ReconnectionSettings.BackoffMultiplier; + + if (delay > _configuration.ReconnectionSettings.MaxDelay) + { + delay = _configuration.ReconnectionSettings.MaxDelay; + } + } + + if (_logger.IsEnabled(LogLevel.Information)) + { + _logger.LogInformation("Retrying connection attempt {RetryCount} with delay {Delay}", retryCount, + delay); + } + + await Task.Delay(delay, token); + } + } while (true); + } + + private async Task GetCurrentLeaderNodeAsync(CancellationToken token) + { + try + { + var clusterMetadata = await GetClusterMetadataAsync(token); + if (clusterMetadata == null) + { + return null; + } + + // Single-node cluster (clustering disabled) - no redirection needed + if (clusterMetadata.Nodes.Count() == 1) + { + return null; + } + + var leaderNode = clusterMetadata.Nodes.FirstOrDefault(x => x.Role == ClusterNodeRole.Leader); + if (leaderNode == null) + { + throw new MissingLeaderException(); + } + + return leaderNode; + } + // todo: change after error refactoring, error code 5 is for feature not supported + catch (IggyInvalidStatusCodeException e) when (e.StatusCode == 5) + { + return null; + } + } + + private async Task CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings) + { + ValidateCertificatePath(tlsSettings.CertificatePath); + + _customCaStore = new X509Certificate2Collection(); + _customCaStore.ImportFromPemFile(tlsSettings.CertificatePath); + var stream = new NetworkStream(socket, true); + var sslStream = new SslStream(stream, false, RemoteCertificateValidationCallback); + + await sslStream.AuthenticateAsClientAsync(tlsSettings.Hostname); + + return new TcpConnectionStream(sslStream); + } + + private async Task SendAckAsync(ReadOnlyMemory payload, CancellationToken token = default) + { + using IMemoryOwner _ = await SendWithResponseAsync(payload, token); + } + + private async Task> SendWithResponseAsync(ReadOnlyMemory payload, + CancellationToken token = default) + { + try + { + return await SendRawAsync(payload, token); + } + catch (Exception e) when (IsConnectionException(e) && !_isConnecting) + { + _logger.LogWarning("Connection lost"); + if (!_configuration.ReconnectionSettings.Enabled) + { + _logger.LogWarning("Reconnection is disabled"); + SetConnectionStateAsync(ConnectionState.Disconnected); + throw; + } + + return await HandleReconnectionAsync(payload, token); + } + } + + private async Task> HandleReconnectionAsync(ReadOnlyMemory payload, + CancellationToken token) + { + var currentTime = DateTimeOffset.UtcNow; + await _connectionSemaphore.WaitAsync(token); + + try + { + if (_state is ConnectionState.Connected or ConnectionState.Authenticated + && _lastConnectionTime > currentTime) + { + _logger.LogInformation("Connection already established, sending payload"); + return await SendRawAsync(payload, token); + } + + SetConnectionStateAsync(ConnectionState.Disconnected); + _logger.LogInformation("Reconnecting to the server"); + await ConnectAsync(token); + + _logger.LogInformation("Reconnected to the server"); + + await Task.Delay(_configuration.ReconnectionSettings.WaitAfterReconnect, token); + + return await SendRawAsync(payload, token); + } + finally + { + _connectionSemaphore.Release(); + } + } + + private async Task> SendRawAsync(ReadOnlyMemory payload, CancellationToken token) + { + if (_state is ConnectionState.Disconnected or ConnectionState.Connecting) + { + throw new NotConnectedException(); + } + + await _sendingSemaphore.WaitAsync(token); + + try + { + await _stream.SendAsync(payload, token); + await _stream.FlushAsync(token); + + // Read the 8-byte header (4 bytes status + 4 bytes length) + var totalRead = 0; + while (totalRead < BufferSizes.EXPECTED_RESPONSE_SIZE) + { + var readBytes + = await _stream.ReadAsync( + _responseHeaderBuffer.AsMemory(totalRead, BufferSizes.EXPECTED_RESPONSE_SIZE - totalRead), + token); + if (readBytes == 0) + { + throw new IggyZeroBytesException(); + } + + totalRead += readBytes; + } + + var response = TcpMessageStreamHelpers.GetResponseLengthAndStatus(_responseHeaderBuffer); + + if (response.Status != 0) + { + if (response.Length == 0) + { + throw new IggyInvalidStatusCodeException(response.Status, + $"Invalid response status code: {response.Status}"); + } + + + using var errorBuffer = ArrayPoolHelper.Rent(response.Length); + totalRead = 0; + while (totalRead < response.Length) + { + var readBytes + = await _stream.ReadAsync(errorBuffer.Memory.Slice(totalRead, response.Length - totalRead), token); + if (readBytes == 0) + { + throw new IggyZeroBytesException(); + } + + totalRead += readBytes; + } + + throw new InvalidResponseException(Encoding.UTF8.GetString(errorBuffer.Memory.Span)); + } + + if (response.Length == 0) + { + return EmptyMemoryOwner.Instance; + } + + var responseBuffer = ArrayPoolHelper.Rent(response.Length); + try + { + totalRead = 0; + while (totalRead < response.Length) + { + var readBytes + = await _stream.ReadAsync(responseBuffer.Memory.Slice(totalRead, response.Length - totalRead), + token); + + if (readBytes == 0) + { + throw new IggyZeroBytesException(); + } + + totalRead += readBytes; + } + } + catch + { + responseBuffer.Dispose(); + throw; + } + + return responseBuffer; + } + finally + { + _sendingSemaphore.Release(); + } + } + + private static bool IsConnectionException(Exception ex) + { + return ex is IggyZeroBytesException or + NotConnectedException or + SocketException or + IOException or + ObjectDisposedException; + } + + private static int CalculatePayloadBufferSize(int messageBufferSize) + { + return messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH; + } + + private static int CalculateMessageBufferSize(Identifier streamId, Identifier topicId, Consumer consumer) + { + // Original: 14 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.Id.Length + // Added 1 byte for partition flag + return 15 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.ConsumerId.Length; + } + + /// + /// Sets the connection state and fires the OnConnectionStateChanged event. + /// + /// The new connection state + private void SetConnectionStateAsync(ConnectionState newState) + { + if (_state == newState) + { + return; + } + + var previousState = _state; + _state = newState; + + _logger.LogInformation("Connection state changed: {PreviousState} -> {CurrentState}", previousState, newState); + _connectionEvents.Publish(new ConnectionStateChangedEventArgs(previousState, newState)); + } + + private void ValidateCertificatePath(string tlsCertificatePath) + { + if (string.IsNullOrEmpty(tlsCertificatePath) + || !File.Exists(tlsCertificatePath)) + { + throw new InvalidCertificatePathException(tlsCertificatePath); + } + } + + private bool RemoteCertificateValidationCallback(object sender, X509Certificate? certificate, X509Chain? chain, + SslPolicyErrors sslPolicyErrors) + { + if (sslPolicyErrors == SslPolicyErrors.None) + { + return true; + } + + if (certificate is null) + { + return false; + } + + if (certificate is not X509Certificate2 serverCert) + { + serverCert = new X509Certificate2(certificate); + } + + if (_customCaStore.Any(ca => ca.Thumbprint == serverCert.Thumbprint)) + { + if (DateTime.UtcNow <= serverCert.NotAfter && DateTime.UtcNow >= serverCert.NotBefore) + { + return true; + } + + _logger.LogError( + "Server certificate matches trusted key but is expired. Valid from {NotBefore} to {NotAfter}", + serverCert.NotBefore, serverCert.NotAfter); + return false; + } + + + using var customChain = new X509Chain(); + customChain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; + customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; + foreach (var ca in _customCaStore) + { + customChain.ChainPolicy.CustomTrustStore.Add(ca); + customChain.ChainPolicy.ExtraStore.Add(ca); + } + + customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; + + if (customChain.Build(new X509Certificate2(certificate))) + { + if (!sslPolicyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch)) + { + return true; + } + + _logger.LogError("Custom CA chain is valid, but hostname does not match"); + return false; + } + + foreach (var chainStatus in customChain.ChainStatus) + { + _logger.LogWarning("Certificate validation failed: {ChainStatus} - {StatusInformation}", chainStatus.Status, + chainStatus.StatusInformation); + } + + return false; + } + + private async Task RedirectAsync(CancellationToken token) + { + var currentLeaderNode = await GetCurrentLeaderNodeAsync(token); + if (currentLeaderNode == null) + { + return false; + } + + var leaderAddress = $"{currentLeaderNode.Ip}:{currentLeaderNode.Endpoints.Tcp}"; + if (leaderAddress == _currentAddress) + { + return false; + } + + _currentAddress = leaderAddress; + + _logger.LogInformation("Leader address changed. Trying to reconnect to {Address}", + leaderAddress); + + _stream.Close(); + SetConnectionStateAsync(ConnectionState.Disconnected); + return true; + } + + internal sealed class EmptyMemoryOwner : IMemoryOwner + { + public static readonly EmptyMemoryOwner Instance = new(); + + private EmptyMemoryOwner() + { + } + + public Memory Memory => Memory.Empty; + + public void Dispose() + { + } + } +} + +internal static class ArrayPoolHelper +{ + public static SlicedMemoryOwner Rent(int minimumLength) + { + return new SlicedMemoryOwner(minimumLength); + } + + internal sealed class SlicedMemoryOwner(int minimumLength) : IMemoryOwner + { + private readonly byte[] _value = ArrayPool.Shared.Rent(minimumLength); + private int _disposed; + + public Memory Memory => _value.AsMemory()[..minimumLength]; + + + private void Dispose(bool suppressFinalize) + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + ArrayPool.Shared.Return(_value); + if (suppressFinalize) + { + GC.SuppressFinalize(this); + } + } + + ~SlicedMemoryOwner() + { + Dispose(false); + } + + public void Dispose() + { + Dispose(true); + } + } +} diff --git a/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs b/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs index ffdbf32603..dc41363dcd 100644 --- a/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs +++ b/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs @@ -1,106 +1,106 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Text.Json; -using System.Text.Json.Serialization; -using Apache.Iggy.Contracts; -using Apache.Iggy.Headers; -using Apache.Iggy.Messages; - -namespace Apache.Iggy.JsonConverters; - -internal sealed class MessageResponseConverter : JsonConverter -{ - private static readonly UserHeadersConverter HeadersConverter = new(); - - public override MessageResponse Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - if (reader.TokenType != JsonTokenType.StartObject) - { - throw new JsonException("Expected start of object for MessageResponse."); - } - - MessageHeader? header = null; - byte[]? payload = null; - Dictionary? userHeaders = null; - byte[]? rawUserHeaders = null; - - while (reader.Read()) - { - if (reader.TokenType == JsonTokenType.EndObject) - { - break; - } - - if (reader.TokenType != JsonTokenType.PropertyName) - { - throw new JsonException("Expected property name."); - } - - var propertyName = reader.GetString(); - reader.Read(); - - switch (propertyName) - { - case "header": - header = JsonSerializer.Deserialize(ref reader, options); - break; - case "payload": - payload = reader.GetBytesFromBase64(); - break; - case "user_headers": - if (reader.TokenType == JsonTokenType.String) - { - rawUserHeaders = reader.GetBytesFromBase64(); - } - else if (reader.TokenType == JsonTokenType.Null) - { - userHeaders = null; - } - else - { - userHeaders = HeadersConverter.Read(ref reader, typeof(Dictionary), - options); - } - - break; - default: - reader.Skip(); - break; - } - } - - var response = new MessageResponse - { - Header = header ?? throw new JsonException("Missing 'header' field."), - Payload = payload ?? throw new JsonException("Missing 'payload' field."), - RawUserHeaders = rawUserHeaders - }; - - if (rawUserHeaders is null) - { - response.UserHeaders = userHeaders; - } - - return response; - } - - public override void Write(Utf8JsonWriter writer, MessageResponse value, JsonSerializerOptions options) - { - JsonSerializer.Serialize(writer, value, options); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text.Json; +using System.Text.Json.Serialization; +using Apache.Iggy.Contracts; +using Apache.Iggy.Headers; +using Apache.Iggy.Messages; + +namespace Apache.Iggy.JsonConverters; + +internal sealed class MessageResponseConverter : JsonConverter +{ + private static readonly UserHeadersConverter HeadersConverter = new(); + + public override MessageResponse Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException("Expected start of object for MessageResponse."); + } + + MessageHeader? header = null; + byte[]? payload = null; + Dictionary? userHeaders = null; + byte[]? rawUserHeaders = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + break; + } + + if (reader.TokenType != JsonTokenType.PropertyName) + { + throw new JsonException("Expected property name."); + } + + var propertyName = reader.GetString(); + reader.Read(); + + switch (propertyName) + { + case "header": + header = JsonSerializer.Deserialize(ref reader, options); + break; + case "payload": + payload = reader.GetBytesFromBase64(); + break; + case "user_headers": + if (reader.TokenType == JsonTokenType.String) + { + rawUserHeaders = reader.GetBytesFromBase64(); + } + else if (reader.TokenType == JsonTokenType.Null) + { + userHeaders = null; + } + else + { + userHeaders = HeadersConverter.Read(ref reader, typeof(Dictionary), + options); + } + + break; + default: + reader.Skip(); + break; + } + } + + var response = new MessageResponse + { + Header = header ?? throw new JsonException("Missing 'header' field."), + Payload = payload ?? throw new JsonException("Missing 'payload' field."), + RawUserHeaders = rawUserHeaders + }; + + if (rawUserHeaders is null) + { + response.UserHeaders = userHeaders; + } + + return response; + } + + public override void Write(Utf8JsonWriter writer, MessageResponse value, JsonSerializerOptions options) + { + JsonSerializer.Serialize(writer, value, options); + } +} diff --git a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs index ad47f03990..436413153f 100644 --- a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs @@ -228,38 +228,34 @@ private static (UserResponse response, int position) MapToUserResponse(ReadOnlyS var readBytes = 4 + 8 + 1 + 1 + usernameLength; return (new UserResponse - { - Id = id, - CreatedAt = createdAt, - Status = userStatus, - Username = username - }, + { + Id = id, + CreatedAt = createdAt, + Status = userStatus, + Username = username + }, readBytes); } internal static ClientResponse MapClient(ReadOnlySpan payload) { var (response, position) = MapClientInfo(payload, 0); - var consumerGroups = new List(); - var length = payload.Length; + var consumerGroups = new List(response.ConsumerGroupsCount); - while (position < length) + for (var i = 0; i < response.ConsumerGroupsCount; i++) { - for (var i = 0; i < response.ConsumerGroupsCount; i++) - { - var streamId = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]); - var topicId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 4)..(position + 8)]); - var consumerGroupId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 8)..(position + 12)]); - var consumerGroup - = new ConsumerGroupInfo - { - StreamId = streamId, - TopicId = topicId, - GroupId = consumerGroupId - }; - consumerGroups.Add(consumerGroup); - position += 12; - } + var streamId = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]); + var topicId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 4)..(position + 8)]); + var consumerGroupId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 8)..(position + 12)]); + var consumerGroup + = new ConsumerGroupInfo + { + StreamId = streamId, + TopicId = topicId, + GroupId = consumerGroupId + }; + consumerGroups.Add(consumerGroup); + position += 12; } return new ClientResponse @@ -337,19 +333,6 @@ internal static OffsetResponse MapOffsets(ReadOnlySpan payload) }; } - internal static PolledMessagesRental MapRentedMessages(IMemoryOwner payloadOwner) - { - try - { - return MapRentedMessages(payloadOwner.Memory, payloadOwner); - } - catch - { - payloadOwner.Dispose(); - throw; - } - } - internal static PolledMessagesRental MapRentedMessages(ReadOnlyMemory payload, IMemoryOwner payloadOwner) { @@ -953,11 +936,11 @@ var partitionId } return (new ConsumerGroupMember - { - Id = id, - PartitionsCount = partitionsCount, - Partitions = partitions - }, + { + Id = id, + PartitionsCount = partitionsCount, + Partitions = partitions + }, 8 + partitionsCount * 4); } diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs index e9a8084a8a..374ae18726 100644 --- a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs @@ -1,483 +1,483 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers; -using System.Runtime.CompilerServices; -using System.Text; -using Apache.Iggy.Consumers; -using Apache.Iggy.Contracts; -using Apache.Iggy.Encryption; -using Apache.Iggy.Exceptions; -using Apache.Iggy.IggyClient; -using Apache.Iggy.IggyClient.Implementations; -using Apache.Iggy.Kinds; -using Apache.Iggy.Messages; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; - -namespace Apache.Iggy.Tests.ConsumerTests; - -public class RentedConsumerTests -{ - [Fact] - public async Task ReceiveRentedAsync_Should_YieldMessages_WithExpectedPayloads() - { - var owner = new TrackingMemoryOwner(1024); - IReadOnlyList messages = BuildMessages(owner, 3); - var rental = new PolledMessagesRental(owner) - { - PartitionId = 1, - CurrentOffset = 2, - Messages = messages - }; - Mock client = BuildClientMock(new Queue(new[] { rental })); - var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance); - await consumer.InitAsync(TestContext.Current.CancellationToken); - - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - var got = new List(); - await using (IAsyncEnumerator e = consumer.ReceiveRentedAsync(cts.Token) - .GetAsyncEnumerator(TestContext.Current.CancellationToken)) - { - for (var i = 0; i < 3; i++) - { - Assert.True(await e.MoveNextAsync()); - got.Add(e.Current); - } - } - - Assert.Equal(3, got.Count); - for (var i = 0; i < 3; i++) - { - Assert.Equal(MessageStatus.Success, got[i].Status); - Assert.Equal($"msg-{i}", Encoding.UTF8.GetString(got[i].Message.Payload.Span)); - Assert.Equal((ulong)i, got[i].CurrentOffset); - Assert.Equal(1u, got[i].PartitionId); - Assert.Null(got[i].Error); - } - - // Buffer still rented — none of the messages have been disposed. - Assert.Equal(0, owner.DisposeCount); - - foreach (var m in got) - { - m.Dispose(); - } - - // After disposing every message of the batch, rental returned to pool exactly once. - Assert.Equal(1, owner.DisposeCount); - await consumer.DisposeAsync(); - } - - [Fact] - public async Task ReceiveRentedAsync_Should_NotReturnBuffer_UntilAllMessagesDisposed() - { - var owner = new TrackingMemoryOwner(1024); - IReadOnlyList messages = BuildMessages(owner, 3); - var rental = new PolledMessagesRental(owner) - { - PartitionId = 1, - CurrentOffset = 2, - Messages = messages - }; - Mock client = BuildClientMock(new Queue(new[] { rental })); - var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance); - await consumer.InitAsync(TestContext.Current.CancellationToken); - - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - var got = new List(); - await using (IAsyncEnumerator e = consumer.ReceiveRentedAsync(cts.Token) - .GetAsyncEnumerator(TestContext.Current.CancellationToken)) - { - for (var i = 0; i < 3; i++) - { - Assert.True(await e.MoveNextAsync()); - got.Add(e.Current); - } - } - - // Dispose only the first two: buffer must still be alive. - got[0].Dispose(); - Assert.Equal(0, owner.DisposeCount); - got[1].Dispose(); - Assert.Equal(0, owner.DisposeCount); - - // Final dispose returns the buffer. - got[2].Dispose(); - Assert.Equal(1, owner.DisposeCount); - - // Calling Dispose again is a no-op — refcount must not drop below zero. - got[0].Dispose(); - got[1].Dispose(); - got[2].Dispose(); - Assert.Equal(1, owner.DisposeCount); - - await consumer.DisposeAsync(); - } - - [Fact] - public async Task ReceiveRentedAsync_Should_YieldDecryptionFailed_AndStillReleaseBuffer_WhenEncryptorThrows() - { - var owner = new TrackingMemoryOwner(1024); - IReadOnlyList messages = BuildMessages(owner, 1); - var rental = new PolledMessagesRental(owner) - { - PartitionId = 1, - CurrentOffset = 0, - Messages = messages - }; - Mock client = BuildClientMock(new Queue(new[] { rental })); - var encryptor = new ThrowingEncryptor(); - var config = BuildConfig(); - config.MessageEncryptor = encryptor; - var consumer = new IggyConsumer(client.Object, config, NullLoggerFactory.Instance); - await consumer.InitAsync(TestContext.Current.CancellationToken); - - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - ReceivedRentedMessage? got = null; - await using (IAsyncEnumerator e = consumer.ReceiveRentedAsync(cts.Token) - .GetAsyncEnumerator(TestContext.Current.CancellationToken)) - { - Assert.True(await e.MoveNextAsync()); - got = e.Current; - } - - Assert.NotNull(got); - Assert.Equal(MessageStatus.DecryptionFailed, got!.Status); - Assert.IsType(got.Error); - - Assert.Equal(0, owner.DisposeCount); - got.Dispose(); - Assert.Equal(1, owner.DisposeCount); - - await consumer.DisposeAsync(); - } - - [Fact] - public async Task ReceiveRentedAsync_Should_Throw_WhenConsumerNotInitialized() - { - Mock client = BuildClientMock(new Queue()); - var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance); - - await Assert.ThrowsAsync(async () => - { - await foreach (var _ in consumer.ReceiveRentedAsync(TestContext.Current.CancellationToken)) - { - } - }); - - await consumer.DisposeAsync(); - } - - [Fact] - public void RentedBatchHandle_Release_BeyondAcquired_Throws() - { - var owner = new TrackingMemoryOwner(16); - var rental = new PolledMessagesRental(owner) - { - PartitionId = 1, - CurrentOffset = 0, - Messages = Array.Empty() - }; - var handle = new RentedBatchHandle(rental); - - handle.Release(); // releases self-ref -> 0 -> disposes rental - Assert.Equal(1, owner.DisposeCount); - - Assert.Throws(() => handle.Release()); - } - - [Fact] - public void RentedBatchHandle_AcquireRelease_Balanced_DisposesOnce() - { - var owner = new TrackingMemoryOwner(16); - var rental = new PolledMessagesRental(owner) - { - PartitionId = 1, - CurrentOffset = 0, - Messages = Array.Empty() - }; - var handle = new RentedBatchHandle(rental); // refCount=1 (self-ref) - - handle.Acquire(); // 2 - handle.Acquire(); // 3 - Assert.Equal(0, owner.DisposeCount); - - handle.Release(); // 2 - handle.Release(); // 1 (self-ref still alive) - Assert.Equal(0, owner.DisposeCount); - - handle.Release(); // 0 -> disposed - Assert.Equal(1, owner.DisposeCount); - } - - [Fact] - public void PolledMessagesRental_Dispose_Idempotent() - { - var owner = new TrackingMemoryOwner(16); - var rental = new PolledMessagesRental(owner) - { - PartitionId = 1, - CurrentOffset = 0, - Messages = Array.Empty() - }; - - rental.Dispose(); - rental.Dispose(); - rental.Dispose(); - - Assert.Equal(1, owner.DisposeCount); - } - - [Fact] - public async Task PolledMessagesRental_ConcurrentDispose_ReturnsBufferOnce() - { - var owner = new TrackingMemoryOwner(16); - var rental = new PolledMessagesRental(owner) - { - PartitionId = 1, - CurrentOffset = 0, - Messages = Array.Empty() - }; - - using var start = new ManualResetEventSlim(false); - Task[] tasks = Enumerable.Range(0, 64).Select(_ => Task.Run(() => - { - start.Wait(); - rental.Dispose(); - })).ToArray(); - - start.Set(); - await Task.WhenAll(tasks); - - Assert.Equal(1, owner.DisposeCount); - } - - [Fact] - public void PolledMessagesRental_ForgotDispose_FinalizerReturnsBuffer() - { - var owner = new TrackingMemoryOwner(16); - MakeAndDrop(owner); - - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); - - Assert.Equal(1, owner.DisposeCount); - - [MethodImpl(MethodImplOptions.NoInlining)] - static void MakeAndDrop(IMemoryOwner o) - { - _ = new PolledMessagesRental(o) - { - PartitionId = 1, - CurrentOffset = 0, - Messages = Array.Empty() - }; - } - } - - [Fact] - public async Task PollRented_MidLoopPublishFailure_DoesNotLeakBuffer() - { - var owner = new TrackingMemoryOwner(1024); - IReadOnlyList messages = BuildMessages(owner, 5); - var rental = new PolledMessagesRental(owner) - { - PartitionId = 1, - CurrentOffset = 4, - Messages = messages - }; - Mock client = BuildClientMock(new Queue(new[] { rental })); - var consumer - = new FailingPublishConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance) { FailAfter = 2 }; - await consumer.InitAsync(TestContext.Current.CancellationToken); - - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - var got = new List(); - IAsyncEnumerator enumerator = consumer.ReceiveRentedAsync(cts.Token) - .GetAsyncEnumerator(TestContext.Current.CancellationToken); - - for (var i = 0; i < 2; i++) - { - Assert.True(await enumerator.MoveNextAsync()); - got.Add(enumerator.Current); - } - - // First 2 publishes succeeded, 3rd injected failure aborted the loop. - // Producer self-ref was released in finally; consumer refs (2) still hold the buffer. - Assert.Equal(2, got.Count); - Assert.Equal(0, owner.DisposeCount); - - foreach (var m in got) - { - m.Dispose(); - } - - // All refs released -> rental disposed exactly once. - Assert.Equal(1, owner.DisposeCount); - - cts.Cancel(); - try - { - await enumerator.DisposeAsync(); - } - catch (OperationCanceledException) - { - } - - await consumer.DisposeAsync(); - } - - internal static IggyConsumerConfig BuildConfig() - { - return new IggyConsumerConfig - { - StreamId = Identifier.Numeric(1), - TopicId = Identifier.Numeric(1), - Consumer = Consumer.New(1), - PollingStrategy = PollingStrategy.Next(), - BatchSize = 10, - PartitionId = 1, - AutoCommitMode = AutoCommitMode.Disabled, - AutoCommit = false, - PollingIntervalMs = 0 - }; - } - - /// - /// Slices payload bytes into the supplied owner's memory and returns a list of - /// instances backed by that single rented buffer. - /// - internal static IReadOnlyList BuildMessages(TrackingMemoryOwner owner, int count) - { - var list = new List(count); - Memory buffer = owner.Memory; - var written = 0; - for (var i = 0; i < count; i++) - { - var bytes = Encoding.UTF8.GetBytes($"msg-{i}"); - bytes.CopyTo(buffer.Slice(written, bytes.Length)); - Memory slice = buffer.Slice(written, bytes.Length); - written += bytes.Length; - - list.Add(new RentedMessageResponse - { - Header = new MessageHeader - { - Offset = (ulong)i, - PayloadLength = bytes.Length - }, - Payload = slice, - RawUserHeaders = ReadOnlyMemory.Empty - }); - } - - return list; - } - - /// - /// Builds a that dequeues rentals on each - /// PollMessagesRentedAsync call. When the queue is empty, returns an - /// empty rental so the consumer can spin without dereferencing null. - /// - internal static Mock BuildClientMock(Queue rentals) - { - var mock = new Mock(MockBehavior.Loose); - mock.Setup(c => c.ConnectAsync(It.IsAny())).Returns(Task.CompletedTask); - mock.Setup(c => c.PollMessagesRentedAsync(It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny())) - .ReturnsAsync(() => - { - if (rentals.Count > 0) - { - return rentals.Dequeue(); - } - - return new PolledMessagesRental(TcpMessageStream.EmptyMemoryOwner.Instance) - { - PartitionId = 1, - CurrentOffset = 0, - Messages = Array.Empty() - }; - }); - return mock; - } - - /// - /// wrapper that counts how many times - /// has been invoked, so tests can assert that - /// the rental returns to the pool exactly once. - /// - internal sealed class TrackingMemoryOwner : IMemoryOwner - { - private readonly byte[] _buffer; - - public int DisposeCount { get; private set; } - - public TrackingMemoryOwner(int size) - { - _buffer = new byte[size]; - } - - public Memory Memory => _buffer; - - public void Dispose() - { - DisposeCount++; - } - } - - private sealed class ThrowingEncryptor : IMessageEncryptor - { - public byte[] Encrypt(Span plainData) - { - throw new NotSupportedException(); - } - - public byte[] Decrypt(ReadOnlySpan encryptedData) - { - throw new InvalidOperationException("decrypt fail"); - } - } - - private sealed class FailingPublishConsumer : IggyConsumer - { - private int _calls; - - public int FailAfter { get; set; } - - public FailingPublishConsumer(IIggyClient client, IggyConsumerConfig config, - ILoggerFactory loggerFactory) - : base(client, config, loggerFactory) - { - } - - protected override async Task PublishRentedAsync(RentedBatchHandle rental, RentedMessageResponse message, - uint partitionId, MessageStatus status, Exception? error, CancellationToken ct) - { - if (Interlocked.Increment(ref _calls) > FailAfter) - { - throw new InvalidOperationException("inject publish failure"); - } - - await base.PublishRentedAsync(rental, message, partitionId, status, error, ct); - } - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Text; +using Apache.Iggy.Consumers; +using Apache.Iggy.Contracts; +using Apache.Iggy.Encryption; +using Apache.Iggy.Exceptions; +using Apache.Iggy.IggyClient; +using Apache.Iggy.IggyClient.Implementations; +using Apache.Iggy.Kinds; +using Apache.Iggy.Messages; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace Apache.Iggy.Tests.ConsumerTests; + +public class RentedConsumerTests +{ + [Fact] + public async Task ReceiveRentedAsync_Should_YieldMessages_WithExpectedPayloads() + { + var owner = new TrackingMemoryOwner(1024); + IReadOnlyList messages = BuildMessages(owner, 3); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 2, + Messages = messages + }; + Mock client = BuildClientMock(new Queue(new[] { rental })); + var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance); + await consumer.InitAsync(TestContext.Current.CancellationToken); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var got = new List(); + await using (IAsyncEnumerator e = consumer.ReceiveRentedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + for (var i = 0; i < 3; i++) + { + Assert.True(await e.MoveNextAsync()); + got.Add(e.Current); + } + } + + Assert.Equal(3, got.Count); + for (var i = 0; i < 3; i++) + { + Assert.Equal(MessageStatus.Success, got[i].Status); + Assert.Equal($"msg-{i}", Encoding.UTF8.GetString(got[i].Message.Payload.Span)); + Assert.Equal((ulong)i, got[i].CurrentOffset); + Assert.Equal(1u, got[i].PartitionId); + Assert.Null(got[i].Error); + } + + // Buffer still rented — none of the messages have been disposed. + Assert.Equal(0, owner.DisposeCount); + + foreach (var m in got) + { + m.Dispose(); + } + + // After disposing every message of the batch, rental returned to pool exactly once. + Assert.Equal(1, owner.DisposeCount); + await consumer.DisposeAsync(); + } + + [Fact] + public async Task ReceiveRentedAsync_Should_NotReturnBuffer_UntilAllMessagesDisposed() + { + var owner = new TrackingMemoryOwner(1024); + IReadOnlyList messages = BuildMessages(owner, 3); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 2, + Messages = messages + }; + Mock client = BuildClientMock(new Queue(new[] { rental })); + var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance); + await consumer.InitAsync(TestContext.Current.CancellationToken); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var got = new List(); + await using (IAsyncEnumerator e = consumer.ReceiveRentedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + for (var i = 0; i < 3; i++) + { + Assert.True(await e.MoveNextAsync()); + got.Add(e.Current); + } + } + + // Dispose only the first two: buffer must still be alive. + got[0].Dispose(); + Assert.Equal(0, owner.DisposeCount); + got[1].Dispose(); + Assert.Equal(0, owner.DisposeCount); + + // Final dispose returns the buffer. + got[2].Dispose(); + Assert.Equal(1, owner.DisposeCount); + + // Calling Dispose again is a no-op — refcount must not drop below zero. + got[0].Dispose(); + got[1].Dispose(); + got[2].Dispose(); + Assert.Equal(1, owner.DisposeCount); + + await consumer.DisposeAsync(); + } + + [Fact] + public async Task ReceiveRentedAsync_Should_YieldDecryptionFailed_AndStillReleaseBuffer_WhenEncryptorThrows() + { + var owner = new TrackingMemoryOwner(1024); + IReadOnlyList messages = BuildMessages(owner, 1); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = messages + }; + Mock client = BuildClientMock(new Queue(new[] { rental })); + var encryptor = new ThrowingEncryptor(); + var config = BuildConfig(); + config.MessageEncryptor = encryptor; + var consumer = new IggyConsumer(client.Object, config, NullLoggerFactory.Instance); + await consumer.InitAsync(TestContext.Current.CancellationToken); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + ReceivedRentedMessage? got = null; + await using (IAsyncEnumerator e = consumer.ReceiveRentedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + Assert.True(await e.MoveNextAsync()); + got = e.Current; + } + + Assert.NotNull(got); + Assert.Equal(MessageStatus.DecryptionFailed, got!.Status); + Assert.IsType(got.Error); + + Assert.Equal(0, owner.DisposeCount); + got.Dispose(); + Assert.Equal(1, owner.DisposeCount); + + await consumer.DisposeAsync(); + } + + [Fact] + public async Task ReceiveRentedAsync_Should_Throw_WhenConsumerNotInitialized() + { + Mock client = BuildClientMock(new Queue()); + var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance); + + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in consumer.ReceiveRentedAsync(TestContext.Current.CancellationToken)) + { + } + }); + + await consumer.DisposeAsync(); + } + + [Fact] + public void RentedBatchHandle_Release_BeyondAcquired_Throws() + { + var owner = new TrackingMemoryOwner(16); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = Array.Empty() + }; + var handle = new RentedBatchHandle(rental); + + handle.Release(); // releases self-ref -> 0 -> disposes rental + Assert.Equal(1, owner.DisposeCount); + + Assert.Throws(() => handle.Release()); + } + + [Fact] + public void RentedBatchHandle_AcquireRelease_Balanced_DisposesOnce() + { + var owner = new TrackingMemoryOwner(16); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = Array.Empty() + }; + var handle = new RentedBatchHandle(rental); // refCount=1 (self-ref) + + handle.Acquire(); // 2 + handle.Acquire(); // 3 + Assert.Equal(0, owner.DisposeCount); + + handle.Release(); // 2 + handle.Release(); // 1 (self-ref still alive) + Assert.Equal(0, owner.DisposeCount); + + handle.Release(); // 0 -> disposed + Assert.Equal(1, owner.DisposeCount); + } + + [Fact] + public void PolledMessagesRental_Dispose_Idempotent() + { + var owner = new TrackingMemoryOwner(16); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = Array.Empty() + }; + + rental.Dispose(); + rental.Dispose(); + rental.Dispose(); + + Assert.Equal(1, owner.DisposeCount); + } + + [Fact] + public async Task PolledMessagesRental_ConcurrentDispose_ReturnsBufferOnce() + { + var owner = new TrackingMemoryOwner(16); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = Array.Empty() + }; + + using var start = new ManualResetEventSlim(false); + Task[] tasks = Enumerable.Range(0, 64).Select(_ => Task.Run(() => + { + start.Wait(); + rental.Dispose(); + })).ToArray(); + + start.Set(); + await Task.WhenAll(tasks); + + Assert.Equal(1, owner.DisposeCount); + } + + [Fact] + public void PolledMessagesRental_ForgotDispose_FinalizerReturnsBuffer() + { + var owner = new TrackingMemoryOwner(16); + MakeAndDrop(owner); + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + Assert.Equal(1, owner.DisposeCount); + + [MethodImpl(MethodImplOptions.NoInlining)] + static void MakeAndDrop(IMemoryOwner o) + { + _ = new PolledMessagesRental(o) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = Array.Empty() + }; + } + } + + [Fact] + public async Task PollRented_MidLoopPublishFailure_DoesNotLeakBuffer() + { + var owner = new TrackingMemoryOwner(1024); + IReadOnlyList messages = BuildMessages(owner, 5); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 4, + Messages = messages + }; + Mock client = BuildClientMock(new Queue(new[] { rental })); + var consumer + = new FailingPublishConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance) { FailAfter = 2 }; + await consumer.InitAsync(TestContext.Current.CancellationToken); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var got = new List(); + IAsyncEnumerator enumerator = consumer.ReceiveRentedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken); + + for (var i = 0; i < 2; i++) + { + Assert.True(await enumerator.MoveNextAsync()); + got.Add(enumerator.Current); + } + + // First 2 publishes succeeded, 3rd injected failure aborted the loop. + // Producer self-ref was released in finally; consumer refs (2) still hold the buffer. + Assert.Equal(2, got.Count); + Assert.Equal(0, owner.DisposeCount); + + foreach (var m in got) + { + m.Dispose(); + } + + // All refs released -> rental disposed exactly once. + Assert.Equal(1, owner.DisposeCount); + + cts.Cancel(); + try + { + await enumerator.DisposeAsync(); + } + catch (OperationCanceledException) + { + } + + await consumer.DisposeAsync(); + } + + internal static IggyConsumerConfig BuildConfig() + { + return new IggyConsumerConfig + { + StreamId = Identifier.Numeric(1), + TopicId = Identifier.Numeric(1), + Consumer = Consumer.New(1), + PollingStrategy = PollingStrategy.Next(), + BatchSize = 10, + PartitionId = 1, + AutoCommitMode = AutoCommitMode.Disabled, + AutoCommit = false, + PollingIntervalMs = 0 + }; + } + + /// + /// Slices payload bytes into the supplied owner's memory and returns a list of + /// instances backed by that single rented buffer. + /// + internal static IReadOnlyList BuildMessages(TrackingMemoryOwner owner, int count) + { + var list = new List(count); + Memory buffer = owner.Memory; + var written = 0; + for (var i = 0; i < count; i++) + { + var bytes = Encoding.UTF8.GetBytes($"msg-{i}"); + bytes.CopyTo(buffer.Slice(written, bytes.Length)); + Memory slice = buffer.Slice(written, bytes.Length); + written += bytes.Length; + + list.Add(new RentedMessageResponse + { + Header = new MessageHeader + { + Offset = (ulong)i, + PayloadLength = bytes.Length + }, + Payload = slice, + RawUserHeaders = ReadOnlyMemory.Empty + }); + } + + return list; + } + + /// + /// Builds a that dequeues rentals on each + /// PollMessagesRentedAsync call. When the queue is empty, returns an + /// empty rental so the consumer can spin without dereferencing null. + /// + internal static Mock BuildClientMock(Queue rentals) + { + var mock = new Mock(MockBehavior.Loose); + mock.Setup(c => c.ConnectAsync(It.IsAny())).Returns(Task.CompletedTask); + mock.Setup(c => c.PollMessagesRentedAsync(It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny())) + .ReturnsAsync(() => + { + if (rentals.Count > 0) + { + return rentals.Dequeue(); + } + + return new PolledMessagesRental(TcpMessageStream.EmptyMemoryOwner.Instance) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = Array.Empty() + }; + }); + return mock; + } + + /// + /// wrapper that counts how many times + /// has been invoked, so tests can assert that + /// the rental returns to the pool exactly once. + /// + internal sealed class TrackingMemoryOwner : IMemoryOwner + { + private readonly byte[] _buffer; + + public int DisposeCount { get; private set; } + + public TrackingMemoryOwner(int size) + { + _buffer = new byte[size]; + } + + public Memory Memory => _buffer; + + public void Dispose() + { + DisposeCount++; + } + } + + private sealed class ThrowingEncryptor : IMessageEncryptor + { + public byte[] Encrypt(byte[] plainData) + { + throw new NotSupportedException(); + } + + public byte[] Decrypt(byte[] encryptedData) + { + throw new InvalidOperationException("decrypt fail"); + } + } + + private sealed class FailingPublishConsumer : IggyConsumer + { + private int _calls; + + public int FailAfter { get; set; } + + public FailingPublishConsumer(IIggyClient client, IggyConsumerConfig config, + ILoggerFactory loggerFactory) + : base(client, config, loggerFactory) + { + } + + protected override async Task PublishRentedAsync(RentedBatchHandle rental, RentedMessageResponse message, + uint partitionId, MessageStatus status, Exception? error, CancellationToken ct) + { + if (Interlocked.Increment(ref _calls) > FailAfter) + { + throw new InvalidOperationException("inject publish failure"); + } + + await base.PublishRentedAsync(rental, message, partitionId, status, error, ct); + } + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedTypedConsumerTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedTypedConsumerTests.cs index 26dfd89e50..6da059b5b6 100644 --- a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedTypedConsumerTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedTypedConsumerTests.cs @@ -1,183 +1,183 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Text; -using Apache.Iggy.Consumers; -using Apache.Iggy.Contracts; -using Apache.Iggy.IggyClient; -using Apache.Iggy.Kinds; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; - -namespace Apache.Iggy.Tests.ConsumerTests; - -public class RentedTypedConsumerTests -{ - [Fact] - public async Task ReceiveRentedDeserializedAsync_Should_YieldDeserialized_AndReturnBuffer() - { - var owner = new RentedConsumerTests.TrackingMemoryOwner(1024); - IReadOnlyList messages = RentedConsumerTests.BuildMessages(owner, 3); - var rental = new PolledMessagesRental(owner) - { - PartitionId = 1, - CurrentOffset = 2, - Messages = messages - }; - Mock client - = RentedConsumerTests.BuildClientMock(new Queue(new[] { rental })); - var deserializer = new StringDeserializer(); - var consumer - = new IggyConsumer(client.Object, BuildTypedConfig(deserializer), NullLoggerFactory.Instance); - await consumer.InitAsync(TestContext.Current.CancellationToken); - - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - var got = new List>(); - await using (IAsyncEnumerator> e = consumer - .ReceiveRentedDeserializedAsync(cts.Token) - .GetAsyncEnumerator(TestContext.Current.CancellationToken)) - { - for (var i = 0; i < 3; i++) - { - Assert.True(await e.MoveNextAsync()); - got.Add(e.Current); - } - } - - Assert.Equal(3, got.Count); - for (var i = 0; i < 3; i++) - { - Assert.Equal(MessageStatus.Success, got[i].Status); - Assert.Equal($"msg-{i}", got[i].Data); - Assert.Equal((ulong)i, got[i].CurrentOffset); - Assert.Equal(1u, got[i].PartitionId); - Assert.Null(got[i].Error); - } - - // Entire batch deserialized before first yield; rental already returned to pool. - Assert.Equal(1, owner.DisposeCount); - await consumer.DisposeAsync(); - } - - [Fact] - public async Task ReceiveRentedDeserializedAsync_Should_ReleaseEntireBatch_BeforeFirstYield() - { - var owner = new RentedConsumerTests.TrackingMemoryOwner(1024); - IReadOnlyList messages = RentedConsumerTests.BuildMessages(owner, 3); - var rental = new PolledMessagesRental(owner) - { - PartitionId = 1, - CurrentOffset = 2, - Messages = messages - }; - Mock client - = RentedConsumerTests.BuildClientMock(new Queue(new[] { rental })); - var deserializer = new StringDeserializer(); - var consumer - = new IggyConsumer(client.Object, BuildTypedConfig(deserializer), NullLoggerFactory.Instance); - await consumer.InitAsync(TestContext.Current.CancellationToken); - - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - await using IAsyncEnumerator> e = consumer.ReceiveRentedDeserializedAsync(cts.Token) - .GetAsyncEnumerator(TestContext.Current.CancellationToken); - - Assert.True(await e.MoveNextAsync()); - Assert.Equal(1, owner.DisposeCount); - Assert.Equal("msg-0", e.Current.Data); - - // Remaining messages come from the pre-deserialized list — no further disposal needed. - Assert.True(await e.MoveNextAsync()); - Assert.Equal(1, owner.DisposeCount); - Assert.Equal("msg-1", e.Current.Data); - - Assert.True(await e.MoveNextAsync()); - Assert.Equal(1, owner.DisposeCount); - Assert.Equal("msg-2", e.Current.Data); - - await consumer.DisposeAsync(); - } - - [Fact] - public async Task DeserializerThrows_Should_YieldDeserializationFailed_AndStillReturnBuffer() - { - var owner = new RentedConsumerTests.TrackingMemoryOwner(1024); - IReadOnlyList messages = RentedConsumerTests.BuildMessages(owner, 1); - var rental = new PolledMessagesRental(owner) - { - PartitionId = 1, - CurrentOffset = 0, - Messages = messages - }; - Mock client - = RentedConsumerTests.BuildClientMock(new Queue(new[] { rental })); - var deserializer = new StringDeserializer { ThrowOnNext = true }; - var consumer - = new IggyConsumer(client.Object, BuildTypedConfig(deserializer), NullLoggerFactory.Instance); - await consumer.InitAsync(TestContext.Current.CancellationToken); - - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - DeserializedMessage? got = null; - await using (IAsyncEnumerator> e = consumer - .ReceiveRentedDeserializedAsync(cts.Token) - .GetAsyncEnumerator(TestContext.Current.CancellationToken)) - { - Assert.True(await e.MoveNextAsync()); - got = e.Current; - } - - Assert.NotNull(got); - Assert.Equal(MessageStatus.DeserializationFailed, got!.Status); - Assert.Null(got.Data); - Assert.IsType(got.Error); - - // Even on deserialization failure, the using-block inside the typed consumer releases the rental. - Assert.Equal(1, owner.DisposeCount); - await consumer.DisposeAsync(); - } - - private static IggyConsumerConfig BuildTypedConfig(IDeserializer deserializer) - { - return new IggyConsumerConfig - { - StreamId = Identifier.Numeric(1), - TopicId = Identifier.Numeric(1), - Consumer = Consumer.New(1), - PollingStrategy = PollingStrategy.Next(), - BatchSize = 10, - PartitionId = 1, - AutoCommitMode = AutoCommitMode.Disabled, - AutoCommit = false, - PollingIntervalMs = 0, - Deserializer = deserializer - }; - } - - private sealed class StringDeserializer : IDeserializer - { - public bool ThrowOnNext { get; set; } - - public string Deserialize(ReadOnlyMemory data) - { - if (ThrowOnNext) - { - throw new InvalidOperationException("deserialize fail"); - } - - return Encoding.UTF8.GetString(data.Span); - } - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text; +using Apache.Iggy.Consumers; +using Apache.Iggy.Contracts; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace Apache.Iggy.Tests.ConsumerTests; + +public class RentedTypedConsumerTests +{ + [Fact] + public async Task ReceiveDeserializedAsync_Should_YieldDeserialized_AndReturnBuffer() + { + var owner = new RentedConsumerTests.TrackingMemoryOwner(1024); + IReadOnlyList messages = RentedConsumerTests.BuildMessages(owner, 3); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 2, + Messages = messages + }; + Mock client + = RentedConsumerTests.BuildClientMock(new Queue(new[] { rental })); + var deserializer = new StringDeserializer(); + var consumer + = new IggyConsumer(client.Object, BuildTypedConfig(deserializer), NullLoggerFactory.Instance); + await consumer.InitAsync(TestContext.Current.CancellationToken); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var got = new List>(); + await using (IAsyncEnumerator> e = consumer + .ReceiveDeserializedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + for (var i = 0; i < 3; i++) + { + Assert.True(await e.MoveNextAsync()); + got.Add(e.Current); + } + } + + Assert.Equal(3, got.Count); + for (var i = 0; i < 3; i++) + { + Assert.Equal(MessageStatus.Success, got[i].Status); + Assert.Equal($"msg-{i}", got[i].Data); + Assert.Equal((ulong)i, got[i].CurrentOffset); + Assert.Equal(1u, got[i].PartitionId); + Assert.Null(got[i].Error); + } + + // Entire batch deserialized before first yield; rental already returned to pool. + Assert.Equal(1, owner.DisposeCount); + await consumer.DisposeAsync(); + } + + [Fact] + public async Task ReceiveDeserializedAsync_Should_ReleaseEntireBatch_BeforeFirstYield() + { + var owner = new RentedConsumerTests.TrackingMemoryOwner(1024); + IReadOnlyList messages = RentedConsumerTests.BuildMessages(owner, 3); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 2, + Messages = messages + }; + Mock client + = RentedConsumerTests.BuildClientMock(new Queue(new[] { rental })); + var deserializer = new StringDeserializer(); + var consumer + = new IggyConsumer(client.Object, BuildTypedConfig(deserializer), NullLoggerFactory.Instance); + await consumer.InitAsync(TestContext.Current.CancellationToken); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await using IAsyncEnumerator> e = consumer.ReceiveDeserializedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken); + + Assert.True(await e.MoveNextAsync()); + Assert.Equal(1, owner.DisposeCount); + Assert.Equal("msg-0", e.Current.Data); + + // Remaining messages come from the pre-deserialized list — no further disposal needed. + Assert.True(await e.MoveNextAsync()); + Assert.Equal(1, owner.DisposeCount); + Assert.Equal("msg-1", e.Current.Data); + + Assert.True(await e.MoveNextAsync()); + Assert.Equal(1, owner.DisposeCount); + Assert.Equal("msg-2", e.Current.Data); + + await consumer.DisposeAsync(); + } + + [Fact] + public async Task DeserializerThrows_Should_YieldDeserializationFailed_AndStillReturnBuffer() + { + var owner = new RentedConsumerTests.TrackingMemoryOwner(1024); + IReadOnlyList messages = RentedConsumerTests.BuildMessages(owner, 1); + var rental = new PolledMessagesRental(owner) + { + PartitionId = 1, + CurrentOffset = 0, + Messages = messages + }; + Mock client + = RentedConsumerTests.BuildClientMock(new Queue(new[] { rental })); + var deserializer = new StringDeserializer { ThrowOnNext = true }; + var consumer + = new IggyConsumer(client.Object, BuildTypedConfig(deserializer), NullLoggerFactory.Instance); + await consumer.InitAsync(TestContext.Current.CancellationToken); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + ReceivedMessage? got = null; + await using (IAsyncEnumerator> e = consumer + .ReceiveDeserializedAsync(cts.Token) + .GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + Assert.True(await e.MoveNextAsync()); + got = e.Current; + } + + Assert.NotNull(got); + Assert.Equal(MessageStatus.DeserializationFailed, got!.Status); + Assert.Null(got.Data); + Assert.IsType(got.Error); + + // Even on deserialization failure, the using-block inside the typed consumer releases the rental. + Assert.Equal(1, owner.DisposeCount); + await consumer.DisposeAsync(); + } + + private static IggyConsumerConfig BuildTypedConfig(IDeserializer deserializer) + { + return new IggyConsumerConfig + { + StreamId = Identifier.Numeric(1), + TopicId = Identifier.Numeric(1), + Consumer = Consumer.New(1), + PollingStrategy = PollingStrategy.Next(), + BatchSize = 10, + PartitionId = 1, + AutoCommitMode = AutoCommitMode.Disabled, + AutoCommit = false, + PollingIntervalMs = 0, + Deserializer = deserializer + }; + } + + private sealed class StringDeserializer : IDeserializer + { + public bool ThrowOnNext { get; set; } + + public string Deserialize(ReadOnlyMemory data) + { + if (ThrowOnNext) + { + throw new InvalidOperationException("deserialize fail"); + } + + return Encoding.UTF8.GetString(data.Span); + } + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/MessageResponseConverterTests.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/MessageResponseConverterTests.cs deleted file mode 100644 index 8b13789179..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/MessageResponseConverterTests.cs +++ /dev/null @@ -1 +0,0 @@ - From 645f42760b01c27f7e5b36a96e4fe8fbff26bd6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Zborek?= Date: Wed, 13 May 2026 23:48:18 +0200 Subject: [PATCH 3/4] refactor(csharp): remove unnecessary whitespace in TcpMessageStream --- .../Implementations/TcpMessageStream.cs | 13 ++- .../ConsumerTests/RentedConsumerTests.cs | 25 ------ .../UtilityTests/SlicedMemoryOwnerTests.cs | 85 +++++++++++++++++++ 3 files changed, 91 insertions(+), 32 deletions(-) create mode 100644 foreign/csharp/Iggy_SDK_Tests/UtilityTests/SlicedMemoryOwnerTests.cs diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs index a804561aae..d91425d947 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs @@ -331,7 +331,7 @@ public async Task PollMessagesRentedAsync(Identifier strea var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize); var payload = ArrayPool.Shared.Rent(payloadBufferSize); IMemoryOwner? responseBuffer = null; - + try { TcpContracts.GetMessages(payload.AsSpan().Slice(8, messageBufferSize), consumer, streamId, @@ -1090,7 +1090,7 @@ var readBytes $"Invalid response status code: {response.Status}"); } - + using var errorBuffer = ArrayPoolHelper.Rent(response.Length); totalRead = 0; while (totalRead < response.Length) @@ -1307,10 +1307,9 @@ internal sealed class SlicedMemoryOwner(int minimumLength) : IMemoryOwner { private readonly byte[] _value = ArrayPool.Shared.Rent(minimumLength); private int _disposed; - + public Memory Memory => _value.AsMemory()[..minimumLength]; - - + private void Dispose(bool suppressFinalize) { if (Interlocked.Exchange(ref _disposed, 1) != 0) @@ -1324,12 +1323,12 @@ private void Dispose(bool suppressFinalize) GC.SuppressFinalize(this); } } - + ~SlicedMemoryOwner() { Dispose(false); } - + public void Dispose() { Dispose(true); diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs index 374ae18726..54bd08b2f7 100644 --- a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs @@ -16,7 +16,6 @@ // under the License. using System.Buffers; -using System.Runtime.CompilerServices; using System.Text; using Apache.Iggy.Consumers; using Apache.Iggy.Contracts; @@ -268,30 +267,6 @@ public async Task PolledMessagesRental_ConcurrentDispose_ReturnsBufferOnce() Assert.Equal(1, owner.DisposeCount); } - [Fact] - public void PolledMessagesRental_ForgotDispose_FinalizerReturnsBuffer() - { - var owner = new TrackingMemoryOwner(16); - MakeAndDrop(owner); - - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); - - Assert.Equal(1, owner.DisposeCount); - - [MethodImpl(MethodImplOptions.NoInlining)] - static void MakeAndDrop(IMemoryOwner o) - { - _ = new PolledMessagesRental(o) - { - PartitionId = 1, - CurrentOffset = 0, - Messages = Array.Empty() - }; - } - } - [Fact] public async Task PollRented_MidLoopPublishFailure_DoesNotLeakBuffer() { diff --git a/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SlicedMemoryOwnerTests.cs b/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SlicedMemoryOwnerTests.cs new file mode 100644 index 0000000000..73e9ba3b9c --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SlicedMemoryOwnerTests.cs @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Apache.Iggy.IggyClient.Implementations; + +namespace Apache.Iggy.Tests.UtilityTests; + +public class SlicedMemoryOwnerTests +{ + private const int BufferSize = 4096; + + [Fact] + public void Dispose_ReturnsBufferToPool() + { + var owner = ArrayPoolHelper.Rent(BufferSize); + byte[] underlying = GetUnderlyingArray(owner); + owner.Dispose(); + + using var second = ArrayPoolHelper.Rent(BufferSize); + Assert.Same(underlying, GetUnderlyingArray(second)); + } + + [Fact] + public void Dispose_IsIdempotent() + { + var owner = ArrayPoolHelper.Rent(BufferSize); + owner.Dispose(); + owner.Dispose(); + owner.Dispose(); + } + + [Fact] + public void FinalizerIsDeclared() + { + Type sliced = typeof(ArrayPoolHelper) + .GetNestedType("SlicedMemoryOwner", BindingFlags.NonPublic)!; + + MethodInfo? finalizer = sliced.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance); + + Assert.NotNull(finalizer); + } + + [Fact] + public void ForgotDispose_FinalizerRunsAndReclaimsInstance() + { + WeakReference weakRef = RentWeak(); + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + Assert.False(weakRef.IsAlive); + + [MethodImpl(MethodImplOptions.NoInlining)] + static WeakReference RentWeak() => new(ArrayPoolHelper.Rent(BufferSize)); + } + + private static byte[] GetUnderlyingArray(IMemoryOwner owner) + { + if (!MemoryMarshal.TryGetArray(owner.Memory, out var segment) || segment.Array is null) + { + throw new InvalidOperationException("SlicedMemoryOwner.Memory must be array-backed."); + } + + return segment.Array; + } +} From 858433eb9b92556de0add70276e05b2eb0283873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Zborek?= Date: Thu, 14 May 2026 18:44:20 +0200 Subject: [PATCH 4/4] refactor(csharp): remove unnecessary whitespace in IggyConsumer.Rented.cs --- .../Iggy_SDK/Consumers/IggyConsumer.Rented.cs | 3 +-- .../csharp/Iggy_SDK/Consumers/IggyConsumer.cs | 9 ++++++--- .../Consumers/ReceivedRentedMessage.cs | 6 +----- .../csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 5 ----- .../ConsumerTests/RentedConsumerTests.cs | 18 ------------------ 5 files changed, 8 insertions(+), 33 deletions(-) diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs index 1644d2f225..e3d35083fd 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs @@ -29,8 +29,6 @@ namespace Apache.Iggy.Consumers; public partial class IggyConsumer { - private readonly Channel _rentedChannel = Channel.CreateUnbounded(); - /// /// Receives messages asynchronously as an async stream of rented messages. Each yielded /// shares its underlying pooled buffer with the other messages from the @@ -157,6 +155,7 @@ protected async Task PollRentedMessagesAsync(CancellationToken ct) var status = MessageStatus.Success; Exception? error = null; + // TODO: fix encryption allocations by moving it to IggyClient if (_config.MessageEncryptor != null) { try diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs index 493e552339..e76e8133f2 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs @@ -37,6 +37,7 @@ namespace Apache.Iggy.Consumers; public partial class IggyConsumer : IAsyncDisposable { private readonly Channel _channel; + private readonly Channel _rentedChannel; private readonly IIggyClient _client; private readonly IggyConsumerConfig _config; private readonly SemaphoreSlim _connectionStateSemaphore = new(1, 1); @@ -66,6 +67,8 @@ public IggyConsumer(IIggyClient client, IggyConsumerConfig config, ILoggerFactor _logger = loggerFactory.CreateLogger(); _channel = Channel.CreateBounded(new BoundedChannelOptions((int)config.BatchSize * 2)); + _rentedChannel + = Channel.CreateBounded(new BoundedChannelOptions((int)config.BatchSize * 2)); _consumerErrorEvents = new EventAggregator(loggerFactory); } @@ -200,14 +203,14 @@ public async IAsyncEnumerable ReceiveAsync([EnumeratorCancellat /// /// The offset to store /// The partition ID - /// + /// /// Cancellation token - public async Task StoreOffsetAsync(ulong offset, uint partitionId, bool resetLastPooled = false, + public async Task StoreOffsetAsync(ulong offset, uint partitionId, bool resetLastPolled = false, CancellationToken ct = default) { await _client.StoreOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, offset, partitionId, ct); - if (resetLastPooled) + if (resetLastPolled) { _lastPolledOffset[(int)partitionId] = offset; } diff --git a/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs b/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs index 60ac5ac7b1..517f43afd1 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs @@ -114,13 +114,9 @@ public void Acquire() public void Release() { var remaining = Interlocked.Decrement(ref _refCount); - if (remaining == 0) + if (remaining <= 0) { _rental.Dispose(); } - else if (remaining < 0) - { - throw new InvalidOperationException("RentedBatchHandle released more times than acquired."); - } } } diff --git a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs index 436413153f..3f979b13d5 100644 --- a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs @@ -420,11 +420,6 @@ internal static PolledMessagesRental MapRentedMessages(ReadOnlyMemory payl internal static PolledMessages MaterializeMessages(PolledMessagesRental rental) { - if (rental.Messages.Count == 0 && rental.PartitionId == 0 && rental.CurrentOffset == 0) - { - return PolledMessages.Empty; - } - var messages = new List(rental.Messages.Count); foreach (var message in rental.Messages) { diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs index 54bd08b2f7..93ee877020 100644 --- a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs @@ -183,24 +183,6 @@ await Assert.ThrowsAsync(async () => await consumer.DisposeAsync(); } - [Fact] - public void RentedBatchHandle_Release_BeyondAcquired_Throws() - { - var owner = new TrackingMemoryOwner(16); - var rental = new PolledMessagesRental(owner) - { - PartitionId = 1, - CurrentOffset = 0, - Messages = Array.Empty() - }; - var handle = new RentedBatchHandle(rental); - - handle.Release(); // releases self-ref -> 0 -> disposes rental - Assert.Equal(1, owner.DisposeCount); - - Assert.Throws(() => handle.Release()); - } - [Fact] public void RentedBatchHandle_AcquireRelease_Balanced_DisposesOnce() {