diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 31600c9f..1967419e 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -25,7 +25,8 @@ jobs: - name: Setup ClickHouse run: | curl https://clickhouse.com/ | sh - ./clickhouse server --daemon + sudo ./clickhouse install --noninteractive + sudo clickhouse start - uses: actions/cache@v5 name: Cache NuGet diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 1d6a4dc1..6b0ac434 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -26,7 +26,8 @@ jobs: shell: wsl-bash -u root {0} run: | curl https://clickhouse.com/ | sh - ./clickhouse server --daemon + ./clickhouse install --noninteractive + clickhouse start - uses: actions/cache@v5 name: Cache NuGet diff --git a/CLAUDE.md b/CLAUDE.md index 4b0a9fc4..5051137c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,7 +3,9 @@ ## Repository Overview ### Project Context -- **ClickHouse.Driver** is the official ADO.NET client for ClickHouse database +- **ClickHouse.Driver** is the official .NET client for ClickHouse database +- **Primary API**: `ClickHouseClient` - thread-safe, singleton-friendly, recommended for most use cases +- **ADO.NET API**: `ClickHouseConnection`/`ClickHouseCommand` - for ORM compatibility (Dapper, EF Core, linq2db) - **Critical priorities**: Stability, correctness, performance, and comprehensive testing - **Tech stack**: C#/.NET targeting `net6.0`, `net8.0`, `net9.0`, `net10.0` - **Tests run on**: `net6.0`, `net8.0`, `net9.0`, `net10.0`; Integration tests: `net10.0`; Benchmarks: `net10.0` @@ -12,11 +14,11 @@ ``` ClickHouse.Driver.sln ├── ClickHouse.Driver/ # Main library (NuGet package) -│ ├── ADO/ # Core ADO.NET (Connection, Command, DataReader, Parameters) +│ ├── Utility/ # ClickHouseClient (primary API), schema, feature detection +│ ├── ADO/ # ADO.NET layer (Connection, Command, DataReader, Parameters) │ ├── Types/ # 60+ ClickHouse type implementations + TypeConverter.cs -│ ├── Copy/ # Bulk copy & binary serialization +│ ├── Copy/ # Binary serialization (used internally by ClickHouseClient) │ ├── Http/ # HTTP layer & connection pooling -│ ├── Utility/ # Schema, feature detection, extensions │ └── PublicAPI/ # Public API surface tracking (analyzer-enforced) ├── ClickHouse.Driver.Tests/ # NUnit tests (multi-framework) ├── ClickHouse.Driver.IntegrationTests/ # Integration tests (net10.0) @@ -24,13 +26,40 @@ ClickHouse.Driver.sln ``` ### Key Files +- **Primary API**: `ClickHouseClient.cs` - main entry point for most applications - **Type system**: `Types/TypeConverter.cs` (14KB, complex), `Types/Grammar/` (type parsing) -- **Core ADO**: `ADO/ClickHouseConnection.cs`, `ADO/ClickHouseCommand.cs`, `ADO/Readers/` -- **Protocol**: Binary serialization in `Copy/Serializer/`, HTTP formatting in `Formats/` +- **ADO.NET layer**: `ADO/ClickHouseConnection.cs`, `ADO/ClickHouseCommand.cs`, `ADO/Readers/` - **Feature detection**: `Utility/ClickHouseFeatureMap.cs` (version-based capabilities) - **Public API**: `PublicAPI/*.txt` (Roslyn analyzer enforces shipped signatures) - **Config**: `.editorconfig` (file-scoped namespaces, StyleCop suppressions) +### API Architecture + +**ClickHouseClient** (recommended): +```csharp +using var client = new ClickHouseClient("Host=localhost"); +await client.ExecuteNonQueryAsync("CREATE TABLE ..."); +await client.InsertBinaryAsync(tableName, columns, rows); // High-performance bulk insert +using var reader = await client.ExecuteReaderAsync("SELECT ..."); +var scalar = await client.ExecuteScalarAsync("SELECT count() ..."); +``` + +**ClickHouseConnection** (for ORMs): +```csharp +// Use ClickHouseDataSource for proper connection lifetime management with ORMs +var dataSource = new ClickHouseDataSource("Host=localhost"); +services.AddSingleton(dataSource); + +// Dapper, EF Core, linq2db work with DbConnection +using var connection = dataSource.CreateConnection(); +var users = connection.Query("SELECT * FROM users"); +``` + +**Key differences**: +- `ClickHouseClient`: Thread-safe, can be singleton, has `InsertBinaryAsync` for bulk inserts +- `ClickHouseConnection`: ADO.NET `DbConnection`, required for ORM compatibility +- `ClickHouseBulkCopy`: **Deprecated** - use `ClickHouseClient.InsertBinaryAsync` instead + --- ## Development Guidelines @@ -68,9 +97,31 @@ ClickHouse.Driver.sln - **Analyzers**: Respect `.editorconfig`, StyleCop suppressions, nullable contexts ### Configuration & Settings -- **Configuration**: happens through connection string and ClickHouseClientSettings +- **Client configuration**: Connection string or `ClickHouseClientSettings` for client-level settings +- **Per-query options**: `QueryOptions` for query-specific settings (QueryId, CustomSettings, Roles, BearerToken) +- **Parameters**: Use `ClickHouseParameterCollection` with `ClickHouseDbParameter` for parameterized queries - **Feature flags**: Consider adding optional behavior behind connection string settings +```csharp +// Client-level settings +var settings = new ClickHouseClientSettings("Host=localhost"); +settings.CustomSettings.Add("max_threads", 4); +using var client = new ClickHouseClient(settings); + +// Per-query options +var options = new QueryOptions +{ + QueryId = "my-query-id", + CustomSettings = new Dictionary { ["max_execution_time"] = 30 }, +}; +await client.ExecuteReaderAsync("SELECT ...", options: options); + +// Parameters +var parameters = new ClickHouseParameterCollection(); +parameters.Add("id", 42UL); +await client.ExecuteReaderAsync("SELECT * FROM t WHERE id = {id:UInt64}", parameters); +``` + ### Observability & Diagnostics - **Error messages**: Must be clear, actionable, include context (connection string, query, server version) - **OpenTelemetry**: Changes to diagnostic paths should maintain telemetry integration diff --git a/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs b/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs index 9e0a6375..d92cf193 100644 --- a/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs +++ b/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs @@ -166,7 +166,7 @@ public async Task ClientShouldSetQueryId() public async Task ClientShouldSetUserAgent() { var headers = new HttpRequestMessage().Headers; - connection.AddDefaultHttpHeaders(headers); + client.AddDefaultHttpHeaders(headers); // Build assembly version defaults to 1.0.0 Assert.That(headers.UserAgent.ToString().Contains("ClickHouse.Driver/1.0.0"), Is.True); } @@ -335,22 +335,6 @@ public void ChangeDatabaseShouldChangeDatabase() Assert.That(conn.Database, Is.EqualTo("default")); } - [Test] - public void ShouldExcludePasswordFromRedactedConnectionString() - { - const string MOCK = "verysecurepassword"; - var settings = new ClickHouseClientSettings() - { - Password = MOCK, - }; - using var conn = new ClickHouseConnection(settings); - Assert.Multiple(() => - { - Assert.That(conn.ConnectionString, Contains.Substring($"Password={MOCK}")); - Assert.That(conn.RedactedConnectionString, Is.Not.Contains($"Password={MOCK}")); - }); - } - [Test] [TestCase("https")] [TestCase("http")] @@ -386,8 +370,8 @@ public async Task ShouldUseQueryIdForRawStream() { var queryId = Guid.NewGuid().ToString(); var httpResponseMessage = await connection.PostStreamAsync("SELECT version()", (_, _) => Task.CompletedTask, false, CancellationToken.None, queryId); - - Assert.That(ClickHouseConnection.ExtractQueryId(httpResponseMessage), Is.EqualTo(queryId)); + var queryResult = new QueryResult(httpResponseMessage); + Assert.That(queryResult.QueryId, Is.EqualTo(queryId)); } private static string[] GetColumnNames(DataTable table) => table.Columns.Cast().Select(dc => dc.ColumnName).ToArray(); @@ -665,97 +649,101 @@ public async Task Constructor_WithSettingsWithPath_ShouldApplyPath() } [Test] - public void InsertRawStreamAsync_WithNullTable_ShouldThrowArgumentException() + public void Dispose_WithOwnedClient_ShouldDisposeClient() { - using var stream = new MemoryStream(Encoding.UTF8.GetBytes("1,2,3")); - var ex = Assert.ThrowsAsync(async () => - await connection.InsertRawStreamAsync(table: null, stream: stream, format: "CSV")); - Assert.That(ex.ParamName, Is.EqualTo("table")); - } + // Arrange - connection created from connection string owns its client + var conn = new ClickHouseConnection("Host=localhost"); + var client = conn.ClickHouseClient; - [Test] - public void InsertRawStreamAsync_WithEmptyTable_ShouldThrowArgumentException() - { - using var stream = new MemoryStream(Encoding.UTF8.GetBytes("1,2,3")); - var ex = Assert.ThrowsAsync(async () => - await connection.InsertRawStreamAsync(table: "", stream: stream, format: "CSV")); - Assert.That(ex.ParamName, Is.EqualTo("table")); - } + // Act + conn.Dispose(); - [Test] - public void InsertRawStreamAsync_WithNullStream_ShouldThrowArgumentNullException() - { - var ex = Assert.ThrowsAsync(async () => - await connection.InsertRawStreamAsync(table: "test", stream: null, format: "CSV")); - Assert.That(ex.ParamName, Is.EqualTo("stream")); + // Assert - client should be disposed (calling Dispose again should be safe but we can't easily verify) + // We verify indirectly by checking that a new connection with the same client would fail + // after the original connection disposed it + Assert.DoesNotThrow(() => client.Dispose()); // Dispose should be idempotent } [Test] - public void InsertRawStreamAsync_WithNullFormat_ShouldThrowArgumentException() + public void Dispose_WithSharedClient_ShouldNotDisposeClient() { - using var stream = new MemoryStream(Encoding.UTF8.GetBytes("1,2,3")); - var ex = Assert.ThrowsAsync(async () => - await connection.InsertRawStreamAsync(table: "test", stream: stream, format: null)); - Assert.That(ex.ParamName, Is.EqualTo("format")); - } + // Arrange - create a shared client + using var sharedClient = new ClickHouseClient("Host=localhost"); - [Test] - public void InsertRawStreamAsync_WithEmptyFormat_ShouldThrowArgumentException() - { - using var stream = new MemoryStream(Encoding.UTF8.GetBytes("1,2,3")); - var ex = Assert.ThrowsAsync(async () => - await connection.InsertRawStreamAsync(table: "test", stream: stream, format: "")); - Assert.That(ex.ParamName, Is.EqualTo("format")); + // Create two connections sharing the same client + var conn1 = new ClickHouseConnection(sharedClient); + var conn2 = new ClickHouseConnection(sharedClient); + + // Act - dispose both connections + conn1.Dispose(); + conn2.Dispose(); + + // Assert - shared client should still be usable (not disposed) + // Ping will fail since there's no server, but it shouldn't throw ObjectDisposedException + Assert.DoesNotThrowAsync(async () => await sharedClient.PingAsync()); } [Test] - public async Task PingAsync_ReturnsTrue_WhenServerResponds() + public void ApplySettings_WithSharedClient_ShouldNotDisposeOriginalClient() { - var trackingHandler = new TrackingHandler(request => - { - return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("Ok.\n") }; - }); - using var httpClient = new HttpClient(trackingHandler); - var settings = new ClickHouseClientSettings { Host = "localhost", HttpClient = httpClient }; - using var conn = new ClickHouseConnection(settings); - await conn.OpenAsync(); + // Arrange - create a shared client + using var sharedClient = new ClickHouseClient("Host=localhost"); + var conn = new ClickHouseConnection(sharedClient); - var result = await conn.PingAsync(); + // Act - change connection string (which calls ApplySettings internally) + conn.ConnectionString = "Host=otherhost"; - Assert.That(result, Is.True); - Assert.That(trackingHandler.Requests.Last().RequestUri.PathAndQuery, Is.EqualTo("/ping")); + // Assert - shared client should still be usable (not disposed) + Assert.DoesNotThrowAsync(async () => await sharedClient.PingAsync()); } [Test] - public async Task PingAsync_ReturnsFalse_WhenServerReturnsError() + public void ApplySettings_AfterChange_ShouldOwnNewClient() { - var trackingHandler = new TrackingHandler(request => - { - return new HttpResponseMessage(HttpStatusCode.InternalServerError); - }); - using var httpClient = new HttpClient(trackingHandler); - var settings = new ClickHouseClientSettings { Host = "localhost", HttpClient = httpClient }; - using var conn = new ClickHouseConnection(settings); - await conn.OpenAsync(); + // Arrange - start with shared client + using var sharedClient = new ClickHouseClient("Host=localhost"); + var conn = new ClickHouseConnection(sharedClient); + + // Act - change connection string creates a new owned client + conn.ConnectionString = "Host=otherhost"; + var newClient = conn.ClickHouseClient; - var result = await conn.PingAsync(); + // Assert - new client should be different from shared client + Assert.That(newClient, Is.Not.SameAs(sharedClient)); - Assert.That(result, Is.False); + // Dispose connection - should dispose the new client (not the shared one) + conn.Dispose(); + Assert.DoesNotThrowAsync(async () => await sharedClient.PingAsync()); } [Test] - public void PingAsync_ThrowsInvalidOperationException_WhenConnectionNotOpen() + public void ChangeDatabase_ShouldBePerConnection() { - using var conn = new ClickHouseConnection("Host=localhost"); + // Arrange - create shared client with default database + using var sharedClient = new ClickHouseClient("Host=localhost;Database=default"); + using var conn1 = new ClickHouseConnection(sharedClient); + using var conn2 = new ClickHouseConnection(sharedClient); - Assert.ThrowsAsync(() => conn.PingAsync()); + // Act - change database on conn1 only + conn1.ChangeDatabase("other_db"); + + // Assert - conn1 should have new database, conn2 should still have default + Assert.That(conn1.Database, Is.EqualTo("other_db")); + Assert.That(conn2.Database, Is.EqualTo("default")); } [Test] - public async Task PingAsync_WithRealServer_ReturnsTrue() + public void ChangeDatabase_ShouldNotAffectClientSettings() { - var result = await connection.PingAsync(); + // Arrange + using var sharedClient = new ClickHouseClient("Host=localhost;Database=default"); + using var conn = new ClickHouseConnection(sharedClient); + + // Act + conn.ChangeDatabase("other_db"); - Assert.That(result, Is.True); + // Assert - client settings should be unchanged + Assert.That(conn.Database, Is.EqualTo("other_db")); + Assert.That(sharedClient.Settings.Database, Is.EqualTo("default")); } } diff --git a/ClickHouse.Driver.Tests/ADO/SessionConnectionTest.cs b/ClickHouse.Driver.Tests/ADO/SessionConnectionTest.cs index 4ae865c1..cd8df84a 100644 --- a/ClickHouse.Driver.Tests/ADO/SessionConnectionTest.cs +++ b/ClickHouse.Driver.Tests/ADO/SessionConnectionTest.cs @@ -113,26 +113,4 @@ public async Task Session_WithCustomHttpClient_ShouldWork() await connection.ExecuteStatementAsync("CREATE TEMPORARY TABLE test_temp_table (value UInt8)"); await connection.ExecuteScalarAsync("SELECT COUNT(*) from test_temp_table"); } - - [Test] - public async Task Session_ConcurrentRequests_AreSerialized() - { - var sessionId = "TEST-" + Guid.NewGuid(); - var marker = Guid.NewGuid().ToString("N"); - - using var connection = (ClickHouseConnection)CreateConnection(useSession: true, sessionId); - - var stopwatch = System.Diagnostics.Stopwatch.StartNew(); - - // Two 300ms sleep queries with markers we can find in query_log - var task1 = connection.ExecuteScalarAsync($"SELECT sleep(0.3), 'marker1_{marker}'"); - var task2 = connection.ExecuteScalarAsync($"SELECT sleep(0.3), 'marker2_{marker}'"); - - await Task.WhenAll(task1, task2); - stopwatch.Stop(); - - // Quick sanity check: should take >550ms if serialized - Assert.That(stopwatch.ElapsedMilliseconds, Is.GreaterThan(550), - $"Requests should be serialized. Expected >550ms but took {stopwatch.ElapsedMilliseconds}ms"); - } } diff --git a/ClickHouse.Driver.Tests/AbstractConnectionTestFixture.cs b/ClickHouse.Driver.Tests/AbstractConnectionTestFixture.cs index 7384e85e..803b9bce 100644 --- a/ClickHouse.Driver.Tests/AbstractConnectionTestFixture.cs +++ b/ClickHouse.Driver.Tests/AbstractConnectionTestFixture.cs @@ -9,13 +9,13 @@ namespace ClickHouse.Driver.Tests; public abstract class AbstractConnectionTestFixture : IDisposable { protected readonly ClickHouseConnection connection; + protected readonly ClickHouseClient client; protected AbstractConnectionTestFixture() { - connection = TestUtilities.GetTestClickHouseConnection(); - using var command = connection.CreateCommand(); - command.CommandText = "CREATE DATABASE IF NOT EXISTS test;"; - command.ExecuteScalar(); + client = TestUtilities.GetTestClickHouseClient(); + connection = client.CreateConnection(); + client.ExecuteNonQueryAsync("CREATE DATABASE IF NOT EXISTS test;").GetAwaiter().GetResult(); } protected static string SanitizeTableName(string input) @@ -31,5 +31,9 @@ protected static string SanitizeTableName(string input) } [OneTimeTearDown] - public void Dispose() => connection?.Dispose(); + public void Dispose() + { + connection?.Dispose(); + client?.Dispose(); + } } diff --git a/ClickHouse.Driver.Tests/ActivitySourceHelperTests.cs b/ClickHouse.Driver.Tests/ActivitySourceHelperTests.cs index d4bcc8ce..bd9ba4c8 100644 --- a/ClickHouse.Driver.Tests/ActivitySourceHelperTests.cs +++ b/ClickHouse.Driver.Tests/ActivitySourceHelperTests.cs @@ -34,7 +34,7 @@ public void ShouldCreateActivity() }; ActivitySource.AddActivityListener(listener); - using var activity = connection.StartActivity("TestActivity"); + using var activity = client.StartActivity("TestActivity"); ClassicAssert.NotNull(activity); } diff --git a/ClickHouse.Driver.Tests/BulkCopy/BulkCopyTests.cs b/ClickHouse.Driver.Tests/BulkCopy/BulkCopyTests.cs index 1cc5a16a..102014f0 100644 --- a/ClickHouse.Driver.Tests/BulkCopy/BulkCopyTests.cs +++ b/ClickHouse.Driver.Tests/BulkCopy/BulkCopyTests.cs @@ -17,6 +17,7 @@ using ClickHouse.Driver.Utility; using NUnit.Framework; using NUnit.Framework.Internal; +#pragma warning disable CS0618 // Type or member is obsolete namespace ClickHouse.Driver.Tests.BulkCopy; @@ -729,22 +730,22 @@ public async Task ShouldInsertJsonFromString() [RequiredFeature(Feature.Json)] public async Task ShouldInsertJsonFromAnonymousObject_BinaryMode() { - using var binaryConnection = TestUtilities.GetTestClickHouseConnection(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); + using var binaryClient = TestUtilities.GetTestClickHouseClient(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); var targetTable = "test." + SanitizeTableName($"bulk_json_anon"); - await binaryConnection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {targetTable}"); - await binaryConnection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value JSON) ENGINE Memory"); + await binaryClient.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {targetTable}"); + await binaryClient.ExecuteNonQueryAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value JSON) ENGINE Memory"); - using var bulkCopy = new ClickHouseBulkCopy(binaryConnection) + using var bulkCopy = new ClickHouseBulkCopy(binaryClient.CreateConnection()) { DestinationTableName = targetTable, }; var obj = new { name = "test", count = 42, active = true, arrayBool = new bool[] { true, false } }; - binaryConnection.RegisterJsonSerializationType(obj.GetType()); - await bulkCopy.InitAsync(); + binaryClient.RegisterJsonSerializationType(obj.GetType()); + await bulkCopy.WriteToServerAsync([[obj]]); - using var reader = await binaryConnection.ExecuteReaderAsync($"SELECT * from {targetTable}"); + using var reader = await binaryClient.ExecuteReaderAsync($"SELECT * from {targetTable}"); Assert.That(reader.Read(), Is.True); var result = (JsonObject)reader.GetValue(0); @@ -771,7 +772,7 @@ public async Task ShouldInsertJsonFromAnonymousObject_StringMode() }; var obj = new { name = "test", count = 42, active = true, arrayBool = new bool[] { true, false } }; - connection.RegisterJsonSerializationType(obj.GetType()); + client.RegisterJsonSerializationType(obj.GetType()); await bulkCopy.WriteToServerAsync([[obj]]); using var reader = await connection.ExecuteReaderAsync($"SELECT * from {targetTable}"); @@ -838,7 +839,7 @@ public void WriteToServerAsync_WithNullDestinationTableName_ThrowsInvalidOperati await bulkCopy.WriteToServerAsync(rows); }); - Assert.That(ex.Message, Does.Contain("Destination table not set")); + Assert.That(ex.Message, Does.Contain("table is null")); } [Test] diff --git a/ClickHouse.Driver.Tests/BulkCopy/BulkCopyWithDefaultsTests.cs b/ClickHouse.Driver.Tests/BulkCopy/BulkCopyWithDefaultsTests.cs index c8fdd7cf..a80a4afe 100644 --- a/ClickHouse.Driver.Tests/BulkCopy/BulkCopyWithDefaultsTests.cs +++ b/ClickHouse.Driver.Tests/BulkCopy/BulkCopyWithDefaultsTests.cs @@ -7,6 +7,7 @@ using ClickHouse.Driver.Tests.Attributes; using ClickHouse.Driver.Utility; using NUnit.Framework; +#pragma warning disable CS0618 // Type or member is obsolete namespace ClickHouse.Driver.Tests.BulkCopy; diff --git a/ClickHouse.Driver.Tests/ClickHouseClientQueryOptionsTests.cs b/ClickHouse.Driver.Tests/ClickHouseClientQueryOptionsTests.cs new file mode 100644 index 00000000..c7765f80 --- /dev/null +++ b/ClickHouse.Driver.Tests/ClickHouseClientQueryOptionsTests.cs @@ -0,0 +1,937 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using ClickHouse.Driver.ADO; +using ClickHouse.Driver.Copy; +using ClickHouse.Driver.Tests.Utilities; + +namespace ClickHouse.Driver.Tests; + +[TestFixture] +public class ClickHouseClientQueryOptionsTests : AbstractConnectionTestFixture +{ + private string CreateTestTableName([CallerMemberName] string testName = null) + => SanitizeTableName($"test_opts_{testName}_{Guid.NewGuid():N}"); + + private async Task CreateSimpleTestTableAsync([CallerMemberName] string testName = null) + { + var tableName = $"test.{CreateTestTableName(testName)}"; + await client.ExecuteNonQueryAsync($@" + CREATE TABLE IF NOT EXISTS {tableName} + (id UInt64, value String) + ENGINE = MergeTree() ORDER BY id"); + return tableName; + } + + private async Task CreateTableWithDefaultsAsync([CallerMemberName] string testName = null) + { + var tableName = $"test.{CreateTestTableName(testName)}"; + await client.ExecuteNonQueryAsync($@" + CREATE TABLE IF NOT EXISTS {tableName} + ( + id UInt64, + name String, + created_at DateTime DEFAULT now(), + value Float32 DEFAULT 42.5 + ) + ENGINE = MergeTree() ORDER BY id"); + return tableName; + } + + private static IEnumerable GenerateTestRows(int count, ulong startId = 1) + { + for (ulong i = 0; i < (ulong)count; i++) + yield return new object[] { startId + i, $"Value_{startId + i}" }; + } + + private (ClickHouseClient client, TrackingHandler handler) CreateClientWithTracking() + { + var handler = new TrackingHandler(new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + }); + var httpClient = new HttpClient(handler); + var builder = TestUtilities.GetConnectionStringBuilder(); + var settings = new ClickHouseClientSettings(builder) + { + HttpClient = httpClient, + }; + return (new ClickHouseClient(settings), handler); + } + + [Test] + public async Task ExecuteScalarAsync_WithCustomQueryId_QueryIdAppearsInSystemQueryLog() + { + var customQueryId = $"test_query_id_{Guid.NewGuid():N}"; + var options = new QueryOptions { QueryId = customQueryId }; + + await client.ExecuteScalarAsync("SELECT 1", options: options); + + // Wait for query_log flush + await Task.Delay(500); + await client.ExecuteNonQueryAsync("SYSTEM FLUSH LOGS"); + + var count = await client.ExecuteScalarAsync( + $"SELECT count() FROM system.query_log WHERE query_id = '{customQueryId}'"); + Assert.That(count, Is.GreaterThan(0UL)); + } + + [Test] + public async Task ExecuteScalarAsync_WithoutQueryId_AutoGeneratesGuid() + { + // Use a unique marker to find this specific query + var marker = $"auto_guid_test_{Guid.NewGuid():N}"; + await client.ExecuteScalarAsync($"SELECT 42 /* {marker} */"); + + await client.ExecuteNonQueryAsync("SYSTEM FLUSH LOGS"); + + // Find the query in system.query_log - should have a valid GUID as query_id + using var reader = await client.ExecuteReaderAsync( + $"SELECT query_id FROM system.query_log WHERE query LIKE '%{marker}%' AND type = 'QueryFinish' ORDER BY event_time DESC LIMIT 1"); + + Assert.That(reader.Read(), Is.True, "Query should appear in query_log"); + var queryId = reader.GetString(0); + Assert.That(Guid.TryParse(queryId, out _), Is.True, "Query ID should be a valid GUID"); + } + + [Test] + public async Task ExecuteReaderAsync_WithQueryId_QueryIdPassedToServer() + { + var customQueryId = $"custom_reader_qid_{Guid.NewGuid():N}"; + var options = new QueryOptions { QueryId = customQueryId }; + + using var reader = await client.ExecuteReaderAsync("SELECT 1", options: options); + while (reader.Read()) { } // Consume results + + await client.ExecuteNonQueryAsync("SYSTEM FLUSH LOGS"); + + var count = await client.ExecuteScalarAsync( + $"SELECT count() FROM system.query_log WHERE query_id = '{customQueryId}'"); + Assert.That(count, Is.GreaterThan(0UL), "Custom query ID should appear in query_log"); + } + + [Test] + public async Task ExecuteScalarAsync_WithDatabaseOverride_QueriesSpecifiedDatabase() + { + await client.ExecuteNonQueryAsync("CREATE DATABASE IF NOT EXISTS test_secondary"); + + var options = new QueryOptions { Database = "test_secondary" }; + var result = await client.ExecuteScalarAsync("SELECT currentDatabase()", options: options); + + Assert.That(result, Is.EqualTo("test_secondary")); + } + + [Test] + public async Task ExecuteNonQueryAsync_WithDatabaseOverride_CreatesTableInSpecifiedDatabase() + { + await client.ExecuteNonQueryAsync("CREATE DATABASE IF NOT EXISTS test_secondary"); + var tableName = $"test_table_{Guid.NewGuid():N}"[..30]; + + try + { + var options = new QueryOptions { Database = "test_secondary" }; + await client.ExecuteNonQueryAsync( + $"CREATE TABLE {tableName} (id UInt64) ENGINE = MergeTree() ORDER BY id", + options: options); + + // Verify table exists in test_secondary + var exists = await client.ExecuteScalarAsync( + $"SELECT count() FROM system.tables WHERE database = 'test_secondary' AND name = '{tableName}'"); + Assert.That(exists, Is.EqualTo(1UL)); + } + finally + { + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS test_secondary.{tableName}"); + } + } + + [Test] + public void ExecuteScalarAsync_WithNonExistentDatabase_ThrowsServerException() + { + var options = new QueryOptions { Database = "nonexistent_database_12345" }; + + var ex = Assert.ThrowsAsync(async () => + await client.ExecuteScalarAsync("SELECT 1", options: options)); + + Assert.That(ex!.ErrorCode, Is.EqualTo(81)); // UNKNOWN_DATABASE + } + + [Test] + public async Task ExecuteReaderAsync_WithDatabaseOverride_ReturnsDataFromSpecifiedDatabase() + { + await client.ExecuteNonQueryAsync("CREATE DATABASE IF NOT EXISTS test_secondary"); + var tableName = $"test_data_{Guid.NewGuid():N}"[..30]; + + try + { + // Create and populate table in test_secondary + await client.ExecuteNonQueryAsync( + $"CREATE TABLE test_secondary.{tableName} (id UInt64, value String) ENGINE = MergeTree() ORDER BY id"); + await client.ExecuteNonQueryAsync( + $"INSERT INTO test_secondary.{tableName} VALUES (1, 'from_secondary')"); + + var options = new QueryOptions { Database = "test_secondary" }; + using var reader = await client.ExecuteReaderAsync( + $"SELECT value FROM {tableName}", options: options); + + Assert.That(reader.Read(), Is.True); + Assert.That(reader.GetString(0), Is.EqualTo("from_secondary")); + } + finally + { + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS test_secondary.{tableName}"); + } + } + + [Test] + public async Task ExecuteScalarAsync_WithMultipleRoles_CurrentRolesReturnsAllRoles() + { + if (TestUtilities.TestEnvironment != TestEnv.LocalSingleNode) + { + Assert.Ignore("Requires local_single_node environment with access storage"); + } + + var role1 = $"TEST_ROLE1_{Guid.NewGuid():N}"[..30]; + var role2 = $"TEST_ROLE2_{Guid.NewGuid():N}"[..30]; + var userName = $"test_user_{Guid.NewGuid():N}"[..30]; + var password = $"Pass_{Guid.NewGuid():N}"; + + try + { + await client.ExecuteNonQueryAsync($"CREATE ROLE IF NOT EXISTS {role1}"); + await client.ExecuteNonQueryAsync($"CREATE ROLE IF NOT EXISTS {role2}"); + await client.ExecuteNonQueryAsync($"CREATE USER IF NOT EXISTS {userName} IDENTIFIED BY '{password}'"); + await client.ExecuteNonQueryAsync($"GRANT {role1}, {role2} TO {userName}"); + + var builder = TestUtilities.GetConnectionStringBuilder(); + builder.Username = userName; + builder.Password = password; + using var userClient = new ClickHouseClient(builder.ConnectionString); + + var options = new QueryOptions { Roles = new[] { role1, role2 } }; + var result = await userClient.ExecuteScalarAsync("SELECT currentRoles()", options: options); + + Assert.That(result, Is.InstanceOf()); + var roles = (string[])result; + Assert.That(roles, Contains.Item(role1)); + Assert.That(roles, Contains.Item(role2)); + } + finally + { + await client.ExecuteNonQueryAsync($"DROP USER IF EXISTS {userName}"); + await client.ExecuteNonQueryAsync($"DROP ROLE IF EXISTS {role1}"); + await client.ExecuteNonQueryAsync($"DROP ROLE IF EXISTS {role2}"); + } + } + + [Test] + public async Task ExecuteNonQueryAsync_WithRoleThatCanInsert_InsertSucceeds() + { + if (TestUtilities.TestEnvironment != TestEnv.LocalSingleNode) + { + Assert.Ignore("Requires local_single_node environment with access storage"); + } + + var roleName = $"INSERT_ROLE_{Guid.NewGuid():N}"[..30]; + var userName = $"test_user_{Guid.NewGuid():N}"[..30]; + var password = $"Pass_{Guid.NewGuid():N}"; + var tableName = await CreateSimpleTestTableAsync(); + + try + { + await client.ExecuteNonQueryAsync($"CREATE ROLE IF NOT EXISTS {roleName}"); + await client.ExecuteNonQueryAsync($"GRANT INSERT ON {tableName} TO {roleName}"); + await client.ExecuteNonQueryAsync($"CREATE USER IF NOT EXISTS {userName} IDENTIFIED BY '{password}'"); + await client.ExecuteNonQueryAsync($"GRANT {roleName} TO {userName}"); + + var builder = TestUtilities.GetConnectionStringBuilder(); + builder.Username = userName; + builder.Password = password; + using var userClient = new ClickHouseClient(builder.ConnectionString); + + var options = new QueryOptions { Roles = new[] { roleName } }; + await userClient.ExecuteNonQueryAsync( + $"INSERT INTO {tableName} VALUES (1, 'test')", options: options); + + var count = await client.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); + Assert.That(count, Is.EqualTo(1UL)); + } + finally + { + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP USER IF EXISTS {userName}"); + await client.ExecuteNonQueryAsync($"DROP ROLE IF EXISTS {roleName}"); + } + } + + [Test] + public async Task ExecuteScalarAsync_QueryRolesOverrideConnectionRoles_QueryRolesUsed() + { + if (TestUtilities.TestEnvironment != TestEnv.LocalSingleNode) + Assert.Ignore("Requires local_single_node environment with access storage"); + + var connectionRole = $"CONN_ROLE_{Guid.NewGuid():N}"[..30]; + var queryRole = $"QUERY_ROLE_{Guid.NewGuid():N}"[..30]; + var userName = $"test_user_{Guid.NewGuid():N}"[..30]; + var password = $"Pass_{Guid.NewGuid():N}"; + + try + { + await client.ExecuteNonQueryAsync($"CREATE ROLE IF NOT EXISTS {connectionRole}"); + await client.ExecuteNonQueryAsync($"CREATE ROLE IF NOT EXISTS {queryRole}"); + await client.ExecuteNonQueryAsync($"CREATE USER IF NOT EXISTS {userName} IDENTIFIED BY '{password}'"); + await client.ExecuteNonQueryAsync($"GRANT {connectionRole}, {queryRole} TO {userName}"); + + var builder = TestUtilities.GetConnectionStringBuilder(); + builder.Username = userName; + builder.Password = password; + var settings = new ClickHouseClientSettings(builder) { Roles = new[] { connectionRole } }; + using var userClient = new ClickHouseClient(settings); + + // Query-level roles should override connection-level + var options = new QueryOptions { Roles = new[] { queryRole } }; + var result = await userClient.ExecuteScalarAsync("SELECT currentRoles()", options: options); + + var roles = (string[])result; + Assert.That(roles, Contains.Item(queryRole)); + Assert.That(roles, Does.Not.Contain(connectionRole)); + } + finally + { + await client.ExecuteNonQueryAsync($"DROP USER IF EXISTS {userName}"); + await client.ExecuteNonQueryAsync($"DROP ROLE IF EXISTS {connectionRole}"); + await client.ExecuteNonQueryAsync($"DROP ROLE IF EXISTS {queryRole}"); + } + } + + [Test] + public async Task ExecuteScalarAsync_WithMaxMemoryUsageSetting_SettingIsApplied() + { + var options = new QueryOptions + { + CustomSettings = new Dictionary { { "max_memory_usage", 1000000000 } } + }; + + var result = await client.ExecuteScalarAsync("SELECT getSetting('max_memory_usage')", options: options); + Assert.That(result, Is.EqualTo(1000000000UL)); + } + + [Test] + public async Task ExecuteScalarAsync_WithMultipleSettings_AllSettingsApplied() + { + var options = new QueryOptions + { + CustomSettings = new Dictionary + { + { "max_threads", 2 }, + { "max_block_size", 5000 } + } + }; + + using var reader = await client.ExecuteReaderAsync( + "SELECT getSetting('max_threads'), getSetting('max_block_size')", options: options); + + Assert.That(reader.Read(), Is.True); + Assert.That(reader.GetValue(0), Is.EqualTo(2UL)); + Assert.That(reader.GetValue(1), Is.EqualTo(5000UL)); + } + + [Test] + public async Task ExecuteScalarAsync_QueryCustomSettingsOverrideConnectionSettings_QuerySettingsWin() + { + var builder = TestUtilities.GetConnectionStringBuilder(); + builder["set_max_threads"] = 8; + using var settingsClient = new ClickHouseClient(builder.ConnectionString); + + var options = new QueryOptions + { + CustomSettings = new Dictionary { { "max_threads", 2 } } + }; + + // Query-level max_threads=2 should override connection-level max_threads=8 + var result = await settingsClient.ExecuteScalarAsync("SELECT getSetting('max_threads')", options: options); + Assert.That(result, Is.EqualTo(2UL)); + } + + [Test] + public async Task ExecuteScalarAsync_WithCustomHeader_HeaderIsSent() + { + var (trackedClient, handler) = CreateClientWithTracking(); + using (trackedClient) + { + var options = new QueryOptions + { + CustomHeaders = new Dictionary { { "X-Test-Header", "test-value" } } + }; + + await trackedClient.ExecuteScalarAsync("SELECT 1", options: options); + + var request = handler.Requests.Last(); + Assert.That(request.Headers.Contains("X-Test-Header"), Is.True); + Assert.That(request.Headers.GetValues("X-Test-Header").First(), Is.EqualTo("test-value")); + } + } + + [Test] + public async Task ExecuteScalarAsync_WithBlockedHeader_HeaderIsIgnored() + { + var (trackedClient, handler) = CreateClientWithTracking(); + using (trackedClient) + { + var options = new QueryOptions + { + CustomHeaders = new Dictionary { { "Authorization", "Bearer evil-token" } } + }; + + await trackedClient.ExecuteScalarAsync("SELECT 1", options: options); + + var request = handler.Requests.Last(); + // Authorization should be Basic (default), not Bearer + Assert.That(request.Headers.Authorization?.Scheme, Is.EqualTo("Basic")); + } + } + + [Test] + public async Task ExecuteScalarAsync_WithMultipleCustomHeaders_AllHeadersSent() + { + var (trackedClient, handler) = CreateClientWithTracking(); + using (trackedClient) + { + var options = new QueryOptions + { + CustomHeaders = new Dictionary + { + { "X-Header-One", "value1" }, + { "X-Header-Two", "value2" }, + { "X-Header-Three", "value3" } + } + }; + + await trackedClient.ExecuteScalarAsync("SELECT 1", options: options); + + var request = handler.Requests.Last(); + Assert.That(request.Headers.GetValues("X-Header-One").First(), Is.EqualTo("value1")); + Assert.That(request.Headers.GetValues("X-Header-Two").First(), Is.EqualTo("value2")); + Assert.That(request.Headers.GetValues("X-Header-Three").First(), Is.EqualTo("value3")); + } + } + + [Test] + public async Task ExecuteScalarAsync_CustomHeaderOverridesConnectionLevel_QueryHeaderWins() + { + var handler = new TrackingHandler(new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + }); + var httpClient = new HttpClient(handler); + var builder = TestUtilities.GetConnectionStringBuilder(); + var settings = new ClickHouseClientSettings(builder) + { + HttpClient = httpClient, + CustomHeaders = new Dictionary { { "X-Custom", "connection-level" } } + }; + using var trackedClient = new ClickHouseClient(settings); + + var options = new QueryOptions + { + CustomHeaders = new Dictionary { { "X-Custom", "query-level" } } + }; + + await trackedClient.ExecuteScalarAsync("SELECT 1", options: options); + + var request = handler.Requests.Last(); + Assert.That(request.Headers.GetValues("X-Custom").First(), Is.EqualTo("query-level")); + } + + [Test] + public async Task ExecuteNonQueryAsync_WithUseSessionTrue_TempTablePersistsAcrossQueries() + { + var sessionId = $"test_session_{Guid.NewGuid():N}"; + var options = new QueryOptions + { + UseSession = true, + SessionId = sessionId + }; + + await client.ExecuteNonQueryAsync( + "CREATE TEMPORARY TABLE temp_test_persist (id UInt8)", options: options); + + // Should be accessible with same session + var count = await client.ExecuteScalarAsync( + "SELECT count() FROM temp_test_persist", options: options); + Assert.That(count, Is.EqualTo(0UL)); + } + + [Test] + public async Task ExecuteNonQueryAsync_WithUseSessionFalse_TempTableNotAccessible() + { + // First create a temp table with session disabled + var options = new QueryOptions { UseSession = false }; + + await client.ExecuteNonQueryAsync( + "CREATE TEMPORARY TABLE temp_test_nosession (id UInt8)", options: options); + + // Without session, temp table not accessible in next query + var ex = Assert.ThrowsAsync(async () => + await client.ExecuteScalarAsync("SELECT count() FROM temp_test_nosession", options: options)); + + Assert.That(ex!.ErrorCode, Is.EqualTo(60)); // UNKNOWN_TABLE + } + + [Test] + public async Task ExecuteScalarAsync_WithSameSessionId_SharesSessionState() + { + var sessionId = $"shared_session_{Guid.NewGuid():N}"; + var options = new QueryOptions + { + UseSession = true, + SessionId = sessionId + }; + + // Set a session variable + await client.ExecuteNonQueryAsync("SET max_threads = 3", options: options); + + // Verify it's accessible in the same session + var result = await client.ExecuteScalarAsync("SELECT getSetting('max_threads')", options: options); + Assert.That(result, Is.EqualTo(3UL)); + } + + [Test] + public async Task ExecuteScalarAsync_WithBearerToken_AuthorizationHeaderSet() + { + var (trackedClient, handler) = CreateClientWithTracking(); + using (trackedClient) + { + var options = new QueryOptions { BearerToken = "test_bearer_token_123" }; + + // This will fail auth, but we can verify the header was set + try + { + await trackedClient.ExecuteScalarAsync("SELECT 1", options: options); + } + catch (ClickHouseServerException) + { + // Expected - invalid token + } + + var request = handler.Requests.Last(); + Assert.That(request.Headers.Authorization?.Scheme, Is.EqualTo("Bearer")); + Assert.That(request.Headers.Authorization?.Parameter, Is.EqualTo("test_bearer_token_123")); + } + } + + [Test] + public async Task ExecuteScalarAsync_BearerTokenOverridesBasicAuth_BearerUsed() + { + var handler = new TrackingHandler(new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + }); + var httpClient = new HttpClient(handler); + var builder = TestUtilities.GetConnectionStringBuilder(); + builder.Username = "some_user"; + builder.Password = "some_password"; + var settings = new ClickHouseClientSettings(builder) + { + HttpClient = httpClient, + }; + using var trackedClient = new ClickHouseClient(settings); + + var options = new QueryOptions { BearerToken = "my_jwt_token" }; + + try + { + await trackedClient.ExecuteScalarAsync("SELECT 1", options: options); + } + catch (ClickHouseServerException) + { + // Expected - invalid token + } + + var request = handler.Requests.Last(); + // Should use Bearer, not Basic + Assert.That(request.Headers.Authorization?.Scheme, Is.EqualTo("Bearer")); + Assert.That(request.Headers.Authorization?.Parameter, Is.EqualTo("my_jwt_token")); + } + + [Test] + public async Task ExecuteScalarAsync_WithNullBearerToken_UsesClientLevelBearerToken() + { + var handler = new TrackingHandler(new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + }); + var httpClient = new HttpClient(handler); + var builder = TestUtilities.GetConnectionStringBuilder(); + var settings = new ClickHouseClientSettings(builder) + { + HttpClient = httpClient, + BearerToken = "client_level_token" + }; + using var trackedClient = new ClickHouseClient(settings); + + // QueryOptions with null BearerToken should fall back to client-level + var options = new QueryOptions { BearerToken = null }; + + try + { + await trackedClient.ExecuteScalarAsync("SELECT 1", options: options); + } + catch (ClickHouseServerException) + { + // Expected - invalid token + } + + var request = handler.Requests.Last(); + Assert.That(request.Headers.Authorization?.Scheme, Is.EqualTo("Bearer")); + Assert.That(request.Headers.Authorization?.Parameter, Is.EqualTo("client_level_token")); + } + + [Test] + public async Task ExecuteScalarAsync_WithNullCustomHeaders_UsesClientLevelHeaders() + { + var handler = new TrackingHandler(new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + }); + var httpClient = new HttpClient(handler); + var builder = TestUtilities.GetConnectionStringBuilder(); + var settings = new ClickHouseClientSettings(builder) + { + HttpClient = httpClient, + CustomHeaders = new Dictionary { { "X-Client-Header", "client-value" } } + }; + using var trackedClient = new ClickHouseClient(settings); + + // QueryOptions with null CustomHeaders should preserve client-level headers + var options = new QueryOptions { CustomHeaders = null }; + + await trackedClient.ExecuteScalarAsync("SELECT 1", options: options); + + var request = handler.Requests.Last(); + Assert.That(request.Headers.Contains("X-Client-Header"), Is.True); + Assert.That(request.Headers.GetValues("X-Client-Header").First(), Is.EqualTo("client-value")); + } + + [Test] + public async Task ExecuteScalarAsync_WithNullRoles_UsesClientLevelRoles() + { + if (TestUtilities.TestEnvironment != TestEnv.LocalSingleNode) + Assert.Ignore("Requires local_single_node environment with access storage"); + + var clientRole = $"CLIENT_ROLE_{Guid.NewGuid():N}"[..30]; + var userName = $"test_user_{Guid.NewGuid():N}"[..30]; + var password = $"Pass_{Guid.NewGuid():N}"; + + try + { + await client.ExecuteNonQueryAsync($"CREATE ROLE IF NOT EXISTS {clientRole}"); + await client.ExecuteNonQueryAsync($"CREATE USER IF NOT EXISTS {userName} IDENTIFIED BY '{password}'"); + await client.ExecuteNonQueryAsync($"GRANT {clientRole} TO {userName}"); + + var builder = TestUtilities.GetConnectionStringBuilder(); + builder.Username = userName; + builder.Password = password; + var settings = new ClickHouseClientSettings(builder) { Roles = new[] { clientRole } }; + using var userClient = new ClickHouseClient(settings); + + // Null roles in QueryOptions should use client-level roles + var options = new QueryOptions { Roles = null }; + var result = await userClient.ExecuteScalarAsync("SELECT currentRoles()", options: options); + + var roles = (string[])result; + Assert.That(roles, Contains.Item(clientRole)); + } + finally + { + await client.ExecuteNonQueryAsync($"DROP USER IF EXISTS {userName}"); + await client.ExecuteNonQueryAsync($"DROP ROLE IF EXISTS {clientRole}"); + } + } + + [Test] + public async Task ExecuteScalarAsync_WithNullCustomSettings_UsesClientLevelSettings() + { + var builder = TestUtilities.GetConnectionStringBuilder(); + builder["set_max_threads"] = 4; + using var settingsClient = new ClickHouseClient(builder.ConnectionString); + + // Null CustomSettings should preserve client-level settings + var options = new QueryOptions { CustomSettings = null }; + + var result = await settingsClient.ExecuteScalarAsync("SELECT getSetting('max_threads')", options: options); + Assert.That(result, Is.EqualTo(4UL)); + } + + [Test] + public async Task ExecuteScalarAsync_WithMaxExecutionTime_SettingIsApplied() + { + var options = new QueryOptions { MaxExecutionTime = TimeSpan.FromSeconds(30) }; + + var result = await client.ExecuteScalarAsync("SELECT getSetting('max_execution_time')", options: options); + Assert.That(result, Is.EqualTo(30UL)); + } + + [Test] + public void ExecuteScalarAsync_WithMaxExecutionTime_LongQueryTimesOut() + { + var options = new QueryOptions { MaxExecutionTime = TimeSpan.FromSeconds(1) }; + + var ex = Assert.ThrowsAsync(async () => + await client.ExecuteScalarAsync("SELECT sleep(3)", options: options)); + + // TIMEOUT_EXCEEDED = 159 + Assert.That(ex!.ErrorCode, Is.EqualTo(159)); + } + + [Test] + public async Task InsertBinaryAsync_WithSmallBatchSize_InsertsInMultipleBatches() + { + var tableName = await CreateSimpleTestTableAsync(); + try + { + var options = new InsertOptions { BatchSize = 10 }; + var rows = GenerateTestRows(100).ToList(); + + await client.InsertBinaryAsync(tableName, new[] { "id", "value" }, rows, options); + + // All rows should be inserted regardless of batch size + var count = await client.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); + Assert.That(count, Is.EqualTo(100UL)); + + // Verify data integrity + var distinctCount = await client.ExecuteScalarAsync($"SELECT count(DISTINCT id) FROM {tableName}"); + Assert.That(distinctCount, Is.EqualTo(100UL)); + } + finally + { + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); + } + } + + [TestCase(0)] + [TestCase(-1)] + public void InsertBinaryAsync_WithInvalidBatchSize_ThrowsArgumentOutOfRangeException(int batchSize) + { + var options = new InsertOptions { BatchSize = batchSize }; + var rows = GenerateTestRows(10).ToList(); + + var ex = Assert.ThrowsAsync(async () => + await client.InsertBinaryAsync("test.dummy", new[] { "id", "value" }, rows, options)); + + Assert.That(ex!.ParamName, Is.EqualTo("options")); + } + + [TestCase(0)] + [TestCase(-1)] + public void InsertBinaryAsync_WithInvalidParallelism_ThrowsArgumentOutOfRangeException(int parallelism) + { + var options = new InsertOptions { MaxDegreeOfParallelism = parallelism }; + var rows = GenerateTestRows(10).ToList(); + + var ex = Assert.ThrowsAsync(async () => + await client.InsertBinaryAsync("test.dummy", new[] { "id", "value" }, rows, options)); + + Assert.That(ex!.ParamName, Is.EqualTo("options")); + } + + [Test] + public async Task InsertBinaryAsync_WithBatchesExceedingParallelism_ReturnsCorrectRowCount() + { + var tableName = await CreateSimpleTestTableAsync(); + try + { + var options = new InsertOptions + { + BatchSize = 10, + MaxDegreeOfParallelism = 2 + }; + var rows = GenerateTestRows(50).ToList(); + + var inserted = await client.InsertBinaryAsync( + tableName, new[] { "id", "value" }, rows, options); + + // 50 rows in 5 batches with parallelism 2: must return 50, not 20 + Assert.That(inserted, Is.EqualTo(50)); + + var count = await client.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); + Assert.That(count, Is.EqualTo(50UL)); + } + finally + { + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); + } + } + + [Test] + public async Task InsertBinaryAsync_WithRowBinaryFormat_InsertsSuccessfully() + { + var tableName = await CreateSimpleTestTableAsync(); + try + { + var options = new InsertOptions { Format = RowBinaryFormat.RowBinary }; + var rows = GenerateTestRows(50).ToList(); + + var inserted = await client.InsertBinaryAsync( + tableName, new[] { "id", "value" }, rows, options); + + Assert.That(inserted, Is.EqualTo(50)); + + var count = await client.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); + Assert.That(count, Is.EqualTo(50UL)); + } + finally + { + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); + } + } + + [Test] + public async Task InsertBinaryAsync_WithRowBinaryWithDefaultsFormat_InsertsWithDefaults() + { + var tableName = await CreateTableWithDefaultsAsync(); + try + { + var options = new InsertOptions { Format = RowBinaryFormat.RowBinaryWithDefaults }; + var rows = new List + { + new object[] { 1UL, "Name1", DateTime.UtcNow, 1.5f }, + new object[] { 2UL, "Name2", DateTime.UtcNow, 2.5f }, + }; + + var inserted = await client.InsertBinaryAsync( + tableName, new[] { "id", "name", "created_at", "value" }, rows, options); + + Assert.That(inserted, Is.EqualTo(2)); + + var count = await client.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); + Assert.That(count, Is.EqualTo(2UL)); + } + finally + { + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); + } + } + + [Test] + public async Task InsertBinaryAsync_WithRowBinaryWithDefaultsFormat_OmittedColumnsUseDefaults() + { + var tableName = await CreateTableWithDefaultsAsync(); + try + { + var options = new InsertOptions { Format = RowBinaryFormat.RowBinaryWithDefaults }; + var rows = new List + { + new object[] { 1UL, "Name1" }, + new object[] { 2UL, "Name2" }, + }; + + // Insert only id and name columns - created_at and value should use defaults + var inserted = await client.InsertBinaryAsync( + tableName, new[] { "id", "name" }, rows, options); + + Assert.That(inserted, Is.EqualTo(2)); + + // Verify default values were used + using var reader = await client.ExecuteReaderAsync( + $"SELECT id, name, value FROM {tableName} ORDER BY id"); + + reader.Read(); + Assert.That(reader.GetValue(0), Is.EqualTo(1UL)); + Assert.That(reader.GetString(1), Is.EqualTo("Name1")); + Assert.That(reader.GetFloat(2), Is.EqualTo(42.5f)); // Default value + + reader.Read(); + Assert.That(reader.GetValue(0), Is.EqualTo(2UL)); + Assert.That(reader.GetString(1), Is.EqualTo("Name2")); + Assert.That(reader.GetFloat(2), Is.EqualTo(42.5f)); // Default value + } + finally + { + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); + } + } + + [Test] + public async Task InsertBinaryAsync_WithDatabaseOverride_InsertsToSpecifiedDatabase() + { + await client.ExecuteNonQueryAsync("CREATE DATABASE IF NOT EXISTS test_secondary"); + var tableName = $"insert_test_{Guid.NewGuid():N}"[..30]; + + try + { + await client.ExecuteNonQueryAsync( + $"CREATE TABLE test_secondary.{tableName} (id UInt64, value String) ENGINE = MergeTree() ORDER BY id"); + + var options = new InsertOptions { Database = "test_secondary" }; + var rows = GenerateTestRows(10).ToList(); + + var inserted = await client.InsertBinaryAsync( + tableName, new[] { "id", "value" }, rows, options); + + Assert.That(inserted, Is.EqualTo(10)); + + var count = await client.ExecuteScalarAsync($"SELECT count() FROM test_secondary.{tableName}"); + Assert.That(count, Is.EqualTo(10UL)); + } + finally + { + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS test_secondary.{tableName}"); + } + } + + [Test] + public async Task InsertBinaryAsync_WithQueryId_QueryIdApplied() + { + var tableName = await CreateSimpleTestTableAsync(); + try + { + var customQueryId = $"insert_qid_{Guid.NewGuid():N}"; + var options = new InsertOptions { QueryId = customQueryId }; + var rows = GenerateTestRows(10).ToList(); + + await client.InsertBinaryAsync(tableName, new[] { "id", "value" }, rows, options); + + await client.ExecuteNonQueryAsync("SYSTEM FLUSH LOGS"); + + var count = await client.ExecuteScalarAsync( + $"SELECT count() FROM system.query_log WHERE query_id = '{customQueryId}'"); + Assert.That(count, Is.GreaterThan(0UL), "Custom query ID should appear in query_log"); + } + finally + { + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); + } + } + + [Test] + public async Task InsertBinaryAsync_WithCustomSettings_SettingsApplied() + { + var tableName = await CreateSimpleTestTableAsync(); + try + { + var options = new InsertOptions + { + CustomSettings = new Dictionary + { + { "insert_quorum", 0 } + } + }; + var rows = GenerateTestRows(10).ToList(); + + // Should succeed with the custom setting + var inserted = await client.InsertBinaryAsync( + tableName, new[] { "id", "value" }, rows, options); + + Assert.That(inserted, Is.EqualTo(10)); + } + finally + { + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); + } + } +} diff --git a/ClickHouse.Driver.Tests/ClickHouseClientTests.cs b/ClickHouse.Driver.Tests/ClickHouseClientTests.cs new file mode 100644 index 00000000..1e91a092 --- /dev/null +++ b/ClickHouse.Driver.Tests/ClickHouseClientTests.cs @@ -0,0 +1,166 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using ClickHouse.Driver.ADO; +using ClickHouse.Driver.Tests.Utilities; +namespace ClickHouse.Driver.Tests; + +public class ClickHouseClientTests : AbstractConnectionTestFixture +{ + [Test] + public void InsertRawStreamAsync_WithNullTable_ShouldThrowArgumentException() + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes("1,2,3")); + var ex = Assert.ThrowsAsync(async () => + await client.InsertRawStreamAsync(table: null, stream: stream, format: "CSV")); + Assert.That(ex.ParamName, Is.EqualTo("table")); + } + + [Test] + public void InsertRawStreamAsync_WithEmptyTable_ShouldThrowArgumentException() + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes("1,2,3")); + var ex = Assert.ThrowsAsync(async () => + await client.InsertRawStreamAsync(table: "", stream: stream, format: "CSV")); + Assert.That(ex.ParamName, Is.EqualTo("table")); + } + + [Test] + public void InsertRawStreamAsync_WithNullStream_ShouldThrowArgumentNullException() + { + var ex = Assert.ThrowsAsync(async () => + await client.InsertRawStreamAsync(table: "test", stream: null, format: "CSV")); + Assert.That(ex.ParamName, Is.EqualTo("stream")); + } + + [Test] + public void InsertRawStreamAsync_WithNullFormat_ShouldThrowArgumentException() + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes("1,2,3")); + var ex = Assert.ThrowsAsync(async () => + await client.InsertRawStreamAsync(table: "test", stream: stream, format: null)); + Assert.That(ex.ParamName, Is.EqualTo("format")); + } + + [Test] + public void InsertRawStreamAsync_WithEmptyFormat_ShouldThrowArgumentException() + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes("1,2,3")); + var ex = Assert.ThrowsAsync(async () => + await client.InsertRawStreamAsync(table: "test", stream: stream, format: "")); + Assert.That(ex.ParamName, Is.EqualTo("format")); + } + + [Test] + public async Task PingAsync_ReturnsTrue_WhenServerResponds() + { + var trackingHandler = new TrackingHandler(request => + { + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("Ok.\n") }; + }); + using var httpClient = new HttpClient(trackingHandler); + var settings = new ClickHouseClientSettings { Host = "localhost", HttpClient = httpClient }; + using var client = new ClickHouseClient(settings); + + var result = await client.PingAsync(); + + Assert.That(result, Is.True); + Assert.That(trackingHandler.Requests.Last().RequestUri.PathAndQuery, Is.EqualTo("/ping")); + } + + [Test] + public async Task PingAsync_ReturnsFalse_WhenServerReturnsError() + { + var trackingHandler = new TrackingHandler(request => + { + return new HttpResponseMessage(HttpStatusCode.InternalServerError); + }); + using var httpClient = new HttpClient(trackingHandler); + var settings = new ClickHouseClientSettings { Host = "localhost", HttpClient = httpClient }; + using var client = new ClickHouseClient(settings); + + var result = await client.PingAsync(); + + Assert.That(result, Is.False); + } + + [Test] + public async Task PingAsync_WithRealServer_ReturnsTrue() + { + var result = await client.PingAsync(); + + Assert.That(result, Is.True); + } + + [Test] + public void ShouldExcludePasswordFromRedactedConnectionString() + { + const string MOCK = "verysecurepassword"; + var settings = new ClickHouseClientSettings() + { + Password = MOCK, + }; + using var client = new ClickHouseClient(settings); + Assert.Multiple(() => + { + Assert.That(client.RedactedConnectionString, Is.Not.Contains($"Password={MOCK}")); + }); + } + + [Test] + public async Task Constructor_WithHttpClient_ShouldUseProvidedHttpClient() + { + var trackingHandler = new TrackingHandler(new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + }); + using var httpClient = new HttpClient(trackingHandler); + + var connectionString = TestUtilities.GetConnectionStringBuilder().ToString(); + using var client = new ClickHouseClient(connectionString, httpClient); + + await client.ExecuteScalarAsync("SELECT 1"); + + Assert.That(trackingHandler.RequestCount, Is.GreaterThan(0), "Provided HttpClient should have been used"); + } + + [Test] + public async Task Constructor_WithHttpClientFactory_ShouldUseProvidedFactory() + { + var factory = new TestHttpClientFactory(); + + var connectionString = TestUtilities.GetConnectionStringBuilder().ToString(); + using var client = new ClickHouseClient(connectionString, factory, "test-client"); + + Assert.That(factory.CreateClientCallCount, Is.EqualTo(0)); + + await client.ExecuteScalarAsync("SELECT 1"); + + Assert.Multiple(() => + { + Assert.That(factory.CreateClientCallCount, Is.GreaterThan(0), "Provided HttpClientFactory was not used"); + Assert.That(factory.LastRequestedClientName, Is.EqualTo("test-client")); + }); + } + + [Test] + public async Task Constructor_WithHttpClientFactoryDefaultName_ShouldUseEmptyName() + { + var factory = new TestHttpClientFactory(); + + var connectionString = TestUtilities.GetConnectionStringBuilder().ToString(); + using var client = new ClickHouseClient(connectionString, factory); + + await client.ExecuteScalarAsync("SELECT 1"); + + Assert.Multiple(() => + { + Assert.That(factory.CreateClientCallCount, Is.GreaterThan(0), "Provided HttpClientFactory was not used"); + Assert.That(factory.LastRequestedClientName, Is.EqualTo("")); + }); + } +} diff --git a/ClickHouse.Driver.Tests/Json/JsonTypeRegistrationTests.cs b/ClickHouse.Driver.Tests/Json/JsonTypeRegistrationTests.cs index f0438bca..801744d9 100644 --- a/ClickHouse.Driver.Tests/Json/JsonTypeRegistrationTests.cs +++ b/ClickHouse.Driver.Tests/Json/JsonTypeRegistrationTests.cs @@ -8,7 +8,7 @@ namespace ClickHouse.Driver.Tests.Json; [TestFixture] public class JsonTypeRegistrationTests { - private ClickHouseConnection connection; + private ClickHouseClient client; private class ValidPoco { @@ -47,42 +47,42 @@ private class PocoWithDuplicatePaths [SetUp] public void SetUp() { - connection = new ClickHouseConnection(); + client = new ClickHouseClient(new ClickHouseClientSettings()); } [TearDown] public void TearDown() { - connection?.Dispose(); + client?.Dispose(); } [Test] public void RegisterJsonSerializationType_WithValidPoco_ShouldSucceed() { // Should not throw - connection.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); } [Test] public void RegisterJsonSerializationType_WithNestedPoco_ShouldSucceed() { // Should not throw - nested types are registered automatically - connection.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); } [Test] public void RegisterJsonSerializationType_CalledTwice_ShouldBeIdempotent() { // Should not throw when called multiple times - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); } [Test] public void RegisterJsonSerializationType_WithUnsupportedPropertyType_ShouldThrowWithHelpfulMessage() { var ex = Assert.Throws(() => - connection.RegisterJsonSerializationType()); + client.RegisterJsonSerializationType()); Assert.That(ex.TargetType, Is.EqualTo(typeof(PocoWithUnsupportedProperty))); Assert.That(ex.PropertyName, Is.EqualTo("Pointer")); @@ -96,7 +96,7 @@ public void RegisterJsonSerializationType_WithUnsupportedPropertyType_ShouldThro public void RegisterJsonSerializationType_WithNestedUnsupportedPropertyType_ShouldThrowWithHelpfulMessage() { var ex = Assert.Throws(() => - connection.RegisterJsonSerializationType()); + client.RegisterJsonSerializationType()); // The exception should be about the nested type's unsupported property Assert.That(ex.TargetType, Is.EqualTo(typeof(PocoWithUnsupportedProperty))); @@ -108,14 +108,14 @@ public void RegisterJsonSerializationType_WithNestedUnsupportedPropertyType_Shou public void RegisterJsonSerializationType_WithNullType_ShouldThrowArgumentNullException() { Assert.Throws(() => - connection.RegisterJsonSerializationType(null)); + client.RegisterJsonSerializationType(null)); } [Test] public void RegisterJsonSerializationType_WithDuplicateJsonPaths_ShouldThrow() { var ex = Assert.Throws(() => - connection.RegisterJsonSerializationType()); + client.RegisterJsonSerializationType()); Assert.That(ex.Message, Does.Contain("shared.path")); } diff --git a/ClickHouse.Driver.Tests/Logging/ClickHouseBulkCopyLoggingTests.cs b/ClickHouse.Driver.Tests/Logging/ClickHouseBulkCopyLoggingTests.cs index 0f4e5c03..5607f8fd 100644 --- a/ClickHouse.Driver.Tests/Logging/ClickHouseBulkCopyLoggingTests.cs +++ b/ClickHouse.Driver.Tests/Logging/ClickHouseBulkCopyLoggingTests.cs @@ -10,6 +10,7 @@ using ClickHouse.Driver.Utility; using Microsoft.Extensions.Logging; using NUnit.Framework; +#pragma warning disable CS0618 // Type or member is obsolete namespace ClickHouse.Driver.Tests.Logging; @@ -52,8 +53,8 @@ public async Task WriteToServerAsync_WithDebugLogging_LogsMetadataLoading() await bulkCopy.WriteToServerAsync(rows); // Assert - Assert.That(factory.Loggers, Does.ContainKey(ClickHouseLogCategories.BulkCopy)); - var logger = factory.Loggers[ClickHouseLogCategories.BulkCopy]; + Assert.That(factory.Loggers, Does.ContainKey(ClickHouseLogCategories.Client)); + var logger = factory.Loggers[ClickHouseLogCategories.Client]; // Should have logged metadata loading start var startLog = logger.Logs.Find(l => @@ -96,7 +97,7 @@ public async Task WriteToServerAsync_WithDebugLogging_LogsBulkCopyOperations() await bulkCopy.WriteToServerAsync(rows); // Assert - var logger = factory.Loggers[ClickHouseLogCategories.BulkCopy]; + var logger = factory.Loggers[ClickHouseLogCategories.Client]; // Should have logged bulk copy start var startLog = logger.Logs.Find(l => @@ -135,7 +136,7 @@ public async Task SendBatchAsync_WithDebugLogging_LogsBatchOperations() // Assert - var logger = factory.Loggers[ClickHouseLogCategories.BulkCopy]; + var logger = factory.Loggers[ClickHouseLogCategories.Client]; // Should have logged batch sending var sendingLogs = logger.Logs.FindAll(l => @@ -149,8 +150,7 @@ public async Task SendBatchAsync_WithDebugLogging_LogsBatchOperations() var sentLogs = logger.Logs.FindAll(l => l.LogLevel == LogLevel.Debug && l.Message.Contains("Batch sent to") && - l.Message.Contains(bulkCopy.DestinationTableName) && - l.Message.Contains("Total rows written")); + l.Message.Contains(bulkCopy.DestinationTableName)); Assert.That(sentLogs.Count, Is.GreaterThan(0), "Should log batch sent completion at Debug level"); } @@ -185,7 +185,7 @@ public async Task WriteToServerAsync_WithCompletedBulkCopy_LogsTotalRows() } // Assert - var logger = factory.Loggers[ClickHouseLogCategories.BulkCopy]; + var logger = factory.Loggers[ClickHouseLogCategories.Client]; // Should have logged completion with total rows var completionLog = logger.Logs.Find(l => diff --git a/ClickHouse.Driver.Tests/Logging/ClickHouseLoggingTests.cs b/ClickHouse.Driver.Tests/Logging/ClickHouseLoggingTests.cs index fd982f52..645a9778 100644 --- a/ClickHouse.Driver.Tests/Logging/ClickHouseLoggingTests.cs +++ b/ClickHouse.Driver.Tests/Logging/ClickHouseLoggingTests.cs @@ -21,7 +21,7 @@ public void DataSource_PropagatesLoggerFactoryToConnection() try { using var connection = dataSource.CreateConnection(); - Assert.That(connection.GetLogger("test"), Is.Not.Null); + Assert.That(connection.ClickHouseClient.GetLogger("test"), Is.Not.Null); } finally { diff --git a/ClickHouse.Driver.Tests/Logging/HttpClientLoggingTests.cs b/ClickHouse.Driver.Tests/Logging/HttpClientLoggingTests.cs index bc191a31..91791dba 100644 --- a/ClickHouse.Driver.Tests/Logging/HttpClientLoggingTests.cs +++ b/ClickHouse.Driver.Tests/Logging/HttpClientLoggingTests.cs @@ -16,27 +16,17 @@ namespace ClickHouse.Driver.Tests.Logging; public class HttpClientLoggingTests { [Test] - public void ConnectionOpen_DefaultConnection_LogsHttpClientAndHandlerConfigAtTraceLevel() + public void ClientCreated_DefaultConnection_LogsHttpClientAndHandlerConfigAtTraceLevel() { var factory = new CapturingLoggerFactory(); var settings = new ClickHouseClientSettings(TestUtilities.GetConnectionStringBuilder()) { LoggerFactory = factory, }; - var connection = new ClickHouseConnection(settings); - - try - { - // This will fail to connect but should log the config - connection.Open(); - } - catch - { - // Ignore connection errors - we're just testing logging - } - - Assert.That(factory.Loggers, Does.ContainKey(ClickHouseLogCategories.Connection)); - var logger = factory.Loggers[ClickHouseLogCategories.Connection]; + var client = new ClickHouseClient(settings); + + Assert.That(factory.Loggers, Does.ContainKey(ClickHouseLogCategories.Client)); + var logger = factory.Loggers[ClickHouseLogCategories.Client]; // Should have logged HttpClient config var httpClientConfigLog = logger.Logs.Find(l => l.EventId == LoggingHelpers.HttpClientConfigEventId); @@ -53,25 +43,16 @@ public void ConnectionOpen_DefaultConnection_LogsHttpClientAndHandlerConfigAtTra } [Test] - public void ConnectionOpen_DefaultConnection_LogsSocketsHttpHandlerType() + public void ClientCreated_DefaultConnection_LogsSocketsHttpHandlerType() { var factory = new CapturingLoggerFactory(); var settings = new ClickHouseClientSettings(TestUtilities.GetConnectionStringBuilder()) { LoggerFactory = factory, }; - var connection = new ClickHouseConnection(settings); - - try - { - connection.Open(); - } - catch - { - // Ignore connection errors - } + var client = new ClickHouseClient(settings); - var logger = factory.Loggers[ClickHouseLogCategories.Connection]; + var logger = factory.Loggers[ClickHouseLogCategories.Client]; var handlerConfigLog = logger.Logs.Find(l => l.EventId == LoggingHelpers.HttpClientHandlerConfigEventId); Assert.That(handlerConfigLog, Is.Not.Null); @@ -81,7 +62,7 @@ public void ConnectionOpen_DefaultConnection_LogsSocketsHttpHandlerType() } [Test] - public void ConnectionOpen_CustomHttpClientWithHandlerSettings_LogsCustomConfiguration() + public void ClientCreated_CustomHttpClientWithHandlerSettings_LogsCustomConfiguration() { var factory = new CapturingLoggerFactory(); var handler = new HttpClientHandler @@ -96,18 +77,9 @@ public void ConnectionOpen_CustomHttpClientWithHandlerSettings_LogsCustomConfigu HttpClient = httpClient, LoggerFactory = factory, }; - var connection = new ClickHouseConnection(settings); + var client = new ClickHouseClient(settings); - try - { - connection.Open(); - } - catch - { - // Ignore connection errors - } - - var logger = factory.Loggers[ClickHouseLogCategories.Connection]; + var logger = factory.Loggers[ClickHouseLogCategories.Client]; var handlerConfigLog = logger.Logs.Find(l => l.EventId == LoggingHelpers.HttpClientHandlerConfigEventId); Assert.That(handlerConfigLog, Is.Not.Null); @@ -115,7 +87,7 @@ public void ConnectionOpen_CustomHttpClientWithHandlerSettings_LogsCustomConfigu } [Test] - public void ConnectionOpen_UseSessionEnabled_LogsHttpClientAndHandlerConfig() + public void ClientCreated_UseSessionEnabled_LogsHttpClientAndHandlerConfig() { var factory = new CapturingLoggerFactory(); var settings = new ClickHouseClientSettings(TestUtilities.GetConnectionStringBuilder()) @@ -123,19 +95,10 @@ public void ConnectionOpen_UseSessionEnabled_LogsHttpClientAndHandlerConfig() LoggerFactory = factory, UseSession = true }; - var connection = new ClickHouseConnection(settings); - - try - { - connection.Open(); - } - catch - { - // Ignore connection errors - } + var client = new ClickHouseClient(settings); - Assert.That(factory.Loggers, Does.ContainKey(ClickHouseLogCategories.Connection)); - var logger = factory.Loggers[ClickHouseLogCategories.Connection]; + Assert.That(factory.Loggers, Does.ContainKey(ClickHouseLogCategories.Client)); + var logger = factory.Loggers[ClickHouseLogCategories.Client]; // Should have logged HttpClient config even with UseSession=true var httpClientConfigLog = logger.Logs.Find(l => l.EventId == LoggingHelpers.HttpClientConfigEventId); @@ -148,7 +111,7 @@ public void ConnectionOpen_UseSessionEnabled_LogsHttpClientAndHandlerConfig() } [Test] - public void ConnectionOpen_CustomHttpClientFactory_LogsFactoryTypeName() + public void ClientCreated_CustomHttpClientFactory_LogsFactoryTypeName() { var factory = new CapturingLoggerFactory(); var customFactory = new CustomHttpClientFactory(); @@ -158,18 +121,9 @@ public void ConnectionOpen_CustomHttpClientFactory_LogsFactoryTypeName() HttpClientFactory = customFactory, LoggerFactory = factory, }; - var connection = new ClickHouseConnection(settings); + var client = new ClickHouseClient(settings); - try - { - connection.Open(); - } - catch - { - // Ignore connection errors - } - - var logger = factory.Loggers[ClickHouseLogCategories.Connection]; + var logger = factory.Loggers[ClickHouseLogCategories.Client]; var httpClientConfigLog = logger.Logs.Find(l => l.EventId == LoggingHelpers.HttpClientConfigEventId); Assert.That(httpClientConfigLog, Is.Not.Null); @@ -177,7 +131,7 @@ public void ConnectionOpen_CustomHttpClientFactory_LogsFactoryTypeName() } [Test] - public void ConnectionOpen_TraceLevelNotEnabled_DoesNotLogHttpClientConfig() + public void ClientCreated_TraceLevelNotEnabled_DoesNotLogHttpClientConfig() { // Create a logger that doesn't log Trace level var factory = new CapturingLoggerFactory(); @@ -187,25 +141,16 @@ public void ConnectionOpen_TraceLevelNotEnabled_DoesNotLogHttpClientConfig() { LoggerFactory = factory, }; - var connection = new ClickHouseConnection(settings); - - try - { - connection.Open(); - } - catch - { - // Ignore connection errors - } + var client = new ClickHouseClient(settings); // Should not have logged anything since Trace is not enabled - var logger = factory.Loggers[ClickHouseLogCategories.Connection]; + var logger = factory.Loggers[ClickHouseLogCategories.Client]; Assert.That(logger.Logs.Count(l => l.EventId == LoggingHelpers.HttpClientConfigEventId), Is.EqualTo(0)); } #if NET5_0_OR_GREATER [Test] - public void ConnectionOpen_SocketsHttpHandler_LogsSettings() + public void ClientCreated_SocketsHttpHandler_LogsSettings() { var factory = new CapturingLoggerFactory(); @@ -230,18 +175,9 @@ public void ConnectionOpen_SocketsHttpHandler_LogsSettings() HttpClient = httpClient, LoggerFactory = factory, }; - var connection = new ClickHouseConnection(settings); + var client = new ClickHouseClient(settings); - try - { - connection.Open(); - } - catch - { - // Ignore connection errors - } - - var logger = factory.Loggers[ClickHouseLogCategories.Connection]; + var logger = factory.Loggers[ClickHouseLogCategories.Client]; var handlerConfigLog = logger.Logs.Find(l => l.EventId == LoggingHelpers.HttpClientHandlerConfigEventId); Assert.That(handlerConfigLog, Is.Not.Null); @@ -263,7 +199,7 @@ public void ConnectionOpen_SocketsHttpHandler_LogsSettings() #endif [Test] - public void ConnectionOpen_UnknownHandlerType_LogsUnknownHandlerTypeAtDebugLevel() + public void ClientCreated_UnknownHandlerType_LogsUnknownHandlerTypeAtDebugLevel() { var factory = new CapturingLoggerFactory(); @@ -276,18 +212,9 @@ public void ConnectionOpen_UnknownHandlerType_LogsUnknownHandlerTypeAtDebugLevel HttpClient = httpClient, LoggerFactory = factory, }; - var connection = new ClickHouseConnection(settings); - - try - { - connection.Open(); - } - catch - { - // Ignore connection errors - } + var client = new ClickHouseClient(settings); - var logger = factory.Loggers[ClickHouseLogCategories.Connection]; + var logger = factory.Loggers[ClickHouseLogCategories.Client]; var unknownHandlerLog = logger.Logs.Find(l => l.LogLevel == LogLevel.Debug && l.Message.Contains("Unknown handler type") && diff --git a/ClickHouse.Driver.Tests/SQL/RawStreamInsertTests.cs b/ClickHouse.Driver.Tests/SQL/RawStreamInsertTests.cs index 0d27f538..3b41553c 100644 --- a/ClickHouse.Driver.Tests/SQL/RawStreamInsertTests.cs +++ b/ClickHouse.Driver.Tests/SQL/RawStreamInsertTests.cs @@ -56,7 +56,7 @@ value Float32 var columns = useColumns ? new[] { "id", "name", "value" } : null; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(data)); - using var response = await connection.InsertRawStreamAsync( + using var response = await client.InsertRawStreamAsync( table: tableName, stream: stream, format: format, @@ -88,7 +88,7 @@ away_score UInt8 """); using var stream = new MemoryStream(Encoding.UTF8.GetBytes(JsonCompactEachRowData)); - using var response = await connection.InsertRawStreamAsync( + using var response = await client.InsertRawStreamAsync( table: "test.raw_json_compact", stream: stream, format: "JSONCompactEachRow", @@ -120,18 +120,18 @@ value Float32 var queryId = "test-raw-stream-insert-" + System.Guid.NewGuid().ToString("N"); using var stream = new MemoryStream(Encoding.UTF8.GetBytes(CsvDataNoHeader)); - using var response = await connection.InsertRawStreamAsync( + using var response = await client.InsertRawStreamAsync( table: "test.raw_query_id", stream: stream, format: "CSV", columns: new[] { "id", "name", "value" }, - queryId: queryId); + options: new QueryOptions { QueryId = queryId}); Assert.That(response.IsSuccessStatusCode, Is.True); // Verify query ID from response header - var returnedQueryId = ClickHouseConnection.ExtractQueryId(response); - Assert.That(returnedQueryId, Is.EqualTo(queryId)); + var queryResult = new QueryResult(response); + Assert.That(queryResult.QueryId, Is.EqualTo(queryId)); } [Test] @@ -154,7 +154,7 @@ created DateTime DEFAULT now() """; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(partialCsv)); - using var response = await connection.InsertRawStreamAsync( + using var response = await client.InsertRawStreamAsync( table: "test.raw_partial_columns", stream: stream, format: "CSV", diff --git a/ClickHouse.Driver.Tests/Types/DynamicTests.cs b/ClickHouse.Driver.Tests/Types/DynamicTests.cs index 52005c9a..be00369d 100644 --- a/ClickHouse.Driver.Tests/Types/DynamicTests.cs +++ b/ClickHouse.Driver.Tests/Types/DynamicTests.cs @@ -14,6 +14,7 @@ using ClickHouse.Driver.Types; using ClickHouse.Driver.Utility; using NUnit.Framework.Legacy; +#pragma warning disable CS0618 // Type or member is obsolete namespace ClickHouse.Driver.Tests.Types; diff --git a/ClickHouse.Driver.Tests/Types/JsonModeTests.cs b/ClickHouse.Driver.Tests/Types/JsonModeTests.cs index 152216d9..7502a6c3 100644 --- a/ClickHouse.Driver.Tests/Types/JsonModeTests.cs +++ b/ClickHouse.Driver.Tests/Types/JsonModeTests.cs @@ -10,6 +10,7 @@ using ClickHouse.Driver.Utility; using NUnit.Framework; using NUnit.Framework.Legacy; +#pragma warning disable CS0618 // Type or member is obsolete namespace ClickHouse.Driver.Tests.Types; @@ -123,7 +124,7 @@ public async Task ShouldWriteJsonWithStringMode_BulkCopy() { DestinationTableName = tableName, }; - await bulkCopy.InitAsync(); + var rows = new[] { @@ -162,7 +163,7 @@ public async Task ShouldWriteJsonObjectWithStringMode_ViaToJsonString_BulkCopy() { DestinationTableName = tableName, }; - await bulkCopy.InitAsync(); + var jsonObj = new JsonObject { @@ -200,7 +201,7 @@ public async Task ShouldRoundTrip_WriteStringReadString() { DestinationTableName = tableName, }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync(new[] { new object[] { originalJson } }); // Read back as string @@ -234,7 +235,7 @@ public async Task ShouldWriteJsonObjectWithStringMode_BulkCopy() { DestinationTableName = tableName, }; - await bulkCopy.InitAsync(); + var jsonObj = new JsonObject { @@ -271,7 +272,7 @@ public async Task ShouldWriteJsonNodeWithStringMode_BulkCopy() { DestinationTableName = tableName, }; - await bulkCopy.InitAsync(); + // Create a JsonNode (not JsonObject) by parsing JsonNode jsonNode = JsonNode.Parse("{\"name\": \"Eve\", \"score\": 500}"); @@ -306,7 +307,7 @@ public async Task ShouldWriteSerializedPocoAsStringWithStringMode_BulkCopy() { DestinationTableName = tableName, }; - await bulkCopy.InitAsync(); + // Serialize POCO to JSON string manually var jsonString = System.Text.Json.JsonSerializer.Serialize(new { name = "Frank", score = 600 }); @@ -364,7 +365,7 @@ Id String DestinationTableName = tableName, ColumnNames = ["Document", "Id"], }; - await bulkCopy.InitAsync(); + // JSON with ISO 8601 datetime format (e.g., "2024-07-09T14:06:05.083Z") var jsonDocument = """{"$type":"AwsIamUserModel","Id":"AIDA5BUDWVFSA4MIBMX3J","DisplayName":"lambda_UpdateAlias","SystemCreationTime":"2024-07-09T14:06:05.083Z"}"""; @@ -398,7 +399,7 @@ public async Task ShouldThrowWhenWritingStringWithBinaryMode_BulkCopy() { DestinationTableName = tableName, }; - await bulkCopy.InitAsync(); + var jsonString = """{"name": "test"}"""; @@ -423,7 +424,7 @@ public async Task ShouldThrowWhenWritingJsonNodeWithBinaryMode_BulkCopy() { DestinationTableName = tableName, }; - await bulkCopy.InitAsync(); + var jsonObj = new JsonObject { ["name"] = "test" }; @@ -448,7 +449,7 @@ public async Task ShouldWritePocoWithStringMode_BulkCopy() { DestinationTableName = tableName, }; - await bulkCopy.InitAsync(); + var poco = new { name = "test", value = 42 }; await bulkCopy.WriteToServerAsync(new[] { new object[] { poco } }); @@ -527,7 +528,7 @@ public async Task Roundtrip_JsonObject_StringWriteBinaryRead_ShouldPreserveData( }; using var bulkCopy = new ClickHouseBulkCopy(connection) { DestinationTableName = tableName }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync(new[] { new object[] { 1u, original } }); using var reader = await connection.ExecuteReaderAsync($"SELECT data FROM {tableName}"); @@ -559,7 +560,7 @@ public async Task Roundtrip_String_StringWriteStringRead_ShouldPreserveData() var original = """{"name":"test","value":123}"""; using var bulkCopy = new ClickHouseBulkCopy(connection) { DestinationTableName = tableName }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync(new[] { new object[] { 1u, original } }); using var reader = await connection.ExecuteReaderAsync($"SELECT data FROM {tableName}"); @@ -597,7 +598,7 @@ public async Task Roundtrip_Poco_StringWriteBinaryRead_ShouldPreserveData() var original = new { name = "poco_test", score = 99, enabled = true }; using var bulkCopy = new ClickHouseBulkCopy(connection) { DestinationTableName = tableName }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync(new[] { new object[] { 1u, original } }); using var reader = await connection.ExecuteReaderAsync($"SELECT data FROM {tableName}"); @@ -637,7 +638,7 @@ public async Task Roundtrip_NestedJson_ShouldPreserveStructure() }; using var bulkCopy = new ClickHouseBulkCopy(connection) { DestinationTableName = tableName }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync(new[] { new object[] { 1u, original } }); using var reader = await connection.ExecuteReaderAsync($"SELECT data FROM {tableName}"); @@ -671,7 +672,7 @@ public async Task Roundtrip_MultipleRows_ShouldPreserveAllData() }; using var bulkCopy = new ClickHouseBulkCopy(connection) { DestinationTableName = tableName }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync(rows); using var reader = await connection.ExecuteReaderAsync($"SELECT id, data FROM {tableName} ORDER BY id"); diff --git a/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs b/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs index 593759ad..547cd2e4 100644 --- a/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs +++ b/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs @@ -15,6 +15,7 @@ using ClickHouse.Driver.Utility; using NUnit.Framework; using NUnit.Framework.Legacy; +#pragma warning disable CS0618 // Type or member is obsolete namespace ClickHouse.Driver.Tests.Types; @@ -25,33 +26,33 @@ public class JsonTypeTests : AbstractConnectionTestFixture public void RegisterPocoTypes() { // Register all POCO types used in JSON serialization tests - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); - connection.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); } public static IEnumerable JsonTypeTestCases() @@ -626,10 +627,10 @@ data JSON(Id Int64, Name String) public async Task Write_WithDateTimeHint_BinaryMode_ShouldPreserveDateTime() { // DateTime hints require Binary mode for proper type conversion - using var binaryConnection = TestUtilities.GetTestClickHouseConnection(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); + using var binaryClient = TestUtilities.GetTestClickHouseClient(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); var targetTable = "test.json_write_datetime_hint"; - await binaryConnection.ExecuteStatementAsync( + await binaryClient.ExecuteNonQueryAsync( $@"CREATE OR REPLACE TABLE {targetTable} ( id UInt32, data JSON(Timestamp DateTime) @@ -637,14 +638,14 @@ data JSON(Timestamp DateTime) var dt = new DateTime(2024, 6, 15, 10, 30, 45, DateTimeKind.Utc); var data = new { Timestamp = dt }; - binaryConnection.RegisterJsonSerializationType(data.GetType()); - using var bulkCopy = new ClickHouseBulkCopy(binaryConnection) + binaryClient.RegisterJsonSerializationType(data.GetType()); + using var bulkCopy = new ClickHouseBulkCopy(binaryClient.CreateConnection()) { DestinationTableName = targetTable, }; await bulkCopy.WriteToServerAsync([new object[] { 1u, data }]); - using var reader = await binaryConnection.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); + using var reader = await binaryClient.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); ClassicAssert.IsTrue(reader.Read()); var result = (JsonObject)reader.GetValue(0); var actualDateTime = result["Timestamp"].GetValue(); @@ -659,10 +660,10 @@ private class UnhintedDecimalData { public decimal Price { get; set; } } [RequiredFeature(Feature.Json)] public async Task Write_WithUnhintedDecimal_ShouldPreserveFractionalPart() { - using var binaryConnection = TestUtilities.GetTestClickHouseConnection(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); - binaryConnection.RegisterJsonSerializationType(); + using var binaryClient = TestUtilities.GetTestClickHouseClient(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); + binaryClient.RegisterJsonSerializationType(); var targetTable = "test.json_write_unhinted_decimal"; - await binaryConnection.ExecuteStatementAsync( + await binaryClient.ExecuteNonQueryAsync( $@"CREATE OR REPLACE TABLE {targetTable} ( id UInt32, data JSON @@ -670,13 +671,13 @@ data JSON var data = new UnhintedDecimalData { Price = 123.4567m }; - using var bulkCopy = new ClickHouseBulkCopy(binaryConnection) + using var bulkCopy = new ClickHouseBulkCopy(binaryClient.CreateConnection()) { DestinationTableName = targetTable, }; await bulkCopy.WriteToServerAsync([new object[] { 1u, data }]); - using var reader = await binaryConnection.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); + using var reader = await binaryClient.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); ClassicAssert.IsTrue(reader.Read()); var result = (JsonObject)reader.GetValue(0); @@ -715,24 +716,24 @@ data JSON public async Task Write_WithPocoAndJsonPathAttribute_BinaryMode_ShouldUseCustomPath() { // ClickHouseJsonPath attribute only works in Binary mode - using var binaryConnection = TestUtilities.GetTestClickHouseConnection(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); - binaryConnection.RegisterJsonSerializationType(); + using var binaryClient = TestUtilities.GetTestClickHouseClient(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); + binaryClient.RegisterJsonSerializationType(); var targetTable = "test.json_write_poco_path_attr"; - await binaryConnection.ExecuteStatementAsync( + await binaryClient.ExecuteNonQueryAsync( $@"CREATE OR REPLACE TABLE {targetTable} ( id UInt32, data JSON(user.id Int64, user.name String) ) ENGINE = Memory"); var poco = new TestPocoWithPathAttribute { UserId = 456L, UserName = "CustomPath" }; - using var bulkCopy = new ClickHouseBulkCopy(binaryConnection) + using var bulkCopy = new ClickHouseBulkCopy(binaryClient.CreateConnection()) { DestinationTableName = targetTable, }; await bulkCopy.WriteToServerAsync([new object[] { 1u, poco }]); - using var reader = await binaryConnection.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); + using var reader = await binaryClient.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); ClassicAssert.IsTrue(reader.Read()); var result = (JsonObject)reader.GetValue(0); Assert.That((long)result["user"]["id"], Is.EqualTo(456L)); @@ -744,24 +745,24 @@ data JSON(user.id Int64, user.name String) public async Task Write_WithPocoAndJsonIgnoreAttribute_BinaryMode_ShouldSkipIgnoredProperties() { // ClickHouseJsonIgnore attribute only works in Binary mode - using var binaryConnection = TestUtilities.GetTestClickHouseConnection(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); - binaryConnection.RegisterJsonSerializationType(); + using var binaryClient = TestUtilities.GetTestClickHouseClient(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); + binaryClient.RegisterJsonSerializationType(); var targetTable = "test.json_write_poco_ignore_attr"; - await binaryConnection.ExecuteStatementAsync( + await binaryClient.ExecuteNonQueryAsync( $@"CREATE OR REPLACE TABLE {targetTable} ( id UInt32, data JSON(Name String) ) ENGINE = Memory"); var poco = new TestPocoWithIgnoreAttribute { Name = "Visible", Secret = "Hidden" }; - using var bulkCopy = new ClickHouseBulkCopy(binaryConnection) + using var bulkCopy = new ClickHouseBulkCopy(binaryClient.CreateConnection()) { DestinationTableName = targetTable, }; await bulkCopy.WriteToServerAsync([new object[] { 1u, poco }]); - using var reader = await binaryConnection.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); + using var reader = await binaryClient.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); ClassicAssert.IsTrue(reader.Read()); var result = (JsonObject)reader.GetValue(0); Assert.That(result["Name"].ToString(), Is.EqualTo("Visible")); @@ -865,21 +866,21 @@ private class NullableUnhintedData public async Task Write_WithNullableHintedProperty_ShouldWriteNull() { var targetTable = "test.json_write_nullable_hinted"; - await connection.ExecuteStatementAsync( + await client.ExecuteNonQueryAsync( $@"CREATE OR REPLACE TABLE {targetTable} ( id UInt32, data JSON(Value Nullable(Int32), Name String) ) ENGINE = Memory"); var data = new NullableHintedData { Value = null, Name = "test" }; - connection.RegisterJsonSerializationType(data.GetType()); + client.RegisterJsonSerializationType(data.GetType()); using var bulkCopy = new ClickHouseBulkCopy(connection) { DestinationTableName = targetTable, }; await bulkCopy.WriteToServerAsync([new object[] { 1u, data }]); - using var reader = await connection.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); + using var reader = await client.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); ClassicAssert.IsTrue(reader.Read()); var result = (JsonObject)reader.GetValue(0); Assert.That(result["Value"], Is.Null); @@ -915,9 +916,9 @@ data JSON [Test] public async Task Write_WithNonNullableNullProperty_ShouldSkipField() { - using var binaryConnection = TestUtilities.GetTestClickHouseConnection(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); + using var binaryClient = TestUtilities.GetTestClickHouseClient(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); var targetTable = "test.json_write_nonnullable_null"; - await binaryConnection.ExecuteStatementAsync( + await binaryClient.ExecuteNonQueryAsync( $@"CREATE OR REPLACE TABLE {targetTable} ( id UInt32, data JSON @@ -925,14 +926,14 @@ data JSON // string is a reference type that can be null, but it's not Nullable var data = new NoHintData { Number = 42, Text = null, Flag = true }; - binaryConnection.RegisterJsonSerializationType(); - using var bulkCopy = new ClickHouseBulkCopy(binaryConnection) + binaryClient.RegisterJsonSerializationType(); + using var bulkCopy = new ClickHouseBulkCopy(binaryClient.CreateConnection()) { DestinationTableName = targetTable, }; await bulkCopy.WriteToServerAsync([new object[] { 1u, data }]); - using var reader = await binaryConnection.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); + using var reader = await binaryClient.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); ClassicAssert.IsTrue(reader.Read()); var result = (JsonObject)reader.GetValue(0); Assert.That(result["Number"].GetValue(), Is.EqualTo(42)); @@ -1069,11 +1070,11 @@ private class ComprehensiveTypesData [RequiredFeature(Feature.Json)] public async Task Write_WithManyUnhintedTypes_BinaryMode_ShouldInferAndWriteAllTypes() { - using var binaryConnection = TestUtilities.GetTestClickHouseConnection(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); - binaryConnection.RegisterJsonSerializationType(); + using var binaryClient = TestUtilities.GetTestClickHouseClient(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); + binaryClient.RegisterJsonSerializationType(); var targetTable = "test.json_write_comprehensive_types"; - await binaryConnection.ExecuteStatementAsync( + await binaryClient.ExecuteNonQueryAsync( $@"CREATE OR REPLACE TABLE {targetTable} ( id UInt32, data JSON @@ -1128,13 +1129,13 @@ data JSON NestedIntArray = [[1, 2], [3, 4, 5], [6]] }; - using var bulkCopy = new ClickHouseBulkCopy(binaryConnection) + using var bulkCopy = new ClickHouseBulkCopy(binaryClient.CreateConnection()) { DestinationTableName = targetTable, }; await bulkCopy.WriteToServerAsync([new object[] { 1u, data }]); - using var reader = await binaryConnection.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); + using var reader = await binaryClient.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); ClassicAssert.IsTrue(reader.Read()); var result = (JsonObject)reader.GetValue(0); @@ -1220,9 +1221,9 @@ private class SelfReferencing [RequiredFeature(Feature.Json)] public async Task Write_WithCircularReference_ShouldThrowInvalidOperationException() { - using var binaryConnection = TestUtilities.GetTestClickHouseConnection(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); - binaryConnection.RegisterJsonSerializationType(); - binaryConnection.RegisterJsonSerializationType(); + using var binaryClient = TestUtilities.GetTestClickHouseClient(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); + binaryClient.RegisterJsonSerializationType(); + binaryClient.RegisterJsonSerializationType(); var targetTable = "test.json_write_circular_ref"; await connection.ExecuteStatementAsync( $@"CREATE OR REPLACE TABLE {targetTable} ( @@ -1236,7 +1237,7 @@ data JSON a.RefB = b; b.RefA = a; - using var bulkCopy = new ClickHouseBulkCopy(binaryConnection) + using var bulkCopy = new ClickHouseBulkCopy(binaryClient.CreateConnection()) { DestinationTableName = targetTable, }; @@ -1252,10 +1253,10 @@ data JSON [RequiredFeature(Feature.Json)] public async Task Write_WithSelfReference_ShouldThrowInvalidOperationException() { - using var binaryConnection = TestUtilities.GetTestClickHouseConnection(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); - binaryConnection.RegisterJsonSerializationType(); + using var binaryClient = TestUtilities.GetTestClickHouseClient(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); + binaryClient.RegisterJsonSerializationType(); var targetTable = "test.json_write_self_ref"; - await connection.ExecuteStatementAsync( + await client.ExecuteNonQueryAsync( $@"CREATE OR REPLACE TABLE {targetTable} ( id UInt32, data JSON @@ -1265,7 +1266,7 @@ data JSON var obj = new SelfReferencing { Id = 1 }; obj.Self = obj; - using var bulkCopy = new ClickHouseBulkCopy(binaryConnection) + using var bulkCopy = new ClickHouseBulkCopy(binaryClient.CreateConnection()) { DestinationTableName = targetTable, }; @@ -1351,12 +1352,12 @@ private class WrongTypeData [RequiredFeature(Feature.Json)] public async Task Write_WithWrongPropertyType_BinaryMode_ShouldThrowFormatException() { - using var binaryConnection = TestUtilities.GetTestClickHouseConnection(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); - binaryConnection.RegisterJsonSerializationType(); + using var binaryClient = TestUtilities.GetTestClickHouseClient(jsonWriteMode: JsonWriteMode.Binary, jsonReadMode: JsonReadMode.Binary); + binaryClient.RegisterJsonSerializationType(); // POCO has string property, but schema expects Int64 var targetTable = "test.json_write_wrong_type"; - await binaryConnection.ExecuteStatementAsync( + await binaryClient.ExecuteNonQueryAsync( $@"CREATE OR REPLACE TABLE {targetTable} ( id UInt32, data JSON(Value Int64) @@ -1364,7 +1365,7 @@ data JSON(Value Int64) var data = new WrongTypeData { Value = "not a number" }; - using var bulkCopy = new ClickHouseBulkCopy(binaryConnection) + using var bulkCopy = new ClickHouseBulkCopy(binaryClient.CreateConnection()) { DestinationTableName = targetTable, }; @@ -1489,13 +1490,13 @@ private class PocoWithIndexer public async Task Write_PocoWithIndexer_ShouldIgnoreIndexer() { var targetTable = "test.json_write_indexer"; - await connection.ExecuteStatementAsync( + await client.ExecuteNonQueryAsync( $@"CREATE OR REPLACE TABLE {targetTable} ( id UInt32, data JSON(Id Int32, Name String) ) ENGINE = Memory"); - connection.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); var data = new PocoWithIndexer { Id = 42, Name = "test" }; using var bulkCopy = new ClickHouseBulkCopy(connection) diff --git a/ClickHouse.Driver.Tests/Types/TimezoneHandlingTests.cs b/ClickHouse.Driver.Tests/Types/TimezoneHandlingTests.cs index ab70f57e..b1f69bc8 100644 --- a/ClickHouse.Driver.Tests/Types/TimezoneHandlingTests.cs +++ b/ClickHouse.Driver.Tests/Types/TimezoneHandlingTests.cs @@ -8,6 +8,7 @@ using ClickHouse.Driver.Utility; using NodaTime; using NUnit.Framework; +#pragma warning disable CS0618 // Type or member is obsolete namespace ClickHouse.Driver.Tests.Types; @@ -312,7 +313,7 @@ public async Task HttpParam_UtcKind_ToAmsterdamColumnWithTzHint_PreservesInstant var utcDt = new DateTime(2024, 1, 15, 12, 0, 0, DateTimeKind.Utc); var command = connection.CreateCommand(); - command.AddParameter("dt", "DateTime('Europe/Amsterdam')", utcDt); + command.AddParameter("dt", utcDt); command.CommandText = "INSERT INTO test.datetime_http_test (dt_amsterdam) VALUES ({dt:DateTime('Europe/Amsterdam')})"; await command.ExecuteNonQueryAsync(); @@ -370,7 +371,7 @@ public async Task HttpParam_LocalKind_ToAmsterdamColumn_PreservesInstant() var expectedUtc = new DateTimeOffset(localDt).UtcDateTime; var command = connection.CreateCommand(); - command.AddParameter("dt", "DateTime('Europe/Amsterdam')", localDt); + command.AddParameter("dt", localDt); command.CommandText = "INSERT INTO test.datetime_http_test (dt_amsterdam) VALUES ({dt:DateTime('Europe/Amsterdam')})"; await command.ExecuteNonQueryAsync(); @@ -460,7 +461,7 @@ public async Task HttpParam_DateTimeOffset_ToAmsterdamColumn_PreservesInstant() var dto = new DateTimeOffset(2024, 1, 15, 15, 0, 0, TimeSpan.FromHours(3)); var command = connection.CreateCommand(); - command.AddParameter("dt", "DateTime('Europe/Amsterdam')", dto); + command.AddParameter("dt", dto); command.CommandText = "INSERT INTO test.datetime_http_test (dt_amsterdam) VALUES ({dt:DateTime('Europe/Amsterdam')})"; await command.ExecuteNonQueryAsync(); @@ -680,7 +681,7 @@ public async Task BulkCopy_LocalKind_ToAmsterdamColumn_PreservesInstant() DestinationTableName = "test.datetime_bulk_test", ColumnNames = ["dt_amsterdam"] }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync([[localDt]]); var reader = (ClickHouseDataReader)await connection.ExecuteReaderAsync("SELECT dt_amsterdam FROM test.datetime_bulk_test"); @@ -702,7 +703,7 @@ public async Task BulkCopy_UnspecifiedKind_ToNoTimezoneColumn_PreservesWallClock DestinationTableName = "test.datetime_bulk_test", ColumnNames = ["dt_no_tz"] }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync([[dt]]); var result = (DateTime)await connection.ExecuteScalarAsync("SELECT dt_no_tz FROM test.datetime_bulk_test"); @@ -721,7 +722,7 @@ public async Task BulkCopy_UtcKind_ToNoTimezoneColumn_PreservesInstant() DestinationTableName = "test.datetime_bulk_test", ColumnNames = ["dt_no_tz"] }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync([[utcDt]]); var result = (DateTime)await connection.ExecuteScalarAsync("SELECT dt_no_tz FROM test.datetime_bulk_test"); @@ -742,7 +743,7 @@ public async Task BulkCopy_LocalKind_ToNoTimezoneColumn_PreservesInstant() DestinationTableName = "test.datetime_bulk_test", ColumnNames = ["dt_no_tz"] }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync([[localDt]]); var result = (DateTime)await connection.ExecuteScalarAsync("SELECT dt_no_tz FROM test.datetime_bulk_test"); @@ -783,7 +784,7 @@ public async Task BulkCopy_DateTimeUnixEpoch_ShouldWork() DestinationTableName = "test.datetime_bulk_test", ColumnNames = ["dt_utc"] }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync([[dt]]); var result = (DateTime)await connection.ExecuteScalarAsync("SELECT dt_utc FROM test.datetime_bulk_test"); @@ -803,7 +804,7 @@ public async Task BulkCopy_DateTimeOffsetUnixEpoch_ShouldWork() DestinationTableName = "test.datetime_bulk_test", ColumnNames = ["dt_utc"] }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync([[dto]]); var result = (DateTime)await connection.ExecuteScalarAsync("SELECT dt_utc FROM test.datetime_bulk_test"); @@ -842,7 +843,7 @@ public async Task HttpParam_DateTime64_UnixEpoch_ShouldWork() var dt = DateTimeConversions.DateTimeEpochStart; var command = connection.CreateCommand(); - command.AddParameter("dt", "DateTime64(3)", dt); + command.AddParameter("dt", dt); command.CommandText = "INSERT INTO test.datetime64_http_test (dt64_utc) VALUES ({dt:DateTime64(3)})"; await command.ExecuteNonQueryAsync(); @@ -856,7 +857,7 @@ public async Task HttpParam_DateTime64_UtcKind_ToUtcColumn_PreservesInstant() var utcDt = new DateTime(2024, 1, 15, 12, 30, 45, 123, DateTimeKind.Utc); var command = connection.CreateCommand(); - command.AddParameter("dt", "DateTime64(3)", utcDt); + command.AddParameter("dt", utcDt); command.CommandText = "INSERT INTO test.datetime64_http_test (dt64_utc) VALUES ({dt:DateTime64(3)})"; await command.ExecuteNonQueryAsync(); @@ -871,7 +872,7 @@ public async Task HttpParam_DateTime64_UnspecifiedKind_ToUtcColumn_PreservesWall var dt = new DateTime(2024, 1, 15, 12, 30, 45, 123, DateTimeKind.Unspecified); var command = connection.CreateCommand(); - command.AddParameter("dt", "DateTime64(3)", dt); + command.AddParameter("dt", dt); command.CommandText = "INSERT INTO test.datetime64_http_test (dt64_utc) VALUES ({dt:DateTime64(3)})"; await command.ExecuteNonQueryAsync(); @@ -887,7 +888,7 @@ public async Task HttpParam_DateTime64_UtcKind_ToAmsterdamColumn_PreservesInstan var utcDt = new DateTime(2024, 1, 15, 12, 0, 0, DateTimeKind.Utc); var command = connection.CreateCommand(); - command.AddParameter("dt", "DateTime64(3)", utcDt); + command.AddParameter("dt", utcDt); command.CommandText = "INSERT INTO test.datetime64_http_test (dt64_amsterdam) VALUES ({dt:DateTime64(3)})"; await command.ExecuteNonQueryAsync(); @@ -904,7 +905,7 @@ public async Task HttpParam_DateTime64_LocalKind_ToUtcColumn_PreservesInstant() var expectedUtc = new DateTimeOffset(localDt).UtcDateTime; var command = connection.CreateCommand(); - command.AddParameter("dt", "DateTime64(3)", localDt); + command.AddParameter("dt", localDt); command.CommandText = "INSERT INTO test.datetime64_http_test (dt64_utc) VALUES ({dt:DateTime64(3)})"; await command.ExecuteNonQueryAsync(); @@ -919,7 +920,7 @@ public async Task HttpParam_DateTime64_DateTimeOffset_PreservesInstant() var dto = new DateTimeOffset(2024, 1, 15, 15, 0, 0, 123, TimeSpan.FromHours(3)); // 15:00 +03:00 = 12:00 UTC var command = connection.CreateCommand(); - command.AddParameter("dt", "DateTime64(3)", dto); + command.AddParameter("dt", dto); command.CommandText = "INSERT INTO test.datetime64_http_test (dt64_utc) VALUES ({dt:DateTime64(3)})"; await command.ExecuteNonQueryAsync(); @@ -964,7 +965,7 @@ public async Task BulkCopy_DateTime64_UnixEpoch_ShouldWork() DestinationTableName = "test.datetime64_bulk_test", ColumnNames = ["dt64_utc"] }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync([[dt]]); var result = (DateTime)await connection.ExecuteScalarAsync("SELECT dt64_utc FROM test.datetime64_bulk_test"); @@ -981,7 +982,7 @@ public async Task BulkCopy_DateTime64_UtcKind_ToUtcColumn_PreservesInstant() DestinationTableName = "test.datetime64_bulk_test", ColumnNames = ["dt64_utc"] }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync([[utcDt]]); var result = (DateTime)await connection.ExecuteScalarAsync("SELECT dt64_utc FROM test.datetime64_bulk_test"); @@ -999,7 +1000,7 @@ public async Task BulkCopy_DateTime64_UnspecifiedKind_ToUtcColumn_PreservesWallC DestinationTableName = "test.datetime64_bulk_test", ColumnNames = ["dt64_utc"] }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync([[dt]]); var result = (DateTime)await connection.ExecuteScalarAsync("SELECT dt64_utc FROM test.datetime64_bulk_test"); @@ -1021,7 +1022,7 @@ public async Task BulkCopy_DateTime64_UtcKind_ToAmsterdamColumn_PreservesInstant DestinationTableName = "test.datetime64_bulk_test", ColumnNames = ["dt64_amsterdam"] }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync([[utcDt]]); var result = (DateTime)await connection.ExecuteScalarAsync("SELECT dt64_amsterdam FROM test.datetime64_bulk_test"); @@ -1040,7 +1041,7 @@ public async Task BulkCopy_DateTime64_LocalKind_ToUtcColumn_PreservesInstant() DestinationTableName = "test.datetime64_bulk_test", ColumnNames = ["dt64_utc"] }; - await bulkCopy.InitAsync(); + await bulkCopy.WriteToServerAsync([[localDt]]); var result = (DateTime)await connection.ExecuteScalarAsync("SELECT dt64_utc FROM test.datetime64_bulk_test"); diff --git a/ClickHouse.Driver.Tests/Utilities/TestUtilities.cs b/ClickHouse.Driver.Tests/Utilities/TestUtilities.cs index baefb0a3..dc84f308 100644 --- a/ClickHouse.Driver.Tests/Utilities/TestUtilities.cs +++ b/ClickHouse.Driver.Tests/Utilities/TestUtilities.cs @@ -99,11 +99,7 @@ public static void AssertEqual(object expected, object result) } } - /// - /// Utility method to allow to redirect ClickHouse connections to different machine, in case of Windows development environment - /// - /// - public static ClickHouseConnection GetTestClickHouseConnection(bool compression = true, bool session = false, bool customDecimals = true, string password = null, bool useFormDataParameters = false, JsonReadMode jsonReadMode = JsonReadMode.Binary, JsonWriteMode jsonWriteMode = JsonWriteMode.String) + public static ClickHouseClientSettings GetTestClickHouseClientSettings(bool compression = true, bool session = false, bool customDecimals = true, string password = null, bool useFormDataParameters = false, JsonReadMode jsonReadMode = JsonReadMode.Binary, JsonWriteMode jsonWriteMode = JsonWriteMode.String) { var builder = GetConnectionStringBuilder(); builder.Compression = compression; @@ -142,7 +138,7 @@ public static ClickHouseConnection GetTestClickHouseConnection(bool compression } if (SupportedFeatures.HasFlag(Feature.Geometry)) { - // Revisit this if the Geometry type is updated to not require this setting in the future + // Revisit this if the Geometry type is updated to not require this setting in the future // it could cause problems by hiding other issues builder["set_allow_suspicious_variant_types"] = 1; } @@ -152,14 +148,30 @@ public static ClickHouseConnection GetTestClickHouseConnection(bool compression builder["set_allow_experimental_qbit_type"] = 1; } - var settings = new ClickHouseClientSettings(builder) + return new ClickHouseClientSettings(builder) { UseFormDataParameters = useFormDataParameters }; + } - var connection = new ClickHouseConnection(settings); - connection.Open(); - return connection; + public static ClickHouseClient GetTestClickHouseClient(bool compression = true, bool session = false, bool customDecimals = true, string password = null, bool useFormDataParameters = false, JsonReadMode jsonReadMode = JsonReadMode.Binary, JsonWriteMode jsonWriteMode = JsonWriteMode.String) + { + var settings = GetTestClickHouseClientSettings(compression, session, customDecimals, password, useFormDataParameters, jsonReadMode, jsonWriteMode); + return new ClickHouseClient(settings); + } + + /// + /// Utility method to allow to redirect ClickHouse connections to different machine, in case of Windows development environment + /// + /// + public static ClickHouseConnection GetTestClickHouseConnection(bool compression = true, bool session = false, bool customDecimals = true, string password = null, bool useFormDataParameters = false, JsonReadMode jsonReadMode = JsonReadMode.Binary, JsonWriteMode jsonWriteMode = JsonWriteMode.String) + { + // Construct from settings so the connection owns its internal ClickHouseClient. + // Using client.CreateConnection() would set ownsClient=false, leaking the client on dispose. + var settings = GetTestClickHouseClientSettings(compression, session, customDecimals, password, useFormDataParameters, jsonReadMode, jsonWriteMode); + var conn = new ClickHouseConnection(settings); + conn.Open(); + return conn; } public static ClickHouseConnectionStringBuilder GetConnectionStringBuilder() diff --git a/ClickHouse.Driver/ADO/ClickHouseCommand.cs b/ClickHouse.Driver/ADO/ClickHouseCommand.cs index 986abb5d..3898755e 100644 --- a/ClickHouse.Driver/ADO/ClickHouseCommand.cs +++ b/ClickHouse.Driver/ADO/ClickHouseCommand.cs @@ -2,23 +2,13 @@ using System.Collections.Generic; using System.Data; using System.Data.Common; -using System.Diagnostics; -using System.Linq; -using System.Net; using System.Net.Http; -using System.Net.Http.Headers; using System.Text; -using System.Text.Json; using System.Threading; using System.Threading.Tasks; using ClickHouse.Driver.ADO.Parameters; using ClickHouse.Driver.ADO.Readers; -using ClickHouse.Driver.Diagnostic; using ClickHouse.Driver.Formats; -using ClickHouse.Driver.Json; -using ClickHouse.Driver.Logging; -using ClickHouse.Driver.Utility; -using Microsoft.Extensions.Logging; namespace ClickHouse.Driver.ADO; @@ -130,11 +120,7 @@ public override async Task ExecuteNonQueryAsync(CancellationToken cancellat using var lcts = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, cancellationToken); using var response = await PostSqlQueryAsync(CommandText, lcts.Token).ConfigureAwait(false); -#if NET5_0_OR_GREATER using var reader = new ExtendedBinaryReader(await response.Content.ReadAsStreamAsync(lcts.Token).ConfigureAwait(false)); -#else - using var reader = new ExtendedBinaryReader(await response.Content.ReadAsStreamAsync().ConfigureAwait(false)); -#endif return reader.PeekChar() != -1 ? reader.Read7BitEncodedInt() : 0; } @@ -209,176 +195,30 @@ protected override async Task ExecuteDbDataReaderAsync(CommandBeha default: break; } + var result = await PostSqlQueryAsync(sqlBuilder.ToString(), lcts.Token).ConfigureAwait(false); - return await ClickHouseDataReader.FromHttpResponseAsync(result, connection.TypeSettings).ConfigureAwait(false); + return await ClickHouseDataReader.FromHttpResponseAsync(result, connection.ClickHouseClient.TypeSettings).ConfigureAwait(false); } private async Task PostSqlQueryAsync(string sqlQuery, CancellationToken token) { - if (connection == null) - throw new InvalidOperationException("Connection not set"); - - using var activity = connection.StartActivity("PostSqlQueryAsync"); - - var uriBuilder = connection.CreateUriBuilder(); - uriBuilder.QueryId = QueryId; - uriBuilder.CommandQueryStringParameters = customSettings; - uriBuilder.CommandRoles = roles; - - var logger = connection.GetLogger(ClickHouseLogCategories.Command); - var isDebugLoggingEnabled = logger?.IsEnabled(LogLevel.Debug) ?? false; - Stopwatch stopwatch = null; - if (isDebugLoggingEnabled) - { - stopwatch = Stopwatch.StartNew(); - logger.LogDebug("Executing SQL query. QueryId: {QueryId}", uriBuilder.GetEffectiveQueryId()); - } - - await connection.EnsureOpenAsync().ConfigureAwait(false); // Preserve old behavior - - using var postMessage = connection.UseFormDataParameters - ? BuildHttpRequestMessageWithFormData( - sqlQuery: sqlQuery, - uriBuilder: uriBuilder) - : BuildHttpRequestMessageWithQueryParams( - sqlQuery: sqlQuery, - uriBuilder: uriBuilder); - - activity.SetQuery(sqlQuery); - - HttpResponseMessage response = null; - try - { - response = await connection - .SendAsync(postMessage, HttpCompletionOption.ResponseHeadersRead, token) - .ConfigureAwait(false); - - QueryId = ClickHouseConnection.ExtractQueryId(response); - QueryStats = ExtractQueryStats(response); - ServerTimezone = ExtractTimezone(response); - activity.SetQueryStats(QueryStats); - - var handled = await ClickHouseConnection.HandleError(response, sqlQuery, activity).ConfigureAwait(false); - - if (isDebugLoggingEnabled) - { - LogQuerySuccess(stopwatch, QueryId, logger); - } - - return handled; - } - catch (Exception ex) - { - logger?.LogError(ex, "Query (QueryId: {QueryId}) failed.", uriBuilder.GetEffectiveQueryId()); - activity?.SetException(ex); - throw; - } - } - - private void LogQuerySuccess(Stopwatch stopwatch, string queryId, ILogger logger) - { - stopwatch.Stop(); - logger.LogDebug( - "Query (QueryId: {QueryId}) succeeded in {ElapsedMilliseconds:F2} ms. Query Stats: {QueryStats}", - queryId, - stopwatch.Elapsed.TotalMilliseconds, - QueryStats); - } - - private HttpRequestMessage BuildHttpRequestMessageWithQueryParams(string sqlQuery, ClickHouseUriBuilder uriBuilder) - { - if (commandParameters != null) - { - var typeHints = SqlParameterTypeExtractor.ExtractTypeHints(sqlQuery); - sqlQuery = commandParameters.ReplacePlaceholders(sqlQuery); - foreach (ClickHouseDbParameter parameter in commandParameters) - { - typeHints.TryGetValue(parameter.ParameterName, out var sqlTypeHint); - uriBuilder.AddSqlQueryParameter( - parameter.ParameterName, - HttpParameterFormatter.Format(parameter, connection.TypeSettings, sqlTypeHint)); - } - } - - var uri = uriBuilder.ToString(); - - var postMessage = new HttpRequestMessage(HttpMethod.Post, uri); - - connection.AddDefaultHttpHeaders(postMessage.Headers, bearerTokenOverride: BearerToken); - HttpContent content = new StringContent(sqlQuery); - content.Headers.ContentType = new MediaTypeHeaderValue("text/sql"); - if (connection.UseCompression) - { - content = new CompressedContent(content, DecompressionMethods.GZip); - } - - postMessage.Content = content; - - return postMessage; - } - - private HttpRequestMessage BuildHttpRequestMessageWithFormData(string sqlQuery, ClickHouseUriBuilder uriBuilder) - { - var content = new MultipartFormDataContent(); - - if (commandParameters != null) - { - var typeHints = SqlParameterTypeExtractor.ExtractTypeHints(sqlQuery); - sqlQuery = commandParameters.ReplacePlaceholders(sqlQuery); - - foreach (ClickHouseDbParameter parameter in commandParameters) - { - typeHints.TryGetValue(parameter.ParameterName, out var sqlTypeHint); - content.Add( - content: new StringContent(HttpParameterFormatter.Format(parameter, connection.TypeSettings, sqlTypeHint)), - name: $"param_{parameter.ParameterName}"); - } - } - - content.Add( - content: new StringContent(sqlQuery), - name: "query"); - - var uri = uriBuilder.ToString(); - - var postMessage = new HttpRequestMessage(HttpMethod.Post, uri); - - connection.AddDefaultHttpHeaders(postMessage.Headers, bearerTokenOverride: BearerToken); - - postMessage.Content = content; - - return postMessage; - } - - private static readonly JsonSerializerOptions SummarySerializerOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = new SnakeCaseNamingPolicy(), - NumberHandling = System.Text.Json.Serialization.JsonNumberHandling.AllowReadingFromString, - }; - - private static QueryStats ExtractQueryStats(HttpResponseMessage response) - { - try - { - const string summaryHeader = "X-ClickHouse-Summary"; - if (response.Headers.TryGetValues(summaryHeader, out var values)) - { - return JsonSerializer.Deserialize(values.First(), SummarySerializerOptions); - } - } - catch - { - } - return null; + var options = BuildQueryOptions(); + QueryResult result = await connection.ClickHouseClient.PostSqlQueryAsync(sqlQuery, commandParameters, options, token).ConfigureAwait(false); + QueryId = result.QueryId; + QueryStats = result.QueryStats; + ServerTimezone = result.ServerTimezone; + return result.HttpResponseMessage; } - private static string ExtractTimezone(HttpResponseMessage response) + private QueryOptions BuildQueryOptions() { - const string timezoneHeader = "X-ClickHouse-Timezone"; - if (response.Headers.TryGetValues(timezoneHeader, out var values)) + return new QueryOptions { - return values.FirstOrDefault(); - } - return null; + QueryId = QueryId, + BearerToken = BearerToken, + Database = connection?.Database, + Roles = roles?.Count > 0 ? roles : null, + CustomSettings = customSettings?.Count > 0 ? customSettings : null, + }; } } diff --git a/ClickHouse.Driver/ADO/ClickHouseConnection.cs b/ClickHouse.Driver/ADO/ClickHouseConnection.cs index bc962fba..736abc79 100644 --- a/ClickHouse.Driver/ADO/ClickHouseConnection.cs +++ b/ClickHouse.Driver/ADO/ClickHouseConnection.cs @@ -1,20 +1,11 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Data; using System.Data.Common; -using System.Diagnostics; -using System.Globalization; using System.IO; -using System.Linq; using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; using System.Threading; using System.Threading.Tasks; -using ClickHouse.Driver.Diagnostic; -using ClickHouse.Driver.Http; -using ClickHouse.Driver.Json; using ClickHouse.Driver.Logging; using ClickHouse.Driver.Utility; using Microsoft.Extensions.Logging; @@ -23,38 +14,32 @@ namespace ClickHouse.Driver.ADO; public class ClickHouseConnection : DbConnection, IClickHouseConnection, ICloneable, IDisposable { - private const string CustomSettingPrefix = "set_"; - - private readonly List disposables = new(); - private readonly ConcurrentDictionary> loggerCache = new(); - private readonly JsonTypeRegistry jsonTypeRegistry = new(); private volatile ConnectionState state = ConnectionState.Closed; // Not an autoproperty because of interface implementation + private bool ownsClient = true; + private string selectedDatabase; - // HTTP client management - private HttpClient providedHttpClient; - private IHttpClientFactory providedHttpClientFactory; - private string httpClientName; - private IHttpClientFactory httpClientFactory; - - /// - /// This lock is used to serialize requests when using sessions, - /// because ClickHouse does not support using the same session from multiple connections. - /// - private SemaphoreSlim sessionRequestLock; - - // Configuration fields - private Uri serverUri; + internal ClickHouseClient ClickHouseClient { get; private set; } public ClickHouseConnection() : this(string.Empty) { } + /// + /// Initializes a new instance of the class. + /// Create a ClickHouseConnection based on a connection string, optionally skipping certificate validation. This will create a new internal connection pool. + /// It is recommended to pass a ClickHouseClient to the Connection, or to use ClickHouseDataSource instead. + /// public ClickHouseConnection(string connectionString) : this(ClickHouseClientSettings.FromConnectionString(connectionString)) { } + /// + /// Initializes a new instance of the class. + /// Create a ClickHouseConnection based on a connection string, optionally skipping certificate validation. This will create a new internal connection pool. + /// It is recommended to pass a ClickHouseClient to the Connection, or to use ClickHouseDataSource instead. + /// public ClickHouseConnection(string connectionString, bool skipServerCertificateValidation) { var settings = new ClickHouseClientSettings(connectionString) @@ -62,7 +47,7 @@ public ClickHouseConnection(string connectionString, bool skipServerCertificateV SkipServerCertificateValidation = skipServerCertificateValidation, }; - Settings = settings; + ClickHouseClient = new ClickHouseClient(settings); } /// @@ -78,7 +63,7 @@ public ClickHouseConnection(string connectionString, HttpClient httpClient) HttpClient = httpClient, }; - Settings = settings; + ClickHouseClient = new ClickHouseClient(settings); } /// @@ -124,60 +109,57 @@ public ClickHouseConnection(string connectionString, IHttpClientFactory httpClie HttpClientName = httpClientName, }; - Settings = settings; + ClickHouseClient = new ClickHouseClient(settings); } /// - /// Initializes a new instance of the class using ClickHouseClientSettings. + /// Initializes a new instance of the class using an existing client. + /// The connection does not own the client and will not dispose it. /// - /// The settings to use for this connection - public ClickHouseConnection(ClickHouseClientSettings settings) + /// The client to use for this connection. + public ClickHouseConnection(ClickHouseClient clickHouseClient) { - if (settings == null) - throw new ArgumentNullException(nameof(settings)); - Settings = settings; + ClickHouseClient = clickHouseClient; + ownsClient = false; } - private ILoggerFactory loggerFactory; - /// - /// Gets a logger for the specified category name. - /// Loggers are lazily instantiated and cached for performance. + /// Initializes a new instance of the class using ClickHouseClientSettings. /// - /// The category name for the logger. - /// An ILogger instance, or null if no LoggerFactory is configured. - internal ILogger GetLogger(string categoryName) + /// The settings to use for this connection + public ClickHouseConnection(ClickHouseClientSettings settings) { - if (loggerFactory == null) - return null; - - // Cache is used here in case the logger factory implementation provided does not do caching on its own - return loggerCache.GetOrAdd( - categoryName, - key => new Lazy(() => loggerFactory.CreateLogger(key))).Value; + if (settings == null) + throw new ArgumentNullException(nameof(settings)); + ClickHouseClient = new ClickHouseClient(settings); } /// /// Gets the string defining connection settings for ClickHouse server /// Example: Host=localhost;Port=8123;Username=default;Password=123;Compression=true /// It is generally recommended create a new connection instead of modifying the settings of an existing one. + /// Setting a new connection string will create a new ClickHouseClient instance under the hood. /// public sealed override string ConnectionString { - get => ConnectionStringBuilder.ToString(); + // Some users require ConnectionString to be set after creation + get => ClickHouseClient.ConnectionStringBuilder.ToString(); set => Settings = new ClickHouseClientSettings(value); } + /// + /// Gets or sets the connection settings. + /// Warning: updating the settings will create a new ClickHouseClient instance under the hood. + /// public ClickHouseClientSettings Settings { - get; + get => ClickHouseClient.Settings; set { if (State == ConnectionState.Open) throw new InvalidOperationException("Cannot change settings while connection is open."); - field = value; - ApplySettings(); + ApplySettings(value); } } @@ -185,21 +167,7 @@ public ClickHouseClientSettings Settings public override ConnectionState State => state; - public override string Database => Settings.Database; - - internal string Username => Settings.Username; - - internal Uri ServerUri => serverUri; - - internal string RedactedConnectionString - { - get - { - var builder = ConnectionStringBuilder; - builder.Password = "****"; - return builder.ToString(); - } - } + public override string Database => selectedDatabase ?? Settings.Database; public override string DataSource { get; } @@ -212,207 +180,39 @@ internal string RedactedConnectionString public bool UseFormDataParameters => Settings.UseFormDataParameters; - private void ApplySettings() - { - Settings.Validate(); - - serverUri = new UriBuilder(Settings.Protocol, Settings.Host, Settings.Port, Settings.Path ?? string.Empty).Uri; - - // HttpClientFactory/HttpClient - providedHttpClient = Settings.HttpClient; - providedHttpClientFactory = Settings.HttpClientFactory; - httpClientName = Settings.HttpClientName; - - // Logging - loggerCache.Clear(); - loggerFactory = Settings.LoggerFactory; - -#if NET5_0_OR_GREATER - // Debug mode - if (Settings.EnableDebugMode) - { - TraceHelper.Activate(Settings.LoggerFactory); - } -#endif - - ResetHttpClientFactory(); - } - - private void ResetHttpClientFactory() + private void ApplySettings(ClickHouseClientSettings settings) { - // If current httpClientFactory is owned by this connection, dispose of it - if (httpClientFactory is IDisposable d && disposables.Contains(d)) - { - GetLogger(ClickHouseLogCategories.Connection)?.LogDebug("Disposing HTTP client factory owned by connection."); - d.Dispose(); - disposables.Remove(d); - } - - // Dispose and reset the session lock if it exists - sessionRequestLock?.Dispose(); - sessionRequestLock = null; - - // If we have a HttpClient provided, use it - if (providedHttpClient != null) - { - GetLogger(ClickHouseLogCategories.Connection)?.LogInformation("Using provided HttpClient instance."); - httpClientFactory = new CannedHttpClientFactory(providedHttpClient); - } - - // If we have a provided client factory, use that - else if (providedHttpClientFactory != null) - { - GetLogger(ClickHouseLogCategories.Connection)?.LogInformation("Using provided IHttpClientFactory instance."); - httpClientFactory = providedHttpClientFactory; - } - - // If sessions are enabled without a provided client/factory, use single connection factory - else if (Settings.UseSession && !string.IsNullOrEmpty(Settings.SessionId)) - { - GetLogger(ClickHouseLogCategories.Connection)?.LogInformation("Creating single-connection HttpClientFactory for session {SessionId}.", Settings.SessionId); - var factory = new SingleConnectionHttpClientFactory(SkipServerCertificateValidation) { Timeout = Settings.Timeout }; - disposables.Add(factory); - httpClientFactory = factory; - } - - // Default case - use default connection pool - else - { - GetLogger(ClickHouseLogCategories.Connection)?.LogInformation("Using default pooled HttpClientFactory."); - httpClientFactory = new DefaultPoolHttpClientFactory(SkipServerCertificateValidation) { Timeout = Settings.Timeout }; - } - - // Initialize session lock if sessions are enabled (regardless of how HttpClient is configured) - if (Settings.UseSession && !string.IsNullOrEmpty(Settings.SessionId)) - { - sessionRequestLock = new SemaphoreSlim(1, 1); - GetLogger(ClickHouseLogCategories.Connection)?.LogDebug("Session request serialization enabled for session {SessionId}.", Settings.SessionId); - } + settings.Validate(); + DisposeClientIfOwned(); + ClickHouseClient = new ClickHouseClient(settings); + ownsClient = true; } public override DataTable GetSchema() => GetSchema(null, null); public override DataTable GetSchema(string collectionName) => GetSchema(collectionName, null); - public override DataTable GetSchema(string collectionName, string[] restrictionValues) => SchemaDescriber.DescribeSchema(this, collectionName, restrictionValues); - - internal static async Task HandleError(HttpResponseMessage response, string query, Activity activity) - { - if (response.IsSuccessStatusCode) - { - activity.SetSuccess(); - return response; - } - - var error = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - var ex = ClickHouseServerException.FromServerResponse(error, query); - activity.SetException(ex); - throw ex; - } + public override DataTable GetSchema(string collectionName, string[] restrictionValues) => + SchemaDescriber.DescribeSchema(this, collectionName, restrictionValues); public override void ChangeDatabase(string databaseName) { - Settings.Database = databaseName; + selectedDatabase = databaseName; } - public object Clone() => new ClickHouseConnection(ConnectionString); + public object Clone() => new ClickHouseConnection(ClickHouseClient); public override void Close() => state = ConnectionState.Closed; - /// - /// Pings the ClickHouse server to check if it is available. - /// Requires connection to be in Open state. - /// - /// Cancellation token - /// True if the server responds successfully, false otherwise - public async Task PingAsync(CancellationToken cancellationToken = default) - { - if (State != ConnectionState.Open) - throw new InvalidOperationException("Connection must be open before calling PingAsync."); - - try - { - var pingUri = new Uri(serverUri, "ping"); - var request = new HttpRequestMessage(HttpMethod.Get, pingUri); - AddDefaultHttpHeaders(request.Headers); - - using var response = await SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); - return response.IsSuccessStatusCode; - } - catch (Exception ex) - { - GetLogger(ClickHouseLogCategories.Connection)?.LogWarning(ex, "Ping to {Endpoint} failed.", serverUri); - return false; - } - } - public override void Open() => OpenAsync().ConfigureAwait(false).GetAwaiter().GetResult(); public override Task OpenAsync(CancellationToken cancellationToken) { - GetLogger(ClickHouseLogCategories.Connection)?.LogDebug("Opening ClickHouse connection to {Endpoint}.", serverUri); - LoggingHelpers.LogHttpClientConfiguration(GetLogger(ClickHouseLogCategories.Connection), httpClientFactory); - - if (State == ConnectionState.Open) - return Task.CompletedTask; - + ClickHouseClient.GetLogger(ClickHouseLogCategories.Connection)?.LogDebug("Opening ClickHouse connection to {Endpoint}.", ClickHouseClient.ServerUri); state = ConnectionState.Open; - GetLogger(ClickHouseLogCategories.Connection)?.LogDebug("Connection to {Endpoint} opened.", serverUri); return Task.CompletedTask; } - /// - /// Inserts raw data from a stream into a ClickHouse table. - /// This can be used for inserting data in formats like CSV, JSON, Parquet, etc. directly from files or other streams. - /// - /// The destination table name - /// The stream containing the data to insert - /// The ClickHouse format of the data (e.g., "CSV", "JSONEachRow", "Parquet"). See ClickHouse Formats for the full list. - /// Optional list of column names. If null, all columns are assumed in table order - /// Whether to compress the stream before sending (default: true) - /// Optional query ID for tracking. A query id will be generated if left null or empty - /// Cancellation token - /// Task-wrapped HttpResponseMessage object - public async Task InsertRawStreamAsync( - string table, - Stream stream, - string format, - IEnumerable columns = null, - bool useCompression = true, - string queryId = null, - CancellationToken token = default) - { - if (string.IsNullOrEmpty(table)) - throw new ArgumentException("Table name cannot be null or empty", nameof(table)); - if (stream == null) - throw new ArgumentNullException(nameof(stream)); - if (string.IsNullOrEmpty(format)) - throw new ArgumentException("Format cannot be null or empty", nameof(format)); - - await EnsureOpenAsync().ConfigureAwait(false); - - var columnList = columns != null ? $"({string.Join(", ", columns)})" : string.Empty; - var query = $"INSERT INTO {table} {columnList} FORMAT {format}"; - - HttpContent content = new StreamContent(stream); - if (useCompression) - { - // CompressedContent handles compression and adds Content-Encoding header - content = new CompressedContent(content, System.Net.DecompressionMethods.GZip); - } - - // Pass isCompressed=false since CompressedContent already adds the Content-Encoding header - try - { - return await PostStreamAsync(query, content, isCompressed: false, queryId, token).ConfigureAwait(false); - } - catch - { - content.Dispose(); - throw; - } - } - /// /// Warning: implementation-specific API. Exposed to allow custom optimizations /// May change in future versions @@ -425,8 +225,8 @@ public async Task InsertRawStreamAsync( /// Task-wrapped HttpResponseMessage object public async Task PostStreamAsync(string sql, Stream data, bool isCompressed, CancellationToken token, string queryId = null) { - var content = new StreamContent(data); - return await PostStreamAsync(sql, content, isCompressed, queryId, token).ConfigureAwait(false); + var options = GetQueryOptionsWithQueryId(queryId); + return await ClickHouseClient.PostStreamAsync(sql, data, isCompressed, token, options).ConfigureAwait(false); } /// @@ -441,42 +241,21 @@ public async Task PostStreamAsync(string sql, Stream data, /// Task-wrapped HttpResponseMessage object public async Task PostStreamAsync(string sql, Func callback, bool isCompressed, CancellationToken token, string queryId = null) { - var content = new StreamCallbackContent(callback, token); - return await PostStreamAsync(sql, content, isCompressed, queryId, token).ConfigureAwait(false); + var options = GetQueryOptionsWithQueryId(queryId); + return await ClickHouseClient.PostStreamAsync(sql, callback, isCompressed, token, options).ConfigureAwait(false); } - private async Task PostStreamAsync(string sql, HttpContent content, bool isCompressed, string queryId, CancellationToken token) + private static QueryOptions GetQueryOptionsWithQueryId(string queryId) { - using var activity = this.StartActivity("PostStreamAsync"); - activity.SetQuery(sql); - - var builder = CreateUriBuilder(sql); - builder.QueryId = queryId; - - using var postMessage = new HttpRequestMessage(HttpMethod.Post, builder.ToString()); - AddDefaultHttpHeaders(postMessage.Headers); - - postMessage.Content = content; - postMessage.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - if (isCompressed) - { - postMessage.Content.Headers.Add("Content-Encoding", "gzip"); - } - - GetLogger(ClickHouseLogCategories.Transport)?.LogDebug("Sending streamed request to {Endpoint} (Compressed: {Compressed}).", serverUri, isCompressed); - - try - { - using var response = await SendAsync(postMessage, HttpCompletionOption.ResponseContentRead, token).ConfigureAwait(false); - GetLogger(ClickHouseLogCategories.Transport)?.LogDebug("Streamed request to {Endpoint} received response {StatusCode}.", serverUri, response.StatusCode); - - return await HandleError(response, sql, activity).ConfigureAwait(false); - } - catch (Exception ex) + QueryOptions options = null; + if (queryId != null) { - GetLogger(ClickHouseLogCategories.Transport)?.LogError(ex, "Streamed request to {Endpoint} failed.", serverUri); - throw; + options = new QueryOptions + { + QueryId = queryId, + }; } + return options; } #pragma warning disable CS0109 // Member does not hide an inherited member; new keyword is not required @@ -485,143 +264,20 @@ private async Task PostStreamAsync(string sql, HttpContent void IDisposable.Dispose() { + DisposeClientIfOwned(); GC.SuppressFinalize(this); - foreach (var d in disposables) - d.Dispose(); - sessionRequestLock?.Dispose(); } - internal static string ExtractQueryId(HttpResponseMessage response) - { - const string queryIdHeader = "X-ClickHouse-Query-Id"; - if (response.Headers.Contains(queryIdHeader)) - return response.Headers.GetValues(queryIdHeader).FirstOrDefault(); - else - return null; - } - - internal HttpClient HttpClient => httpClientFactory.CreateClient(httpClientName); - - /// - /// Sends an HTTP request, serializing requests when sessions are enabled to prevent concurrent session access. - /// - internal async Task SendAsync(HttpRequestMessage request, HttpCompletionOption completionOption, CancellationToken cancellationToken) + private void DisposeClientIfOwned() { - HttpResponseMessage response; - - // This will only have a value when sessions are enabled - var lockRef = sessionRequestLock; - if (lockRef != null) - { - await lockRef.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - // Force ResponseContentRead to ensure response is fully buffered before releasing lock - response = await HttpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); - } - finally - { - lockRef.Release(); - } - } - else + if (ownsClient) { - response = await HttpClient.SendAsync(request, completionOption, cancellationToken).ConfigureAwait(false); + ClickHouseClient?.Dispose(); } - - return response; - } - - internal TypeSettings TypeSettings => new TypeSettings(Settings.UseCustomDecimals, Settings.ReadStringsAsByteArrays, jsonTypeRegistry, Settings.JsonReadMode, Settings.JsonWriteMode); - - /// - /// Registers a POCO type for JSON column serialization. - /// Types must be registered before they can be used in bulk copy operations with JSON or Dynamic columns. - /// - /// The POCO type to register. - /// - /// Thrown if any property type cannot be mapped to a ClickHouse type. - /// - public void RegisterJsonSerializationType() - where T : class - => jsonTypeRegistry.RegisterType(); - - /// - /// Registers a POCO type for JSON column serialization. - /// Types must be registered before they can be used in bulk copy operations with JSON or Dynamic columns. - /// - /// The POCO type to register. - /// Thrown if is null. - /// - /// Thrown if any property type cannot be mapped to a ClickHouse type. - /// - public void RegisterJsonSerializationType(Type type) - => jsonTypeRegistry.RegisterType(type); - - internal ClickHouseUriBuilder CreateUriBuilder(string sql = null) - { - var queryParams = CustomSettings.ToDictionary(kvp => kvp.Key, kvp => kvp.Value); - - return new ClickHouseUriBuilder(serverUri) - { - Database = Database, - SessionId = Settings.UseSession ? Settings.SessionId : null, - UseCompression = UseCompression, - ConnectionQueryStringParameters = queryParams, - ConnectionRoles = Settings.Roles, - Sql = sql, - JsonReadMode = Settings.JsonReadMode, - JsonWriteMode = Settings.JsonWriteMode, - }; } internal Task EnsureOpenAsync() => state != ConnectionState.Open ? OpenAsync() : Task.CompletedTask; - internal void AddDefaultHttpHeaders(HttpRequestHeaders headers, string bearerTokenOverride = null) - { - var userAgentInfo = UserAgentProvider.Info; - - // Priority: command-level bearer token > connection-level bearer token > basic auth - var bearerToken = bearerTokenOverride ?? Settings.BearerToken; - if (!string.IsNullOrEmpty(bearerToken)) - { - headers.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken); - } - else - { - headers.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{Settings.Username}:{Settings.Password}"))); - } - - headers.UserAgent.Add(userAgentInfo.DriverProductInfo); - headers.UserAgent.Add(userAgentInfo.SystemProductInfo); - headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - headers.Accept.Add(new MediaTypeWithQualityHeaderValue("text/csv")); - headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/octet-stream")); - if (UseCompression) - { - headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip")); - headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("deflate")); - } - - // Apply custom headers (blocked headers are silently ignored for security) - foreach (var kvp in Settings.CustomHeaders) - { - if (!IsBlockedHeader(kvp.Key)) - { - headers.TryAddWithoutValidation(kvp.Key, kvp.Value); - } - } - } - - private static bool IsBlockedHeader(string headerName) - { - return string.Equals(headerName, "Connection", StringComparison.OrdinalIgnoreCase) || - string.Equals(headerName, "Authorization", StringComparison.OrdinalIgnoreCase) || - string.Equals(headerName, "User-Agent", StringComparison.OrdinalIgnoreCase); - } - - internal ClickHouseConnectionStringBuilder ConnectionStringBuilder => ClickHouseConnectionStringBuilder.FromSettings(Settings); - protected override DbTransaction BeginDbTransaction(IsolationLevel isolationLevel) => throw new NotSupportedException(); protected override DbCommand CreateDbCommand() => CreateCommand(); diff --git a/ClickHouse.Driver/ADO/ClickHouseConnectionStringBuilder.cs b/ClickHouse.Driver/ADO/ClickHouseConnectionStringBuilder.cs index f36d2b39..a02daac6 100644 --- a/ClickHouse.Driver/ADO/ClickHouseConnectionStringBuilder.cs +++ b/ClickHouse.Driver/ADO/ClickHouseConnectionStringBuilder.cs @@ -174,7 +174,8 @@ private int GetIntOrDefault(string name, int @default) return @default; } - private T GetEnumOrDefault(string name, T @default) where T : struct, Enum + private T GetEnumOrDefault(string name, T @default) + where T : struct, Enum { if (TryGetValue(name, out var value) && value is string s && Enum.TryParse(s, ignoreCase: true, out var result)) return result; diff --git a/ClickHouse.Driver/ADO/ClickHouseDataSource.cs b/ClickHouse.Driver/ADO/ClickHouseDataSource.cs index 3f25b4b7..b538dda0 100644 --- a/ClickHouse.Driver/ADO/ClickHouseDataSource.cs +++ b/ClickHouse.Driver/ADO/ClickHouseDataSource.cs @@ -10,8 +10,9 @@ namespace ClickHouse.Driver.ADO; public sealed class ClickHouseDataSource : DbDataSource, IClickHouseDataSource { + private readonly bool disposeClient; private readonly HttpClient httpClient; - private readonly bool disposeHttpClient; + private readonly ClickHouseClient client; /// /// Initializes a new instance of the class using provided HttpClient. @@ -22,12 +23,13 @@ public sealed class ClickHouseDataSource : DbDataSource, IClickHouseDataSource /// dispose of the passed-in instance of HttpClient public ClickHouseDataSource(string connectionString, HttpClient httpClient = null, bool disposeHttpClient = true) { - Settings = new ClickHouseClientSettings(connectionString) + var settings = new ClickHouseClientSettings(connectionString) { HttpClient = httpClient, }; this.httpClient = httpClient; - this.disposeHttpClient = disposeHttpClient; + client = new ClickHouseClient(settings); + disposeClient = disposeHttpClient; } /// @@ -69,11 +71,13 @@ public ClickHouseDataSource(string connectionString, IHttpClientFactory httpClie { ArgumentNullException.ThrowIfNull(httpClientFactory); ArgumentNullException.ThrowIfNull(httpClientName); - Settings = new ClickHouseClientSettings(connectionString) + var settings = new ClickHouseClientSettings(connectionString) { HttpClientFactory = httpClientFactory, HttpClientName = httpClientName, }; + client = new ClickHouseClient(settings); + disposeClient = true; } /// @@ -91,22 +95,29 @@ public ClickHouseDataSource(string connectionString, IHttpClientFactory httpClie /// public ClickHouseDataSource(ClickHouseClientSettings settings) { - Settings = settings; + client = new ClickHouseClient(settings); + disposeClient = true; } - public ClickHouseClientSettings Settings { get; private set; } + public ClickHouseClientSettings Settings => client.Settings; - public override string ConnectionString => ClickHouseConnectionStringBuilder.FromSettings(Settings).ToString(); + public override string ConnectionString => client.ConnectionStringBuilder.ToString(); public ILoggerFactory LoggerFactory { get => Settings.LoggerFactory; - set + init { - Settings = new ClickHouseClientSettings(Settings) + var newSettings = new ClickHouseClientSettings(Settings) { LoggerFactory = value, }; + var oldClient = client; + client = new ClickHouseClient(newSettings); + if (disposeClient) + { + oldClient?.Dispose(); + } } } @@ -114,15 +125,16 @@ protected override void Dispose(bool disposing) { base.Dispose(disposing); - if (disposing && disposeHttpClient) + if (disposing && disposeClient) { + client?.Dispose(); httpClient?.Dispose(); } } protected override DbConnection CreateDbConnection() { - return new ClickHouseConnection(Settings); + return new ClickHouseConnection(client); } public new ClickHouseConnection CreateConnection() => (ClickHouseConnection)CreateDbConnection(); @@ -139,6 +151,8 @@ protected override DbConnection CreateDbConnection() return (ClickHouseConnection)cn; } + public IClickHouseClient GetClient() => client; + async Task IClickHouseDataSource.OpenConnectionAsync(CancellationToken cancellationToken) { var cn = await OpenDbConnectionAsync(cancellationToken).ConfigureAwait(false); diff --git a/ClickHouse.Driver/ADO/Parameters/ClickHouseParameterCollection.cs b/ClickHouse.Driver/ADO/Parameters/ClickHouseParameterCollection.cs index 52b75636..8f65810e 100644 --- a/ClickHouse.Driver/ADO/Parameters/ClickHouseParameterCollection.cs +++ b/ClickHouse.Driver/ADO/Parameters/ClickHouseParameterCollection.cs @@ -7,7 +7,7 @@ namespace ClickHouse.Driver.ADO.Parameters; -internal class ClickHouseParameterCollection : DbParameterCollection +public class ClickHouseParameterCollection : DbParameterCollection { private readonly List parameters = new(); diff --git a/ClickHouse.Driver/ClickHouse.Driver.csproj b/ClickHouse.Driver/ClickHouse.Driver.csproj index c9c571ba..a199b600 100644 --- a/ClickHouse.Driver/ClickHouse.Driver.csproj +++ b/ClickHouse.Driver/ClickHouse.Driver.csproj @@ -34,10 +34,6 @@ - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - @@ -49,11 +45,6 @@ - - - - - diff --git a/ClickHouse.Driver/ClickHouseClient.cs b/ClickHouse.Driver/ClickHouseClient.cs new file mode 100644 index 00000000..c26dec01 --- /dev/null +++ b/ClickHouse.Driver/ClickHouseClient.cs @@ -0,0 +1,773 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using ClickHouse.Driver.ADO; +using ClickHouse.Driver.ADO.Parameters; +using ClickHouse.Driver.ADO.Readers; +using ClickHouse.Driver.Copy; +using ClickHouse.Driver.Copy.Serializer; +using ClickHouse.Driver.Diagnostic; +using ClickHouse.Driver.Formats; +using ClickHouse.Driver.Http; +using ClickHouse.Driver.Json; +using ClickHouse.Driver.Logging; +using ClickHouse.Driver.Types; +using ClickHouse.Driver.Utility; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.IO; + +namespace ClickHouse.Driver; + +/// +/// A high-level client for interacting with ClickHouse. +/// This is the recommended API for new code. It is thread-safe and designed for singleton usage. +/// +/// +/// +/// Unlike , which follows ADO.NET patterns, +/// provides a simpler, more direct API that better matches +/// ClickHouse's HTTP-based protocol. +/// +/// +/// For best performance, create a single instance and reuse it +/// throughout your application. The client manages HTTP connection pooling internally. +/// +/// +public sealed class ClickHouseClient : IClickHouseClient +{ + private const int DefaultMemoryStreamBlockSize = 256 * 1024; // 256 KB + private const int DefaultMaxSmallPoolFreeBytes = 128 * 1024 * 1024; // 128 MB + private const int DefaultMaxLargePoolFreeBytes = 512 * 1024 * 1024; // 512 MB + + private readonly List disposables = new(); + private readonly ConcurrentDictionary> loggerCache = new(); + private readonly JsonTypeRegistry jsonTypeRegistry = new(); + private readonly IHttpClientFactory httpClientFactory; + private readonly string httpClientName; + private readonly Uri serverUri; + private readonly ILoggerFactory loggerFactory; + + private static readonly RecyclableMemoryStreamManager CommonMemoryStreamManager = new(new RecyclableMemoryStreamManager.Options + { + MaximumLargePoolFreeBytes = DefaultMaxLargePoolFreeBytes, + MaximumSmallPoolFreeBytes = DefaultMaxSmallPoolFreeBytes, + BlockSize = DefaultMemoryStreamBlockSize, + }); + + private readonly RecyclableMemoryStreamManager memoryStreamManager; + + /// + /// Gets RecyclableMemoryStreamManager used to create recyclable streams. + /// + public RecyclableMemoryStreamManager MemoryStreamManager + { + get { return memoryStreamManager ?? CommonMemoryStreamManager; } + init { memoryStreamManager = value; } + } + + private bool disposed; + + /// + /// Initializes a new instance of the class with the specified connection string. + /// + /// The ClickHouse connection string. + public ClickHouseClient(string connectionString) + : this(new ClickHouseClientSettings(connectionString)) + { + } + + /// + /// Initializes a new instance of the class with the specified connection string and an HttpClient instance. + /// + /// The ClickHouse connection string. + /// Instance of HttpClient + public ClickHouseClient(string connectionString, HttpClient httpClient) + : this(new ClickHouseClientSettings(connectionString) + { + HttpClient = httpClient, + }) + { + } + + /// + /// Initializes a new instance of the class with the specified connection string and an IHttpClientFactory. + /// + /// The ClickHouse connection string. + /// An IHttpClientFactory + /// The name of the HTTP client you want to be created using the provided factory. If left empty, the default client will be created. + public ClickHouseClient(string connectionString, IHttpClientFactory httpClientFactory, string httpClientName = "") + : this(new ClickHouseClientSettings(connectionString) + { + HttpClientFactory = httpClientFactory, + HttpClientName = httpClientName, + }) + { + } + + /// + /// Initializes a new instance of the class with the specified settings. + /// + /// The client settings. + public ClickHouseClient(ClickHouseClientSettings settings) + { + Settings = settings ?? throw new ArgumentNullException(nameof(settings)); + Settings.Validate(); + + serverUri = new UriBuilder(Settings.Protocol, Settings.Host, Settings.Port, Settings.Path ?? string.Empty).Uri; + httpClientName = Settings.HttpClientName ?? string.Empty; + loggerFactory = Settings.LoggerFactory; + + if (Settings.EnableDebugMode && loggerFactory != null) + { + TraceHelper.Activate(loggerFactory); + } + + httpClientFactory = CreateHttpClientFactory(settings); + } + + /// + /// Gets the settings used by this client. + /// + public ClickHouseClientSettings Settings { get; } + + internal string RedactedConnectionString + { + get + { + var builder = ConnectionStringBuilder; + builder.Password = "****"; + return builder.ToString(); + } + } + + internal ClickHouseConnectionStringBuilder ConnectionStringBuilder => ClickHouseConnectionStringBuilder.FromSettings(Settings); + + /// + /// Gets the type settings for serialization. + /// + internal TypeSettings TypeSettings => new(Settings.UseCustomDecimals, Settings.ReadStringsAsByteArrays, jsonTypeRegistry, Settings.JsonReadMode, Settings.JsonWriteMode); + + /// + /// Gets the server URI. + /// + internal Uri ServerUri => serverUri; + + /// + public async Task PingAsync(QueryOptions queryOptions = null, CancellationToken cancellationToken = default) + { + try + { + var pingUri = new Uri(serverUri, "ping"); + using var request = new HttpRequestMessage(HttpMethod.Get, pingUri); + AddDefaultHttpHeaders(request.Headers, queryOptions); + + using var response = await SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); + return response.IsSuccessStatusCode; + } + catch (Exception ex) + { + GetLogger(ClickHouseLogCategories.Connection)?.LogWarning(ex, "Ping to {Endpoint} failed.", serverUri); + return false; + } + } + + /// + public void RegisterJsonSerializationType() + where T : class + => jsonTypeRegistry.RegisterType(); + + /// + public void RegisterJsonSerializationType(Type type) + => jsonTypeRegistry.RegisterType(type); + + /// + public ClickHouseConnection CreateConnection() + { + return new ClickHouseConnection(this); + } + + /// + public async Task ExecuteNonQueryAsync( + string sql, + ClickHouseParameterCollection parameters = null, + QueryOptions options = null, + CancellationToken cancellationToken = default) + { + var response = await PostSqlQueryAsync(sql, parameters, options, cancellationToken).ConfigureAwait(false); + using var reader = new ExtendedBinaryReader(await response.HttpResponseMessage.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false)); + + return reader.PeekChar() != -1 ? reader.Read7BitEncodedInt() : 0; + } + + /// + public async Task ExecuteScalarAsync( + string sql, + ClickHouseParameterCollection parameters = null, + QueryOptions options = null, + CancellationToken cancellationToken = default) + { + using var reader = await ExecuteReaderAsync(sql, parameters, options, cancellationToken).ConfigureAwait(false); + return reader.Read() ? reader.GetValue(0) : null; + } + + /// + public async Task ExecuteReaderAsync( + string sql, + ClickHouseParameterCollection parameters = null, + QueryOptions options = null, + CancellationToken cancellationToken = default) + { + var result = await PostSqlQueryAsync(sql, parameters, options, cancellationToken).ConfigureAwait(false); + return await ClickHouseDataReader.FromHttpResponseAsync(result.HttpResponseMessage, TypeSettings).ConfigureAwait(false); + } + + internal async Task PostSqlQueryAsync( + string sql, + ClickHouseParameterCollection parameters = null, + QueryOptions options = null, + CancellationToken cancellationToken = default) + { + using var activity = this.StartActivity("PostSqlQueryAsync"); + + var uriBuilder = CreateUriBuilder(queryOverride: options); + + var logger = GetLogger(ClickHouseLogCategories.Command); + var isDebugLoggingEnabled = logger?.IsEnabled(LogLevel.Debug) ?? false; + Stopwatch stopwatch = null; + if (isDebugLoggingEnabled) + { + stopwatch = Stopwatch.StartNew(); + logger.LogDebug("Executing SQL query. QueryId: {QueryId}", uriBuilder.GetEffectiveQueryId()); + } + + using var postMessage = Settings.UseFormDataParameters + ? BuildHttpRequestMessageWithFormData( + sql, + parameters, + uriBuilder, + options) + : BuildHttpRequestMessageWithQueryParams( + sql, + parameters, + uriBuilder, + options); + + activity.SetQuery(sql); + + HttpResponseMessage response = null; + try + { + response = await SendAsync(postMessage, HttpCompletionOption.ResponseHeadersRead, cancellationToken) + .ConfigureAwait(false); + + var handled = await HandleError(response, sql, activity).ConfigureAwait(false); + var result = new QueryResult(handled); + + if (isDebugLoggingEnabled) + { + LogQuerySuccess(stopwatch, uriBuilder.GetEffectiveQueryId(), logger, result.QueryStats); + } + + activity.SetQueryStats(result.QueryStats); + + return result; + } + catch (Exception ex) + { + logger?.LogError(ex, "Query (QueryId: {QueryId}) failed.", uriBuilder.GetEffectiveQueryId()); + activity?.SetException(ex); + throw; + } + } + + private HttpRequestMessage BuildHttpRequestMessageWithQueryParams(string sqlQuery, ClickHouseParameterCollection parameters, ClickHouseUriBuilder uriBuilder, QueryOptions queryOptions) + { + if (parameters != null) + { + var typeHints = SqlParameterTypeExtractor.ExtractTypeHints(sqlQuery); + sqlQuery = parameters.ReplacePlaceholders(sqlQuery); + foreach (ClickHouseDbParameter parameter in parameters) + { + typeHints.TryGetValue(parameter.ParameterName, out var sqlTypeHint); + uriBuilder.AddSqlQueryParameter( + parameter.ParameterName, + HttpParameterFormatter.Format(parameter, TypeSettings, sqlTypeHint)); + } + } + + var uri = uriBuilder.ToString(); + + var postMessage = new HttpRequestMessage(HttpMethod.Post, uri); + + AddDefaultHttpHeaders(postMessage.Headers, queryOptions); + HttpContent content = new StringContent(sqlQuery); + content.Headers.ContentType = new MediaTypeHeaderValue("text/sql"); + if (Settings.UseCompression) + { + content = new CompressedContent(content, DecompressionMethods.GZip); + } + + postMessage.Content = content; + + return postMessage; + } + + private HttpRequestMessage BuildHttpRequestMessageWithFormData(string sqlQuery, ClickHouseParameterCollection parameters, ClickHouseUriBuilder uriBuilder, QueryOptions queryOptions) + { + var content = new MultipartFormDataContent(); + + if (parameters != null) + { + var typeHints = SqlParameterTypeExtractor.ExtractTypeHints(sqlQuery); + sqlQuery = parameters.ReplacePlaceholders(sqlQuery); + + foreach (ClickHouseDbParameter parameter in parameters) + { + typeHints.TryGetValue(parameter.ParameterName, out var sqlTypeHint); + content.Add( + content: new StringContent(HttpParameterFormatter.Format(parameter, TypeSettings, sqlTypeHint)), + name: $"param_{parameter.ParameterName}"); + } + } + + content.Add( + content: new StringContent(sqlQuery), + name: "query"); + + var uri = uriBuilder.ToString(); + + var postMessage = new HttpRequestMessage(HttpMethod.Post, uri); + + AddDefaultHttpHeaders(postMessage.Headers, queryOptions); + + postMessage.Content = content; + + return postMessage; + } + + private static void LogQuerySuccess(Stopwatch stopwatch, string queryId, ILogger logger, QueryStats queryStats) + { + stopwatch.Stop(); + logger.LogDebug( + "Query (QueryId: {QueryId}) succeeded in {ElapsedMilliseconds:F2} ms. Query Stats: {QueryStats}", + queryId, + stopwatch.Elapsed.TotalMilliseconds, + queryStats); + } + + /// + public async Task ExecuteRawResultAsync( + string sql, + QueryOptions options = null, + CancellationToken cancellationToken = default) + { + var response = await PostSqlQueryAsync(sql, null, options, cancellationToken).ConfigureAwait(false); + return new ClickHouseRawResult(response.HttpResponseMessage); + } + + private static string GetColumnsExpression(IEnumerable columns) => columns == null || !columns.Any() ? "*" : string.Join(",", columns); + + private async Task<(string[] names, ClickHouseType[] types)> LoadNamesAndTypesAsync(string destinationTableName, QueryOptions options, IEnumerable columns = null) + { + using var reader = (ClickHouseDataReader)await ExecuteReaderAsync($"SELECT {GetColumnsExpression(columns)} FROM {destinationTableName} WHERE 1=0", null, options).ConfigureAwait(false); + var types = reader.GetClickHouseColumnTypes(); + var names = reader.GetColumnNames().Select(c => c.EncloseColumnName()).ToArray(); + return (names, types); + } + + private async Task SendBatchAsync(string destinationTable, Batch batch, BatchSerializer serializer, InsertOptions insertOptions, Action onBatchSent, CancellationToken token) + { + var logger = GetLogger(ClickHouseLogCategories.Client); + + using (batch) // Dispose object regardless whether sending succeeds + { + using var stream = MemoryStreamManager.GetStream(nameof(SendBatchAsync), 128 * 1024); + // Async serialization + await Task.Run(() => serializer.Serialize(batch, stream), token).ConfigureAwait(false); + + // Seek to beginning as after writing it's at end + stream.Seek(0, SeekOrigin.Begin); + + // Async sending + logger?.LogDebug("Sending batch of {Rows} rows to {Table}.", batch.Size, destinationTable); + await PostStreamAsync(null, stream, true, token, insertOptions).ConfigureAwait(false); + + onBatchSent?.Invoke(batch.Size); + + logger?.LogDebug("Batch sent to {Table}. Rows in batch: {BatchRows}.", destinationTable, batch.Size); + return batch.Size; + } + } + + /// + public Task InsertBinaryAsync( + string table, + IEnumerable columns, + IEnumerable rows, + InsertOptions options = default, + CancellationToken cancellationToken = default) + { + return InsertBinaryAsync(table, columns, rows, options, onBatchSent: null, cancellationToken); + } + + /// + /// Internal version which takes a callback method, to allow us to maintain backwards + /// compat with the BatchSent event in BulkCopy. + /// + internal async Task InsertBinaryAsync( + string table, + IEnumerable columns, + IEnumerable rows, + InsertOptions options, + Action onBatchSent, + CancellationToken cancellationToken) + { + if (table is null) + throw new InvalidOperationException($"{nameof(table)} is null"); + if (rows is null) + throw new ArgumentNullException(nameof(rows)); + + // Use default values if none provided + options ??= new InsertOptions(); + + if (options.BatchSize <= 0) + throw new ArgumentOutOfRangeException(nameof(options), "BatchSize must be greater than zero"); + if (options.MaxDegreeOfParallelism <= 0) + throw new ArgumentOutOfRangeException(nameof(options), "MaxDegreeOfParallelism must be greater than zero"); + + var serializer = BatchSerializer.GetByRowBinaryFormat(options.Format); + + // Load table structure + var logger = GetLogger(ClickHouseLogCategories.Client); + logger?.LogDebug("Loading metadata for table {Table}.", table); + + var (columnNames, columnTypes) = await LoadNamesAndTypesAsync(table, options, columns).ConfigureAwait(false); + if (columnNames == null || columnTypes == null) + throw new InvalidOperationException("Column names not initialized. Initialization failed."); + + if (logger?.IsEnabled(LogLevel.Debug) ?? false) + { + logger.LogDebug("Metadata loaded for table {Table}. Columns: {Columns}.", table, string.Join(", ", columnNames ?? Array.Empty())); + } + + // Insert + var query = $"INSERT INTO {table} ({string.Join(", ", columnNames)}) FORMAT {options.Format.ToString()}"; + + var isDebugLoggingEnabled = logger?.IsEnabled(LogLevel.Debug) ?? false; + Stopwatch stopwatch = null; + if (isDebugLoggingEnabled) + { + stopwatch = Stopwatch.StartNew(); + logger.LogDebug("Starting bulk copy into {Table} with batch size {BatchSize} and degree {Degree}.", table, options.BatchSize, options.MaxDegreeOfParallelism); + } + + long totalRowsWritten = 0; + var batches = IntoBatches(rows, query, columnTypes, options.BatchSize); + + await Parallel.ForEachAsync( + batches, + new ParallelOptions + { + MaxDegreeOfParallelism = options.MaxDegreeOfParallelism, + CancellationToken = cancellationToken, + }, + async (batch, ct) => + { + var count = await SendBatchAsync(table, batch, serializer, options, onBatchSent, ct).ConfigureAwait(false); + Interlocked.Add(ref totalRowsWritten, count); + }).ConfigureAwait(false); + + if (isDebugLoggingEnabled) + { + stopwatch.Stop(); + logger.LogDebug("Bulk copy into {Table} completed in {ElapsedMilliseconds:F2} ms. Total rows: {Rows}.", table, stopwatch.Elapsed.TotalMilliseconds, totalRowsWritten); + } + + return totalRowsWritten; + } + + private static IEnumerable IntoBatches(IEnumerable rows, string query, ClickHouseType[] types, int batchSize) + { + foreach (var (batch, size) in rows.BatchRented(batchSize)) + { + yield return new Batch { Rows = batch, Size = size, Query = query, Types = types }; + } + } + + /// + public async Task InsertRawStreamAsync( + string table, + Stream stream, + string format, + IEnumerable columns = null, + bool useCompression = true, + QueryOptions options = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(table)) + throw new ArgumentException("Table name cannot be null or empty", nameof(table)); + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + if (string.IsNullOrEmpty(format)) + throw new ArgumentException("Format cannot be null or empty", nameof(format)); + + var columnList = columns != null ? $"({string.Join(", ", columns)})" : string.Empty; + var query = $"INSERT INTO {table} {columnList} FORMAT {format}"; + + HttpContent content = new StreamContent(stream); + if (useCompression) + { + // CompressedContent handles compression and adds Content-Encoding header + content = new CompressedContent(content, System.Net.DecompressionMethods.GZip); + } + + // Pass isCompressed=false since CompressedContent already adds the Content-Encoding header + try + { + return await PostStreamAsync(query, content, isCompressed: false, options, cancellationToken).ConfigureAwait(false); + } + catch + { + content.Dispose(); + throw; + } + } + + /// + public async Task PostStreamAsync(string sql, Stream data, bool isCompressed, CancellationToken token, QueryOptions queryOptions = null) + { + var content = new StreamContent(data); + return await PostStreamAsync(sql, content, isCompressed, queryOptions, token).ConfigureAwait(false); + } + + /// + public async Task PostStreamAsync(string sql, Func callback, bool isCompressed, CancellationToken token, QueryOptions queryOptions = null) + { + var content = new StreamCallbackContent(callback, token); + return await PostStreamAsync(sql, content, isCompressed, queryOptions, token).ConfigureAwait(false); + } + + private async Task PostStreamAsync(string sql, HttpContent content, bool isCompressed, QueryOptions queryOptions, CancellationToken token) + { + using var activity = this.StartActivity("PostStreamAsync"); + activity.SetQuery(sql); + + var builder = CreateUriBuilder(sql, queryOptions); + + using var postMessage = new HttpRequestMessage(HttpMethod.Post, builder.ToString()); + AddDefaultHttpHeaders(postMessage.Headers, queryOptions); + + postMessage.Content = content; + postMessage.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + if (isCompressed) + { + postMessage.Content.Headers.Add("Content-Encoding", "gzip"); + } + + GetLogger(ClickHouseLogCategories.Transport)?.LogDebug("Sending streamed request to {Endpoint} (Compressed: {Compressed}).", serverUri, isCompressed); + + try + { + var response = await SendAsync(postMessage, HttpCompletionOption.ResponseContentRead, token).ConfigureAwait(false); + GetLogger(ClickHouseLogCategories.Transport)?.LogDebug("Streamed request to {Endpoint} received response {StatusCode}.", serverUri, response.StatusCode); + + return await HandleError(response, sql, activity).ConfigureAwait(false); + } + catch (Exception ex) + { + GetLogger(ClickHouseLogCategories.Transport)?.LogError(ex, "Streamed request to {Endpoint} failed.", serverUri); + throw; + } + } + + /// + /// Releases all resources used by the client. + /// + public void Dispose() + { + if (disposed) + return; + + disposed = true; + + foreach (var d in disposables) + { + d.Dispose(); + } + + GetLogger(ClickHouseLogCategories.Connection)?.LogDebug("ClickHouseClient disposed."); + } + + /// + /// Gets a logger for the specified category name. + /// + internal ILogger GetLogger(string categoryName) + { + if (loggerFactory == null) + return null; + + return loggerCache.GetOrAdd( + categoryName, + key => new Lazy(() => loggerFactory.CreateLogger(key))).Value; + } + + /// + /// Gets an HTTP client from the factory. + /// + internal HttpClient HttpClient => httpClientFactory.CreateClient(httpClientName); + + /// + /// Creates a URI builder for the specified SQL query. + /// + internal ClickHouseUriBuilder CreateUriBuilder(string sql = null, QueryOptions queryOverride = null) + { + string sessionId = Settings.UseSession ? Settings.SessionId : null; + if (queryOverride?.UseSession != null) + { + // Prioritize query-level setting + sessionId = queryOverride.UseSession.Value ? queryOverride.SessionId : null; + } + + return new ClickHouseUriBuilder(serverUri) + { + Database = queryOverride?.Database ?? Settings.Database, + SessionId = sessionId, + UseCompression = Settings.UseCompression, + ConnectionQueryStringParameters = Settings.CustomSettings, + CommandQueryStringParameters = queryOverride?.CustomSettings, + ConnectionRoles = Settings.Roles, + CommandRoles = queryOverride?.Roles, + Sql = sql, + JsonReadMode = Settings.JsonReadMode, + JsonWriteMode = Settings.JsonWriteMode, + QueryId = queryOverride?.QueryId, + MaxExecutionTime = queryOverride?.MaxExecutionTime, + }; + } + + /// + /// Adds default HTTP headers to a request. + /// + internal void AddDefaultHttpHeaders(HttpRequestHeaders headers, QueryOptions queryOverride = null) + { + var userAgentInfo = UserAgentProvider.Info; + + // Priority: override > connection-level bearer token > basic auth + var bearerToken = queryOverride?.BearerToken ?? Settings.BearerToken; + if (!string.IsNullOrEmpty(bearerToken)) + { + headers.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken); + } + else + { + headers.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{Settings.Username}:{Settings.Password}"))); + } + + headers.UserAgent.Add(userAgentInfo.DriverProductInfo); + headers.UserAgent.Add(userAgentInfo.SystemProductInfo); + headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + headers.Accept.Add(new MediaTypeWithQualityHeaderValue("text/csv")); + headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/octet-stream")); + + if (Settings.UseCompression) + { + headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip")); + headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("deflate")); + } + + // Apply custom headers (blocked headers are silently ignored for security) + ApplyCustomHeaders(headers, Settings.CustomHeaders); + + // Override + ApplyCustomHeaders(headers, queryOverride?.CustomHeaders); + } + + private static void ApplyCustomHeaders(HttpRequestHeaders requestHeaders, IReadOnlyDictionary customHeaders) + { + if (customHeaders != null) + { + foreach (var kvp in customHeaders) + { + if (!IsBlockedHeader(kvp.Key)) + { + requestHeaders.Remove(kvp.Key); + requestHeaders.TryAddWithoutValidation(kvp.Key, kvp.Value); + } + } + } + } + + /// + /// Handles HTTP response errors. + /// + private static async Task HandleError(HttpResponseMessage response, string query, Activity activity) + { + if (response.IsSuccessStatusCode) + { + activity?.SetSuccess(); + return response; + } + + var error = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + var ex = ClickHouseServerException.FromServerResponse(error, query); + activity?.SetException(ex); + throw ex; + } + + private IHttpClientFactory CreateHttpClientFactory(ClickHouseClientSettings settings) + { + IHttpClientFactory factory; + if (settings.HttpClient != null) + { + GetLogger(ClickHouseLogCategories.Connection)?.LogInformation("Using provided HttpClient instance."); + factory = new CannedHttpClientFactory(settings.HttpClient); + } + else if (settings.HttpClientFactory != null) + { + GetLogger(ClickHouseLogCategories.Connection)?.LogInformation("Using IHttpClientFactory from settings."); + factory = settings.HttpClientFactory; + } + else + { + // Default: create pooled factory + GetLogger(ClickHouseLogCategories.Connection)?.LogInformation("Creating default pooled HttpClientFactory."); + var defaultFactory = new DefaultPoolHttpClientFactory(settings.SkipServerCertificateValidation) + { + Timeout = settings.Timeout, + }; + disposables.Add(defaultFactory); + factory = defaultFactory; + } + + LoggingHelpers.LogHttpClientConfiguration(GetLogger(ClickHouseLogCategories.Client), factory); + + return factory; + } + + /// + /// Sends an HTTP request + /// + internal async Task SendAsync(HttpRequestMessage request, HttpCompletionOption completionOption, CancellationToken cancellationToken) + { + return await HttpClient.SendAsync(request, completionOption, cancellationToken).ConfigureAwait(false); + } + + private static bool IsBlockedHeader(string headerName) + { + return string.Equals(headerName, "Connection", StringComparison.OrdinalIgnoreCase) || + string.Equals(headerName, "Authorization", StringComparison.OrdinalIgnoreCase) || + string.Equals(headerName, "User-Agent", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/ClickHouse.Driver/ClickHouseUriBuilder.cs b/ClickHouse.Driver/ClickHouseUriBuilder.cs index 98c97f59..3e81b05f 100644 --- a/ClickHouse.Driver/ClickHouseUriBuilder.cs +++ b/ClickHouse.Driver/ClickHouseUriBuilder.cs @@ -53,6 +53,8 @@ public string QueryId public JsonWriteMode JsonWriteMode { get; set; } + public TimeSpan? MaxExecutionTime { get; set; } + /// /// Gets the effective query ID that will be used in the request. /// If QueryId is not set, generates and caches a new GUID. @@ -99,6 +101,9 @@ public override string ToString() parameters.Set(parameter.Key, Convert.ToString(parameter.Value, CultureInfo.InvariantCulture)); } + if (MaxExecutionTime.HasValue) + parameters.Set("max_execution_time", MaxExecutionTime.Value.TotalSeconds.ToString(CultureInfo.InvariantCulture)); + var queryString = string.Join("&", parameters.Select(kvp => $"{kvp.Key}={HttpUtility.UrlEncode(kvp.Value)}")); // Append role parameters - command roles replace connection roles diff --git a/ClickHouse.Driver/Copy/ClickHouseBulkCopy.cs b/ClickHouse.Driver/Copy/ClickHouseBulkCopy.cs index d14f68e9..6ab75751 100644 --- a/ClickHouse.Driver/Copy/ClickHouseBulkCopy.cs +++ b/ClickHouse.Driver/Copy/ClickHouseBulkCopy.cs @@ -17,24 +17,15 @@ namespace ClickHouse.Driver.Copy; +[Obsolete("The BulkCopy class functionality can now be found in ClickHouseClient. ClickHouseBulkCopy will be removed in a future version.")] public class ClickHouseBulkCopy : IDisposable { - private static readonly RecyclableMemoryStreamManager CommonMemoryStreamManager = new(new RecyclableMemoryStreamManager.Options - { - MaximumLargePoolFreeBytes = 512 * 1024 * 1024, - MaximumSmallPoolFreeBytes = 128 * 1024 * 1024, - BlockSize = 256 * 1024, - }); - private readonly ClickHouseConnection connection; + private readonly ClickHouseClient client; private readonly BatchSerializer batchSerializer; private readonly RowBinaryFormat rowBinaryFormat; private readonly bool ownsConnection; - private readonly RecyclableMemoryStreamManager memoryStreamManager; private long rowsWritten; - private (string[] names, ClickHouseType[] types) columnNamesAndTypes; - private Task initializationTask; - private readonly SemaphoreSlim initializationLock = new SemaphoreSlim(1, 1); public ClickHouseBulkCopy(ClickHouseConnection connection) : this(connection, RowBinaryFormat.RowBinary) { } @@ -45,6 +36,7 @@ public ClickHouseBulkCopy(string connectionString) public ClickHouseBulkCopy(ClickHouseConnection connection, RowBinaryFormat rowBinaryFormat) { this.connection = connection ?? throw new ArgumentNullException(nameof(connection)); + client = connection.ClickHouseClient; this.rowBinaryFormat = rowBinaryFormat; batchSerializer = BatchSerializer.GetByRowBinaryFormat(rowBinaryFormat); } @@ -97,15 +89,6 @@ public long RowsWritten } } - /// - /// Gets RecyclableMemoryStreamManager used to create recyclable streams. - /// - public RecyclableMemoryStreamManager MemoryStreamManager - { - get { return memoryStreamManager ?? CommonMemoryStreamManager; } - init { memoryStreamManager = value; } - } - /// /// Gets total number of rows written by this instance. /// @@ -119,48 +102,6 @@ public RecyclableMemoryStreamManager MemoryStreamManager [Obsolete("InitAsync is no longer required and will be removed in a future version. Initialization now occurs automatically before the first write operation.")] public async Task InitAsync() { - await EnsureInitializedAsync().ConfigureAwait(false); - } - - /// - /// Thread-safe method to ensure the BulkCopy object is initialized by loading the table structure - /// - private async Task EnsureInitializedAsync() - { - // Fast path: already initialized successfully - if (initializationTask != null && initializationTask.Status == TaskStatus.RanToCompletion) - return; - - await initializationLock.WaitAsync().ConfigureAwait(false); - try - { - // Double-check after acquiring lock - if (initializationTask != null && initializationTask.Status == TaskStatus.RanToCompletion) - return; - - initializationTask = InitAsyncCore(); - await initializationTask.ConfigureAwait(false); - } - finally - { - initializationLock.Release(); - } - } - - private async Task InitAsyncCore() - { - if (DestinationTableName is null) - throw new InvalidOperationException($"{nameof(DestinationTableName)} is null"); - - var logger = connection.GetLogger(ClickHouseLogCategories.BulkCopy); - logger?.LogDebug("Loading metadata for table {Table}.", DestinationTableName); - - columnNamesAndTypes = await LoadNamesAndTypesAsync(DestinationTableName, ColumnNames).ConfigureAwait(false); - - if (logger?.IsEnabled(LogLevel.Debug) ?? false) - { - logger.LogDebug("Metadata loaded for table {Table}. Columns: {Columns}.", DestinationTableName, string.Join(", ", columnNamesAndTypes.names ?? Array.Empty())); - } } public Task WriteToServerAsync(IDataReader reader) => WriteToServerAsync(reader, CancellationToken.None); @@ -186,117 +127,33 @@ public Task WriteToServerAsync(DataTable table, CancellationToken token) public async Task WriteToServerAsync(IEnumerable rows, CancellationToken token) { - if (rows is null) - throw new ArgumentNullException(nameof(rows)); - - var logger = connection.GetLogger(ClickHouseLogCategories.BulkCopy); - - if (string.IsNullOrWhiteSpace(DestinationTableName)) - throw new InvalidOperationException("Destination table not set"); - - // Auto-initialize if not already done - await EnsureInitializedAsync().ConfigureAwait(false); - - var (columnNames, columnTypes) = columnNamesAndTypes; - // Safety check (initialization should have succeeded if we got here) - if (columnNames == null || columnTypes == null) - throw new InvalidOperationException("Column names not initialized. Initialization failed."); - - var query = $"INSERT INTO {DestinationTableName} ({string.Join(", ", columnNames)}) FORMAT {rowBinaryFormat.ToString()}"; - - var isDebugLoggingEnabled = logger?.IsEnabled(LogLevel.Debug) ?? false; - Stopwatch stopwatch = null; - if (isDebugLoggingEnabled) + var options = new InsertOptions { - stopwatch = Stopwatch.StartNew(); - logger.LogDebug("Starting bulk copy into {Table} with batch size {BatchSize} and degree {Degree}.", DestinationTableName, BatchSize, MaxDegreeOfParallelism); - } - - var tasks = new Task[MaxDegreeOfParallelism]; - for (var i = 0; i < tasks.Length; i++) - { - tasks[i] = Task.CompletedTask; - } - - foreach (var batch in IntoBatches(rows, query, columnTypes)) - { - while (true) + BatchSize = BatchSize, + MaxDegreeOfParallelism = MaxDegreeOfParallelism, + Format = rowBinaryFormat, + }; + + await client.InsertBinaryAsync( + DestinationTableName, + ColumnNames, + rows, + options, + onBatchSent: batchSize => { - var completedTaskIndex = Array.FindIndex(tasks, t => t.IsCompleted); - if (completedTaskIndex >= 0) - { - tasks[completedTaskIndex] = SendBatchAsync(batch, token); - break; // while (true); go to next batch - } - else - { - var completedTask = await Task.WhenAny(tasks).ConfigureAwait(false); - await completedTask.ConfigureAwait(false); - } - } - } - - await Task.WhenAll(tasks).ConfigureAwait(false); - - if (isDebugLoggingEnabled) - { - stopwatch.Stop(); - logger.LogDebug("Bulk copy into {Table} completed in {ElapsedMilliseconds:F2} ms. Total rows: {Rows}.", DestinationTableName, stopwatch.Elapsed.TotalMilliseconds, RowsWritten); - } - } - - private async Task<(string[] names, ClickHouseType[] types)> LoadNamesAndTypesAsync(string destinationTableName, IReadOnlyCollection columns = null) - { - using var reader = (ClickHouseDataReader)await connection.ExecuteReaderAsync($"SELECT {GetColumnsExpression(columns)} FROM {DestinationTableName} WHERE 1=0").ConfigureAwait(false); - var types = reader.GetClickHouseColumnTypes(); - var names = reader.GetColumnNames().Select(c => c.EncloseColumnName()).ToArray(); - return (names, types); - } - - private async Task SendBatchAsync(Batch batch, CancellationToken token) - { - var logger = connection.GetLogger(ClickHouseLogCategories.BulkCopy); - - using (batch) // Dispose object regardless whether sending succeeds - { - using var stream = MemoryStreamManager.GetStream(nameof(SendBatchAsync), 128 * 1024); - // Async serialization - await Task.Run(() => batchSerializer.Serialize(batch, stream), token).ConfigureAwait(false); - - // Seek to beginning as after writing it's at end - stream.Seek(0, SeekOrigin.Begin); - - // Async sending - logger?.LogDebug("Sending batch of {Rows} rows to {Table}.", batch.Size, DestinationTableName); - await connection.PostStreamAsync(null, stream, true, token).ConfigureAwait(false); - - // Increase counter - var batchRowsWritten = Interlocked.Add(ref rowsWritten, batch.Size); - - // Raise BatchSent event - BatchSent?.Invoke(this, new BatchSentEventArgs(batchRowsWritten)); - - logger?.LogDebug("Batch sent to {Table}. Total rows written: {TotalRows}.", DestinationTableName, batchRowsWritten); - } + var totalWritten = Interlocked.Add(ref rowsWritten, batchSize); + BatchSent?.Invoke(this, new BatchSentEventArgs(totalWritten)); + }, + token).ConfigureAwait(false); } public void Dispose() { if (ownsConnection) { + client?.Dispose(); connection?.Dispose(); } - initializationLock?.Dispose(); GC.SuppressFinalize(this); } - - private static string GetColumnsExpression(IReadOnlyCollection columns) => columns == null || columns.Count == 0 ? "*" : string.Join(",", columns); - - private IEnumerable IntoBatches(IEnumerable rows, string query, ClickHouseType[] types) - { - foreach (var (batch, size) in rows.BatchRented(BatchSize)) - { - yield return new Batch { Rows = batch, Size = size, Query = query, Types = types }; - } - } } diff --git a/ClickHouse.Driver/Diagnostic/ActivitySourceHelper.cs b/ClickHouse.Driver/Diagnostic/ActivitySourceHelper.cs index 221ac77a..a3cee8e9 100644 --- a/ClickHouse.Driver/Diagnostic/ActivitySourceHelper.cs +++ b/ClickHouse.Driver/Diagnostic/ActivitySourceHelper.cs @@ -26,9 +26,9 @@ internal static class ActivitySourceHelper internal static ActivitySource ActivitySource { get; } = CreateActivitySource(); - internal static Activity StartActivity(this ClickHouseConnection connection, string name) + internal static Activity StartActivity(this ClickHouseClient client, string name) { - if (connection is null) throw new ArgumentNullException(nameof(connection)); + if (client is null) throw new ArgumentNullException(nameof(client)); if (string.IsNullOrEmpty(name)) throw new ArgumentNullException(nameof(name)); var activity = ActivitySource.StartActivity(name, ActivityKind.Client); @@ -37,10 +37,10 @@ internal static Activity StartActivity(this ClickHouseConnection connection, str { activity.SetTag(TagThreadId, Environment.CurrentManagedThreadId.ToString(CultureInfo.InvariantCulture)); activity.SetTag(TagDbSystem, "clickhouse"); - activity.SetTag(TagDbConnectionString, connection.RedactedConnectionString); - activity.SetTag(TagDbName, connection.Database); - activity.SetTag(TagUser, connection.Username); - activity.SetTag(TagService, $"{connection.ServerUri.Host}:{connection.ServerUri.Port}{connection.ServerUri.AbsolutePath}"); + activity.SetTag(TagDbConnectionString, client.RedactedConnectionString); + activity.SetTag(TagDbName, client.Settings.Database); + activity.SetTag(TagUser, client.Settings.Username); + activity.SetTag(TagService, $"{client.ServerUri.Host}:{client.ServerUri.Port}{client.ServerUri.AbsolutePath}"); } return activity; diff --git a/ClickHouse.Driver/IClickHouseClient.cs b/ClickHouse.Driver/IClickHouseClient.cs new file mode 100644 index 00000000..12b41399 --- /dev/null +++ b/ClickHouse.Driver/IClickHouseClient.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using ClickHouse.Driver.ADO; +using ClickHouse.Driver.ADO.Parameters; +using ClickHouse.Driver.ADO.Readers; +using ClickHouse.Driver.Json; + +namespace ClickHouse.Driver; + +/// +/// Defines the contract for a ClickHouse client. +/// +public interface IClickHouseClient : IDisposable +{ + /// + /// Gets the settings used by this client. + /// + ClickHouseClientSettings Settings { get; } + + /// + /// Executes a SQL statement and returns the number of rows affected. + /// + /// The SQL statement to execute. + /// Optional parameters for the query. + /// Optional query options to override client defaults. + /// Cancellation token. + /// The number of rows affected. + Task ExecuteNonQueryAsync( + string sql, + ClickHouseParameterCollection parameters = null, + QueryOptions options = null, + CancellationToken cancellationToken = default); + + /// + /// Executes a SQL query and returns the first column of the first row. + /// + /// The SQL query to execute. + /// Optional parameters for the query. + /// Optional query options to override client defaults. + /// Cancellation token. + /// The first column of the first row, or default if no results. + Task ExecuteScalarAsync( + string sql, + ClickHouseParameterCollection parameters = null, + QueryOptions options = null, + CancellationToken cancellationToken = default); + + /// + /// Executes a SQL query and returns a data reader for iterating results. + /// + /// The SQL query to execute. + /// Optional parameters for the query. + /// Optional query options to override client defaults. + /// Cancellation token. + /// A data reader for the query results. + Task ExecuteReaderAsync( + string sql, + ClickHouseParameterCollection parameters = null, + QueryOptions options = null, + CancellationToken cancellationToken = default); + + /// + /// Executes a SQL query and returns a raw result for custom format handling. + /// + /// The SQL query to execute. + /// Optional query options to override client defaults. + /// Cancellation token. + /// A raw result containing the response stream. + Task ExecuteRawResultAsync( + string sql, + QueryOptions options = null, + CancellationToken cancellationToken = default); + + /// + /// Inserts rows into a table using the binary protocol. + /// + /// The destination table name. + /// The column names to insert into. + /// The rows to insert, where each row is an array of column values. + /// Optional insert options. + /// Cancellation token. + /// The number of rows inserted. + Task InsertBinaryAsync( + string table, + IEnumerable columns, + IEnumerable rows, + InsertOptions options = null, + CancellationToken cancellationToken = default); + + /// + /// Inserts data from a stream into a table using the specified format. + /// + /// The destination table name. + /// The stream containing the data to insert. + /// The ClickHouse format of the data (e.g., "CSV", "JSONEachRow", "Parquet"). + /// Optional column names. If null, all columns are assumed in table order. + /// Whether to compress the stream before sending (default: true) + /// Optional query options. + /// Cancellation token. + /// A task representing the asynchronous operation. + Task InsertRawStreamAsync( + string table, + Stream stream, + string format, + IEnumerable columns = null, + bool useCompression = true, + QueryOptions options = null, + CancellationToken cancellationToken = default); + + /// + /// Post a raw stream to the server. + /// + /// SQL query to add to URL, may be empty + /// Raw stream to be sent. May contain SQL query at the beginning. May be gzip-compressed + /// Indicates whether "Content-Encoding: gzip" header should be added + /// Cancellation token + /// Query options that override connection-level options + /// Task-wrapped HttpResponseMessage object + Task PostStreamAsync(string sql, Stream data, bool isCompressed, CancellationToken token, QueryOptions queryOptions = null); + + /// + /// Post a raw stream to the server using a stream-generating callback. + /// + /// SQL query to add to URL, may be empty + /// Callback invoked to write to the stream. May contain SQL query at the beginning. May be gzip-compressed + /// Iindicates whether "Content-Encoding: gzip" header should be added + /// Cancellation token + /// Query options that override connection-level options + /// Task-wrapped HttpResponseMessage object + Task PostStreamAsync(string sql, Func callback, bool isCompressed, CancellationToken token, QueryOptions queryOptions = null); + + /// + /// Pings the ClickHouse server to check if it is available. + /// + /// Query options that override connection-level options + /// Cancellation token. + /// True if the server responds successfully, false otherwise. + Task PingAsync(QueryOptions queryOptions = null, CancellationToken cancellationToken = default); + + /// + /// Registers a POCO type for JSON column serialization. + /// Types must be registered before they can be used in operations with JSON or Dynamic columns. + /// + /// The POCO type to register. + /// + /// Thrown if any property type cannot be mapped to a ClickHouse type. + /// + void RegisterJsonSerializationType() + where T : class; + + /// + /// Registers a POCO type for JSON column serialization. + /// Types must be registered before they can be used in operations with JSON or Dynamic columns. + /// + /// The POCO type to register. + /// Thrown if is null. + /// + /// Thrown if any property type cannot be mapped to a ClickHouse type. + /// + void RegisterJsonSerializationType(Type type); + + /// + /// Creates a new that uses this client's HTTP connection pool. + /// These connection instances are light and can be short-lived. + /// + /// A new connection instance. + ClickHouseConnection CreateConnection(); +} diff --git a/ClickHouse.Driver/IClickHouseConnection.cs b/ClickHouse.Driver/IClickHouseConnection.cs index c9638437..9762464c 100644 --- a/ClickHouse.Driver/IClickHouseConnection.cs +++ b/ClickHouse.Driver/IClickHouseConnection.cs @@ -12,18 +12,4 @@ public interface IClickHouseConnection : IDbConnection new ClickHouseCommand CreateCommand(string commandText = null); #pragma warning restore CS0109 // Member does not hide an inherited member; new keyword is not required - Task PingAsync(CancellationToken cancellationToken = default); - - /// - /// Registers a POCO type for JSON column serialization. - /// - /// The POCO type to register. - void RegisterJsonSerializationType() - where T : class; - - /// - /// Registers a POCO type for JSON column serialization. - /// - /// The POCO type to register. - void RegisterJsonSerializationType(Type type); } diff --git a/ClickHouse.Driver/IClickHouseDataSource.cs b/ClickHouse.Driver/IClickHouseDataSource.cs index d46f636f..3a4d0d60 100644 --- a/ClickHouse.Driver/IClickHouseDataSource.cs +++ b/ClickHouse.Driver/IClickHouseDataSource.cs @@ -13,5 +13,7 @@ public interface IClickHouseDataSource IClickHouseConnection OpenConnection(); Task OpenConnectionAsync(CancellationToken cancellationToken = default); + + IClickHouseClient GetClient(); } #endif diff --git a/ClickHouse.Driver/InsertOptions.cs b/ClickHouse.Driver/InsertOptions.cs new file mode 100644 index 00000000..18fce600 --- /dev/null +++ b/ClickHouse.Driver/InsertOptions.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using ClickHouse.Driver.Copy; + +namespace ClickHouse.Driver; + +/// +/// Options for binary insert operations that can override client-level defaults. +/// +public sealed class InsertOptions : QueryOptions +{ + /// + /// Gets or sets the number of rows per batch. Default is 100,000. + /// + public int BatchSize { get; init; } = 100_000; + + /// + /// Gets or sets the maximum number of parallel batch insert operations. Default is 1. + /// + public int MaxDegreeOfParallelism { get; init; } = 1; + + /// + /// Gets or sets the row binary format to use. Default is RowBinary. + /// + public RowBinaryFormat Format { get; init; } = RowBinaryFormat.RowBinary; +} diff --git a/ClickHouse.Driver/Logging/ClickHouseLogCategories.cs b/ClickHouse.Driver/Logging/ClickHouseLogCategories.cs index 6eaad491..93a58d7f 100644 --- a/ClickHouse.Driver/Logging/ClickHouseLogCategories.cs +++ b/ClickHouse.Driver/Logging/ClickHouseLogCategories.cs @@ -6,4 +6,5 @@ internal static class ClickHouseLogCategories internal const string Command = "ClickHouse.Driver.Command"; internal const string Transport = "ClickHouse.Driver.Transport"; internal const string BulkCopy = "ClickHouse.Driver.BulkCopy"; + internal const string Client = "ClickHouse.Driver.Client"; } diff --git a/ClickHouse.Driver/QueryOptions.cs b/ClickHouse.Driver/QueryOptions.cs new file mode 100644 index 00000000..92dd2566 --- /dev/null +++ b/ClickHouse.Driver/QueryOptions.cs @@ -0,0 +1,71 @@ +#nullable enable +using System; +using System.Collections.Generic; +using ClickHouse.Driver.ADO; + +namespace ClickHouse.Driver; + +/// +/// Options for query execution that can override client-level defaults. +/// +public class QueryOptions +{ + /// + /// Gets or sets the query identifier for tracking and logging purposes. + /// + public string? QueryId { get; init; } + + /// + /// Gets or sets the database to use for this query, overriding the client default. + /// + public string? Database { get; init; } + + /// + /// Gets or sets the roles to use for this query, overriding the client default. + /// + public IReadOnlyList? Roles { get; init; } + + /// + /// Gets or sets custom ClickHouse settings for this query (e.g., max_threads, max_memory_usage). + /// + public IDictionary? CustomSettings { get; init; } + + /// + /// Gets or sets custom HTTP headers to send with each request. + /// These headers are applied after the default headers, allowing you to override most headers. + /// The following headers cannot be overridden and will be silently ignored: + /// Connection, Authorization, User-Agent + /// Default: null + /// + public IReadOnlyDictionary? CustomHeaders { get; init; } + + /// + /// Gets or sets whether to use sessions for the connection. + /// If set to null, will not override client settings. + /// Default: null + /// + public bool? UseSession { get; init; } + + /// + /// Gets or sets the session ID to use (the value is only used if UseSession is true). + /// Default: null + /// + public string? SessionId { get; init; } + + /// + /// Gets or sets the bearer token for JWT authentication. + /// When set, Bearer authentication is used instead of Basic authentication + /// (Username and Password are ignored for the Authorization header). + /// The token should be provided as-is (already encoded if required by your auth provider). + /// Default: null + /// + public string? BearerToken { get; init; } + + /// + /// Gets or sets the maximum execution time for this query. + /// When set, this value is passed to ClickHouse as the max_execution_time setting, + /// which causes the server to cancel the query if it exceeds this duration. + /// Default: null (no limit) + /// + public TimeSpan? MaxExecutionTime { get; init; } +} diff --git a/ClickHouse.Driver/QueryResult.cs b/ClickHouse.Driver/QueryResult.cs new file mode 100644 index 00000000..4053919e --- /dev/null +++ b/ClickHouse.Driver/QueryResult.cs @@ -0,0 +1,82 @@ +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using ClickHouse.Driver.ADO; +using ClickHouse.Driver.Json; + +namespace ClickHouse.Driver; + +internal class QueryResult +{ + /// + /// Gets the HTTP response message. + /// + public HttpResponseMessage HttpResponseMessage { get; init; } + + /// + /// Gets or sets QueryId associated with command. + /// If not set before execution, a GUID will be automatically generated. + /// + public string QueryId { get; init; } + + /// + /// Gets statistics from the last executed query (rows read, bytes read, elapsed time, etc.). + /// Populated after query execution from the X-ClickHouse-Summary header. + /// + public QueryStats QueryStats { get; init; } + + /// + /// Gets the server's timezone from the last executed query response. + /// This is extracted from the X-ClickHouse-Timezone header. + /// + public string ServerTimezone { get; init; } + + public QueryResult(HttpResponseMessage httpResponseMessage) + { + HttpResponseMessage = httpResponseMessage; + QueryId = ExtractQueryId(httpResponseMessage); + QueryStats = ExtractQueryStats(httpResponseMessage); + ServerTimezone = ExtractTimezone(httpResponseMessage); + } + + internal static string ExtractQueryId(HttpResponseMessage response) + { + const string queryIdHeader = "X-ClickHouse-Query-Id"; + if (response.Headers.Contains(queryIdHeader)) + return response.Headers.GetValues(queryIdHeader).FirstOrDefault(); + else + return null; + } + + private static readonly JsonSerializerOptions SummarySerializerOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = new SnakeCaseNamingPolicy(), + NumberHandling = System.Text.Json.Serialization.JsonNumberHandling.AllowReadingFromString, + }; + + private static QueryStats ExtractQueryStats(HttpResponseMessage response) + { + try + { + const string summaryHeader = "X-ClickHouse-Summary"; + if (response.Headers.TryGetValues(summaryHeader, out var values)) + { + return JsonSerializer.Deserialize(values.First(), SummarySerializerOptions); + } + } + catch + { + } + return null; + } + + private static string ExtractTimezone(HttpResponseMessage response) + { + const string timezoneHeader = "X-ClickHouse-Timezone"; + if (response.Headers.TryGetValues(timezoneHeader, out var values)) + { + return values.FirstOrDefault(); + } + return null; + } +} diff --git a/ClickHouse.Driver/Utility/ClickHouseParameterCollectionExtensions.cs b/ClickHouse.Driver/Utility/ClickHouseParameterCollectionExtensions.cs new file mode 100644 index 00000000..5749c4f3 --- /dev/null +++ b/ClickHouse.Driver/Utility/ClickHouseParameterCollectionExtensions.cs @@ -0,0 +1,11 @@ +using ClickHouse.Driver.ADO.Parameters; + +namespace ClickHouse.Driver.Utility; + +public static class ClickHouseParameterCollectionExtensions +{ + public static void AddParameter(this ClickHouseParameterCollection parameters, string parameterName, object value) + { + parameters.Add(new ClickHouseDbParameter { ParameterName = parameterName, Value = value }); + } +} diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 803bdb06..cc727ebd 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -3,8 +3,42 @@ v? **Documentation and Usage Examples:** Coinciding with the 1.0.0 release of the driver, we have greatly expanded the documentation and usage examples. - * Documentation: https://clickhouse.com/docs/integrations/csharp - * Usage examples: https://github.com/ClickHouse/clickhouse-cs/tree/main/examples +* Documentation: https://clickhouse.com/docs/integrations/csharp +* Usage examples: https://github.com/ClickHouse/clickhouse-cs/tree/main/examples + +--- + +**New: ClickHouseClient - Simplified Primary API** + +`ClickHouseClient` is the new recommended way to interact with ClickHouse. Thread-safe, singleton-friendly, and simpler than ADO.NET classes. + +```csharp +using var client = new ClickHouseClient("Host=localhost"); +``` + +| Method | Description | +|--------|--------------------------------------------------------------| +| `ExecuteNonQueryAsync` | Execute DDL/DML (CREATE, INSERT, ALTER, DROP) | +| `ExecuteScalarAsync` | Return first column of first row | +| `ExecuteReaderAsync` | Stream results via `ClickHouseDataReader` | +| `InsertBinaryAsync` | High-performance bulk insert (replaces `ClickHouseBulkCopy`) | +| `ExecuteRawResultAsync` | Get raw result stream bypassing the parser | +| `InsertRawStreamAsync` | Insert from stream (CSV, JSON, Parquet, etc.) | +| `PingAsync` | Check server connectivity | +| `CreateConnection()` | Get `ClickHouseConnection` for ORM compatibility | + +**Per-query configuration** via `QueryOptions`. + +**Parameters** via `ClickHouseParameterCollection`: +```csharp +var parameters = new ClickHouseParameterCollection(); +parameters.Add("id", 42UL); +await client.ExecuteReaderAsync("SELECT * FROM t WHERE id = {id:UInt64}", parameters); +``` + +**Deprecation:** `ClickHouseBulkCopy` is deprecated. Use `client.InsertBinaryAsync(table, columns, rows)` instead. + +--- **Breaking Changes:** * **Dropped support for .NET Framework and .NET Standard.** The library now targets only `net6.0`, `net8.0`, `net9.0`, and `net10.0`. Removed support for `net462`, `net48`, and `netstandard2.1`. If you are using .NET Framework, you will need to stay on the previous version or migrate to .NET 6.0+. @@ -27,8 +61,8 @@ Coinciding with the 1.0.0 release of the driver, we have greatly expanded the do | `DateTime('Europe/Amsterdam')` | `DateTime` with `Kind=Unspecified` | `DateTime` with `Kind=Unspecified` (unchanged). Reading as DateTimeOffset has correct offset applied. | **Migration guidance:** If you need timezone-aware behavior, either: - 1. Use explicit timezones in your column definitions: `DateTime('UTC')` or `DateTime('Europe/Amsterdam')` - 2. Apply the timezone yourself after reading. + 1. Use explicit timezones in your column definitions: `DateTime('UTC')` or `DateTime('Europe/Amsterdam')` + 2. Apply the timezone yourself after reading. * **DateTime writing now respects `DateTime.Kind` property.** Previously, all `DateTime` values were treated as wall-clock time in the target column's timezone regardless of their `Kind` property. The new behavior: @@ -47,14 +81,14 @@ Coinciding with the 1.0.0 release of the driver, we have greatly expanded the do var wallClockTime = DateTime.SpecifyKind(myTime, DateTimeKind.Unspecified); ``` - **Important:** When using parameters, you must specify the timezone in the parameter type hint to have string values interpreted in the column timezone: + **Important:** When using parameters, you must specify the timezone in the parameter type hint to have string values interpreted in the column timezone: ```csharp + command.AddParameter("dt", myDateTime); + // Correct: timezone in type hint ensures proper interpretation - command.AddParameter("dt", myDateTime, "DateTime('Europe/Amsterdam')"); command.CommandText = "INSERT INTO table (dt_column) VALUES ({dt:DateTime('Europe/Amsterdam')})"; // Gotcha: without timezone hint, UTC is used for interpretation - command.AddParameter("dt", myDateTime); command.CommandText = "INSERT INTO table (dt_column) VALUES ({dt:DateTime})"; // ^ String value interpreted in UTC, not column timezone! ``` @@ -75,41 +109,49 @@ Coinciding with the 1.0.0 release of the driver, we have greatly expanded the do | POCO (unregistered) | Exception | Serialized via `JsonSerializer.Serialize()` | **Impact if you don't modify your code:** - - JSON writing will still work, but uses string serialization instead of binary encoding; the json string will be parsed on the server instead of the client. This could lead to subtle changes in paths without type hints, eg values previously parsed as ints may be parsed as longs. - - `ClickHouseJsonPath` and `ClickHouseJsonIgnore` attributes are ignored in String mode (they only work in Binary mode). Serialization happens via System.Text.Json, so you can use those attributes instead. - - Server setting `input_format_binary_read_json_as_string=1` is automatically set when using String write mode + - JSON writing will still work, but uses string serialization instead of binary encoding; the JSON string will be parsed on the server instead of the client. This could lead to subtle changes in paths without type hints, e.g., values previously parsed as ints may be parsed as longs. + - `ClickHouseJsonPath` and `ClickHouseJsonIgnore` attributes are ignored in String mode (they only work in Binary mode). Serialization happens via `System.Text.Json`, so you can use those attributes instead. + - Server setting `input_format_binary_read_json_as_string=1` is automatically set when using String write mode **New Features/Improvements:** - * `ClickHouseBulkCopy.InitAsync()` is now called automatically before the first write operation. Manual calls to `InitAsync()` are no longer required and the method has been marked as `[Obsolete]`. It will be removed in a future version. Existing code that calls `InitAsync()` will continue to work but will generate a compiler warning. - * **Automatic parameter type extraction from SQL.** Types specified in the SQL query using `{name:Type}` syntax are now automatically used for parameter formatting, eliminating the need to specify the type twice: - ```csharp - // Before: type specified twice - command.CommandText = "SELECT {dt:DateTime('Europe/Amsterdam')}"; - command.AddParameter("dt", "DateTime('Europe/Amsterdam')", value); - - // After: type extracted from SQL automatically - command.CommandText = "SELECT {dt:DateTime('Europe/Amsterdam')}"; - command.AddParameter("dt", value); - ``` - The `AddParameter(name, type, value)` overload is now marked obsolete. Use `AddParameterWithTypeOverride()` if you need to explicitly override the SQL type hint. - * Added POCO serialization support for JSON columns. When writing POCOs to JSON columns with typed hints (e.g., `JSON(id Int64, name String)`), the driver now serializes properties using the hinted types for full type fidelity. Properties without a corresponding hinted path will have their ClickHouse types inferred automatically. Two attributes are available: `[ClickHouseJsonPath("path")]` for custom JSON paths and `[ClickHouseJsonIgnore]` to exclude properties. Property name matching to hint paths is case-sensitive (matching ClickHouse behavior which allows paths like `userName` and `UserName` to coexist). Types must be explicitly registered on the connection using `connection.RegisterJsonSerializationType()`. - * Added `JsonReadMode` and `JsonWriteMode` connection string settings for configurable JSON handling: - - `JsonReadMode.Binary` (default): Returns `System.Text.Json.Nodes.JsonObject` - - `JsonReadMode.String`: Returns raw JSON string. Sets server setting `output_format_binary_write_json_as_string=1`. - - `JsonWriteMode.String` (default): Accepts `JsonObject`, `JsonNode`, strings, and any object (serialized via `System.Text.Json.JsonSerializer`). Sets Server setting `input_format_binary_read_json_as_string=1`. - - `JsonWriteMode.Binary`: Only accepts registered POCO types with full type hint support and custom path attributes. Writing `string` or `JsonNode` values with `JsonWriteMode.Binary` throws an exception. - * Added support for QBit data type. QBit is a transposed vector column, designed to allow the user to choose a desired quantization level at runtime, speeding up approximate similarity searches. See the GitHub repo for usage examples. - * Added support for writing `ReadOnlyMemory` and `Stream` values to String and FixedString columns via BulkCopy. - * Added `ReadStringsAsByteArrays` connection string setting to read String columns as `byte[]` instead of `string`. This is useful when storing binary data in String columns that may not be valid UTF-8. - * Added support for setting roles at the connection and command levels. - * Added support for custom headers at the connection level. - * Added support for JWT/Bearer token authentication at both connection and command levels. - * Added `InsertRawStreamAsync` method to `ClickHouseConnection` for inserting raw data streams (CSV, JSON, Parquet, etc.) directly from files or memory. Check out the examples on GitHub for usage examples of all the above. - * When the query id has not been set, it will now be automatically generated by the client. - * Added support for writing `byte[]` values to String type columns via BulkCopy. - * Added `PingAsync` method to `ClickHouseConnection` for checking server availability via the `/ping` endpoint. - * Added support for detecting mid-stream exceptions via the `X-ClickHouse-Exception-Tag` header (ClickHouse 25.11+). When `http_write_exception_in_output_format` is set to 0 on the server, exceptions that occur while streaming results are now properly detected and thrown as `ClickHouseServerException` (which includes the exception message) instead of `EndOfStreamException`. - * Added support for writing to `Dynamic` type columns via BulkCopy. Values are automatically type-inferred from their .NET types and serialized with the appropriate binary type header. Supports all common types including integers, floating point, strings, booleans, DateTime, Guid, decimal, arrays, lists, and dictionaries. + +* **Automatic parameter type extraction from SQL.** Types specified in the SQL query using `{name:Type}` syntax are now automatically used for parameter formatting, eliminating the need to specify the type twice: + ```csharp + // Before: type specified twice + command.CommandText = "SELECT {dt:DateTime('Europe/Amsterdam')}"; + command.AddParameter("dt", "DateTime('Europe/Amsterdam')", value); + + // After: type extracted from SQL automatically + command.CommandText = "SELECT {dt:DateTime('Europe/Amsterdam')}"; + command.AddParameter("dt", value); + ``` + The `AddParameter(name, type, value)` overload is now marked obsolete. Use `AddParameterWithTypeOverride()` if you need to explicitly override the SQL type hint. + +* **POCO serialization support for JSON columns.** When writing POCOs to JSON columns with typed hints (e.g., `JSON(id Int64, name String)`), the driver serializes properties using the hinted types for full type fidelity. Properties without a corresponding hinted path will have their ClickHouse types inferred automatically. Two attributes are available: `[ClickHouseJsonPath("path")]` for custom JSON paths and `[ClickHouseJsonIgnore]` to exclude properties. Property name matching to hint paths is case-sensitive (matching ClickHouse behavior which allows paths like `userName` and `UserName` to coexist). Register types via `client.RegisterJsonSerializationType()`. + +* **`JsonReadMode` and `JsonWriteMode` connection string settings** for configurable JSON handling: + - `JsonReadMode.Binary` (default): Returns `System.Text.Json.Nodes.JsonObject` + - `JsonReadMode.String`: Returns raw JSON string. Sets server setting `output_format_binary_write_json_as_string=1`. + - `JsonWriteMode.String` (default): Accepts `JsonObject`, `JsonNode`, strings, and any object (serialized via `System.Text.Json.JsonSerializer`). Sets server setting `input_format_binary_read_json_as_string=1`. + - `JsonWriteMode.Binary`: Only accepts registered POCO types with full type hint support and custom path attributes. Writing `string` or `JsonNode` values with `JsonWriteMode.Binary` throws an exception. + +* **QBit data type support.** QBit is a transposed vector column, designed to allow the user to choose a desired quantization level at runtime, speeding up approximate similarity searches. See the GitHub repo for usage examples. + +* **Dynamic type binary writing support** via `InsertBinaryAsync`. Values are automatically type-inferred from their .NET types and serialized with the appropriate binary type header. Supports all common types including integers, floating point, strings, booleans, DateTime, Guid, decimal, arrays, lists, and dictionaries. + +* **Binary data in String/FixedString columns.** Write `byte[]`, `ReadOnlyMemory`, or `Stream` values to String and FixedString columns via `InsertBinaryAsync`. Read binary data back using the `ReadStringsAsByteArrays` connection string setting, which returns String columns as `byte[]` instead of `string`. Useful for storing binary data that may not be valid UTF-8. + +* **First-class support for roles**, with query-level override. + +* **Custom HTTP headers** at the connection level for proxy/infrastructure integration. + +* **Support for JWT authentication**, with query-level override. + +* **Mid-stream exception detection** via `X-ClickHouse-Exception-Tag` header (ClickHouse 25.11+). When `http_write_exception_in_output_format` is set to 0 on the server, exceptions that occur while streaming results are now properly detected and thrown as `ClickHouseServerException` (which includes the exception message) instead of `EndOfStreamException`. + +* **Query ID auto-generation.** When the query ID has not been set, it will now be automatically generated by the client. + +* **`AddParameter()` convenience method** for `ClickHouseParameterCollection`, simplifying parameter creation. **Bug Fixes:** - * Fixed a crash when reading a Map with duplicate keys. The current behavior is to return only the last value for a given key. +* Fixed a crash when reading a Map with duplicate keys. The current behavior is to return only the last value for a given key. diff --git a/examples/Advanced/Advanced_001_QueryIdUsage.cs b/examples/Advanced/Advanced_001_QueryIdUsage.cs index 92ef296d..3fc4b7ce 100644 --- a/examples/Advanced/Advanced_001_QueryIdUsage.cs +++ b/examples/Advanced/Advanced_001_QueryIdUsage.cs @@ -1,4 +1,5 @@ using ClickHouse.Driver.ADO; +using ClickHouse.Driver.ADO.Parameters; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -15,83 +16,68 @@ public static class QueryIdUsage { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); Console.WriteLine("Query ID Usage Examples\n"); // Example 1: Automatic Query ID Console.WriteLine("1. Automatic Query ID assignment:"); - await Example1_AutomaticQueryId(connection); + await Example1_AutomaticQueryId(client); // Example 2: Custom Query ID Console.WriteLine("\n2. Setting a custom Query ID:"); - await Example2_CustomQueryId(connection); + await Example2_CustomQueryId(client); // Example 3: Tracking query execution Console.WriteLine("\n3. Tracking query execution in system.query_log:"); - await Example3_TrackingQueryExecution(connection); + await Example3_TrackingQueryExecution(client); // Example 4: Cancelling a query by Query ID Console.WriteLine("\n4. Query cancellation using Query ID:"); - await Example4_QueryCancellation(connection); + await Example4_QueryCancellation(client); Console.WriteLine("\nAll Query ID examples completed!"); } - private static async Task Example1_AutomaticQueryId(ClickHouseConnection connection) + private static async Task Example1_AutomaticQueryId(ClickHouseClient client) { // When you don't set a QueryId, the client automatically generates a GUID - using var command = connection.CreateCommand(); - command.CommandText = "SELECT 'Hello from ClickHouse' AS message"; - - Console.WriteLine($" QueryId before execution: {command.QueryId ?? "(null)"}"); - - var result = await command.ExecuteScalarAsync(); - - // After execution, the QueryId contains the auto-generated GUID - Console.WriteLine($" QueryId after execution: {command.QueryId}"); + var result = await client.ExecuteScalarAsync("SELECT 'Hello from ClickHouse' AS message"); Console.WriteLine($" Result: {result}"); + Console.WriteLine(" (QueryId was auto-generated for this query)"); } - private static async Task Example2_CustomQueryId(ClickHouseConnection connection) + private static async Task Example2_CustomQueryId(ClickHouseClient client) { // You can set your own Query ID before executing a query // This is useful for correlation with your application logs var customQueryId = $"example-{Guid.NewGuid()}"; - - using var command = connection.CreateCommand(); - command.CommandText = "SELECT version()"; - command.QueryId = customQueryId; + var options = new QueryOptions { QueryId = customQueryId }; Console.WriteLine($" Custom QueryId: {customQueryId}"); - var version = await command.ExecuteScalarAsync(); + var version = await client.ExecuteScalarAsync("SELECT version()", options: options); Console.WriteLine($" ClickHouse version: {version}"); - Console.WriteLine($" QueryId remained: {command.QueryId}"); } - private static async Task Example3_TrackingQueryExecution(ClickHouseConnection connection) + private static async Task Example3_TrackingQueryExecution(ClickHouseClient client) { // Execute a query with a custom Query ID var trackableQueryId = $"trackable-{Guid.NewGuid()}"; - using (var command = connection.CreateCommand()) - { - command.CommandText = $"SELECT 1"; - command.QueryId = trackableQueryId; - await command.ExecuteNonQueryAsync(); - } + var options = new QueryOptions { QueryId = trackableQueryId }; + await client.ExecuteNonQueryAsync("SELECT 1", options: options); Console.WriteLine($" Executed query with ID: {trackableQueryId}"); // Wait a moment for the query to be logged - await Task.Delay(1000); + await Task.Delay(2000); // Query system.query_log to get information about our query - // Note: system.query_log may need to be enabled in your ClickHouse configuration - using (var command = connection.CreateCommand()) + var parameters = new ClickHouseParameterCollection(); + parameters.AddParameter("queryId", trackableQueryId); + try { - command.CommandText = @" + using var reader = await client.ExecuteReaderAsync(@" SELECT query_id, type, @@ -104,52 +90,44 @@ FROM system.query_log AND type = 'QueryFinish' ORDER BY event_time DESC LIMIT 1 - "; - command.AddParameter("queryId", trackableQueryId); + ", parameters); - try + if (reader.Read()) { - using var reader = await command.ExecuteReaderAsync(); - if (reader.Read()) - { - Console.WriteLine(" Query execution details from system.query_log:"); - Console.WriteLine($" Query ID: {reader.GetString(0)}"); - Console.WriteLine($" Type: {reader.GetString(1)}"); - Console.WriteLine($" Duration: {reader.GetFieldValue(2)} ms"); - Console.WriteLine($" Rows read: {reader.GetFieldValue(3)}"); - Console.WriteLine($" Rows written: {reader.GetFieldValue(4)}"); - Console.WriteLine($" Memory usage: {reader.GetFieldValue(5)} bytes"); - } - else - { - Console.WriteLine(" (Query not yet in system.query_log - this table may have a delay or be disabled)"); - } + Console.WriteLine(" Query execution details from system.query_log:"); + Console.WriteLine($" Query ID: {reader.GetString(0)}"); + Console.WriteLine($" Type: {reader.GetString(1)}"); + Console.WriteLine($" Duration: {reader.GetFieldValue(2)} ms"); + Console.WriteLine($" Rows read: {reader.GetFieldValue(3)}"); + Console.WriteLine($" Rows written: {reader.GetFieldValue(4)}"); + Console.WriteLine($" Memory usage: {reader.GetFieldValue(5)} bytes"); } - catch (ClickHouseServerException ex) when (ex.ErrorCode == 60) + else { - Console.WriteLine(" (system.query_log table not available on this server)"); + Console.WriteLine(" (Query not yet in system.query_log - this table may have a delay or be disabled)"); } } + catch (ClickHouseServerException ex) when (ex.ErrorCode == 60) + { + Console.WriteLine(" (system.query_log table not available on this server)"); + } } - private static async Task Example4_QueryCancellation(ClickHouseConnection connection) + private static async Task Example4_QueryCancellation(ClickHouseClient client) { // Demonstrate cancelling a long-running query using Query ID var cancellableQueryId = $"cancellable-{Guid.NewGuid()}"; + var options = new QueryOptions { QueryId = cancellableQueryId }; Console.WriteLine($" Query ID: {cancellableQueryId}"); - Console.WriteLine(" Starting a long-running query (SELECT sleep(5))..."); + Console.WriteLine(" Starting a long-running query (SELECT sleep(3))..."); // Start the long-running query in a background task var queryTask = Task.Run(async () => { try { - using var command = connection.CreateCommand(); - command.CommandText = "SELECT sleep(3)"; - command.QueryId = cancellableQueryId; - - await command.ExecuteScalarAsync(); + await client.ExecuteScalarAsync("SELECT sleep(3)", options: options); Console.WriteLine(" Query completed (should have been cancelled)"); } catch (ClickHouseServerException ex) @@ -163,21 +141,16 @@ private static async Task Example4_QueryCancellation(ClickHouseConnection connec } }, CancellationToken.None); - // Wait a bit for the query to start and be present in the log + // Wait a bit for the query to start await Task.Delay(1000); - // Cancel using KILL QUERY from another connection. Note that closing a connection will NOT kill any running queries opened by that connection. + // Cancel using KILL QUERY Console.WriteLine($" Cancelling query using KILL QUERY..."); try { - // Create a separate connection for cancellation - using var cancelConnection = new ClickHouseConnection("Host=localhost"); - await cancelConnection.OpenAsync(); - - using var cancelCommand = cancelConnection.CreateCommand(); - cancelCommand.CommandText = $"KILL QUERY WHERE query_id = '{cancellableQueryId}'"; - await cancelCommand.ExecuteNonQueryAsync(); - + var killParams = new ClickHouseParameterCollection(); + killParams.AddParameter("queryId", cancellableQueryId); + await client.ExecuteNonQueryAsync("KILL QUERY WHERE query_id = {queryId:String}", killParams); Console.WriteLine(" KILL QUERY command sent"); } catch (Exception ex) diff --git a/examples/Advanced/Advanced_002_SessionIdUsage.cs b/examples/Advanced/Advanced_002_SessionIdUsage.cs index 81c774bd..67a401e0 100644 --- a/examples/Advanced/Advanced_002_SessionIdUsage.cs +++ b/examples/Advanced/Advanced_002_SessionIdUsage.cs @@ -8,16 +8,6 @@ namespace ClickHouse.Driver.Examples; /// Sessions are primarily used for: /// - Creating and using temporary tables /// - Maintaining query context across multiple statements -/// -/// IMPORTANT LIMITATION: When UseSession is enabled with a SessionId, the driver creates -/// a single-connection HttpClientFactory instead of using a pooled connection. This means -/// all queries in the session will use the same underlying HTTP connection, which is not -/// suitable for high-performance or high-concurrency scenarios. -/// -/// Making queries using the same id from multiple connections simultaneously will cause errors. -/// -/// Consider using regular tables with TTL instead of temporary tables -/// if you need to share data across multiple connections /// public static class SessionIdUsage { @@ -25,7 +15,6 @@ public static async Task Run() { Console.WriteLine("Session ID Usage Examples\n"); - // Example 1: Using sessions for temporary tables // To use temporary tables, you must enable sessions var settings = new ClickHouseClientSettings { @@ -34,14 +23,13 @@ public static async Task Run() // If you don't set SessionId, a GUID will be automatically generated }; - using var connection = new ClickHouseConnection(settings); - await connection.OpenAsync(); + using var client = new ClickHouseClient(settings); Console.WriteLine($" Session ID: {settings.SessionId}"); // Create a temporary table // Temporary tables only exist within the session and are automatically dropped - await connection.ExecuteStatementAsync(@" + await client.ExecuteNonQueryAsync(@" CREATE TEMPORARY TABLE temp_users ( id UInt64, @@ -52,18 +40,15 @@ email String Console.WriteLine(" Created temporary table 'temp_users'"); // Insert data into the temporary table - using (var command = connection.CreateCommand()) + var rows = new List { - command.CommandText = "INSERT INTO temp_users (id, name, email) VALUES ({id:UInt64}, {name:String}, {email:String})"; - command.AddParameter("id", 1UL); - command.AddParameter("name", "Alice"); - command.AddParameter("email", "alice@example.com"); - await command.ExecuteNonQueryAsync(); - } + new object[] { 1UL, "Alice", "alice@example.com" }, + }; + await client.InsertBinaryAsync("temp_users", new[] { "id", "name", "email" }, rows); Console.WriteLine(" Inserted data into temporary table"); // Query the temporary table - using (var reader = await connection.ExecuteReaderAsync("SELECT id, name, email FROM temp_users ORDER BY id")) + using (var reader = await client.ExecuteReaderAsync("SELECT id, name, email FROM temp_users ORDER BY id")) { Console.WriteLine("\n Data from temporary table:"); Console.WriteLine(" ID\tName\tEmail"); @@ -77,8 +62,8 @@ email String } } - // Temporary tables are automatically dropped when the connection closes - Console.WriteLine("\n Temporary table will be dropped when connection closes"); + // Temporary tables are automatically dropped when the session ends + Console.WriteLine("\n Temporary table will be dropped when session ends"); Console.WriteLine("\nAll Session ID examples completed!"); } diff --git a/examples/Advanced/Advanced_003_LongRunningQueries.cs b/examples/Advanced/Advanced_003_LongRunningQueries.cs index 42e0eb9d..8b9df619 100644 --- a/examples/Advanced/Advanced_003_LongRunningQueries.cs +++ b/examples/Advanced/Advanced_003_LongRunningQueries.cs @@ -1,4 +1,5 @@ using ClickHouse.Driver.ADO; +using ClickHouse.Driver.ADO.Parameters; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -7,7 +8,7 @@ namespace ClickHouse.Driver.Examples; /// Demonstrates strategies for handling long-running queries that might be terminated /// by load balancers or proxies due to idle connections. Typical idle timeouts are ~30s. /// -/// These are typically INSERT .. FROM SELECT type queries that move large amounts of data. +/// These are typically INSERT .. FROM SELECT type queries that move large amounts of data. /// /// Two main approaches are shown: /// 1. Progress Headers: Keep the connection alive by having ClickHouse send periodic progress updates @@ -32,65 +33,63 @@ public static async Task Run() private static async Task Example1_ProgressHeaders() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); Console.WriteLine(" Configuring query with progress headers..."); Console.WriteLine(" This approach keeps the HTTP connection alive by sending periodic progress updates."); Console.WriteLine(); - // Execute a query with progress headers enabled - using (var command = connection.CreateCommand()) + // Enable progress headers to keep the connection alive + // These settings tell ClickHouse to send HTTP headers with progress information + // If your queries are extremely long, they may fill up the default 64kb header buffer. + // In that case you could modify HttpClientHandler's MaxResponseHeadersLength property, + // but the preferred solution is to break up the data into more manageable chunks. + var options = new QueryOptions { - // This simulates a long-running query using the sleep() function - command.CommandText = @" - SELECT - *, - sleep(0.01) as delay - FROM system.numbers - LIMIT 100 - "; - - // Enable progress headers to keep the connection alive - // These settings tell ClickHouse to send HTTP headers with progress information - // It is also possible to set these settings at the Connection level. - // If your queries are extremely long, they may fill up the default 64kb header buffer. - // In that you could modify HttpClientHandler's MaxResponseHeadersLength property, - // but the preferred solution is to break up the data into more manageable chunks. - command.CustomSettings.Add("send_progress_in_http_headers", 1); - command.CustomSettings.Add("http_headers_progress_interval_ms", "1000"); - - Console.WriteLine(" Custom Settings configured:"); - Console.WriteLine(" send_progress_in_http_headers = 1"); - Console.WriteLine(" http_headers_progress_interval_ms = 1000"); - Console.WriteLine(); - Console.WriteLine(" Executing query (this may take a moment)..."); - - var startTime = DateTime.UtcNow; - var rowCount = 0; - - using var reader = await command.ExecuteReaderAsync(); - while (reader.Read()) + CustomSettings = new Dictionary { - rowCount++; - } + ["send_progress_in_http_headers"] = 1, + ["http_headers_progress_interval_ms"] = "1000", + }, + }; + + Console.WriteLine(" Custom Settings configured:"); + Console.WriteLine(" send_progress_in_http_headers = 1"); + Console.WriteLine(" http_headers_progress_interval_ms = 1000"); + Console.WriteLine(); + Console.WriteLine(" Executing query (this may take a moment)..."); - var duration = DateTime.UtcNow - startTime; - Console.WriteLine($" Query completed: {rowCount} rows in {duration.TotalMilliseconds:F0}ms"); + var startTime = DateTime.UtcNow; + var rowCount = 0; + + // This simulates a long-running query using the sleep() function + using var reader = await client.ExecuteReaderAsync(@" + SELECT + *, + sleep(0.01) as delay + FROM system.numbers + LIMIT 100 + ", options: options); + + while (reader.Read()) + { + rowCount++; } + + var duration = DateTime.UtcNow - startTime; + Console.WriteLine($" Query completed: {rowCount} rows in {duration.TotalMilliseconds:F0}ms"); } private static async Task Example2_FireAndForget() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); Console.WriteLine(" Fire-and-forget pattern for very long queries..."); Console.WriteLine(); // Create a test table var tableName = "example_fire_and_forget"; - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( id UInt64, @@ -102,6 +101,7 @@ ORDER BY id // Generate a unique query ID var queryId = $"long-query-{Guid.NewGuid()}"; + var options = new QueryOptions { QueryId = queryId }; Console.WriteLine($" Query ID: {queryId}"); // Start a long-running query (in practice, this would be much longer) @@ -112,27 +112,23 @@ ORDER BY id // 4. Periodically poll system.query_log to check status Console.WriteLine("\n Starting the query..."); - using (var command = connection.CreateCommand()) - { - command.CommandText = $@" - INSERT INTO {tableName} (id, data) - SELECT number, toString(number) - FROM numbers(10000) - "; - command.QueryId = queryId; - - await command.ExecuteNonQueryAsync(); - Console.WriteLine(" Query submitted successfully"); - } + await client.ExecuteNonQueryAsync($@" + INSERT INTO {tableName} (id, data) + SELECT number, toString(number) + FROM numbers(10000) + ", options: options); + Console.WriteLine(" Query submitted successfully"); // Wait a moment for the query to be logged await Task.Delay(500); // Poll system.query_log to check query status Console.WriteLine("\n Checking query status in system.query_log..."); - using (var command = connection.CreateCommand()) + var parameters = new ClickHouseParameterCollection(); + parameters.AddParameter("queryId", queryId); + try { - command.CommandText = @" + using var reader = await client.ExecuteReaderAsync(@" SELECT query_id, type, @@ -144,36 +140,31 @@ FROM system.query_log WHERE query_id = {queryId:String} ORDER BY event_time DESC LIMIT 2 - "; - command.AddParameter("queryId", queryId); + ", parameters); - try + if (reader.Read()) { - using var reader = await command.ExecuteReaderAsync(); - if (reader.Read()) - { - Console.WriteLine(" Query status from system.query_log:"); - Console.WriteLine($" Query ID: {reader.GetString(0)}"); - Console.WriteLine($" Status: {reader.GetString(1)}"); - Console.WriteLine($" Duration: {reader.GetFieldValue(2)} ms"); - Console.WriteLine($" Rows read: {reader.GetFieldValue(3)}"); - Console.WriteLine($" Rows written: {reader.GetFieldValue(4)}"); - Console.WriteLine($" Result rows: {reader.GetFieldValue(5)}"); - } - else - { - Console.WriteLine(" (Query not yet in system.query_log)"); - } + Console.WriteLine(" Query status from system.query_log:"); + Console.WriteLine($" Query ID: {reader.GetString(0)}"); + Console.WriteLine($" Status: {reader.GetString(1)}"); + Console.WriteLine($" Duration: {reader.GetFieldValue(2)} ms"); + Console.WriteLine($" Rows read: {reader.GetFieldValue(3)}"); + Console.WriteLine($" Rows written: {reader.GetFieldValue(4)}"); + Console.WriteLine($" Result rows: {reader.GetFieldValue(5)}"); } - catch (ClickHouseServerException ex) when (ex.ErrorCode == 60) + else { - Console.WriteLine(" (system.query_log not available)"); + Console.WriteLine(" (Query not yet in system.query_log)"); } } + catch (ClickHouseServerException ex) when (ex.ErrorCode == 60) + { + Console.WriteLine(" (system.query_log not available)"); + } Console.WriteLine(); // Clean up - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } } diff --git a/examples/Advanced/Advanced_004_CustomSettings.cs b/examples/Advanced/Advanced_004_CustomSettings.cs index 17c17a70..dc82e3ab 100644 --- a/examples/Advanced/Advanced_004_CustomSettings.cs +++ b/examples/Advanced/Advanced_004_CustomSettings.cs @@ -1,5 +1,4 @@ using ClickHouse.Driver.ADO; -using ClickHouse.Driver.ADO.Readers; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -7,8 +6,8 @@ namespace ClickHouse.Driver.Examples; /// /// Demonstrates how to use custom ClickHouse settings to control query behavior. /// Settings can be applied at: -/// - Connection level (applies to all queries on that connection) -/// - Command level (applies to a specific query) +/// - Client level (applies to all queries) +/// - Query level via QueryOptions (applies to a specific query) /// /// Common use cases: /// - Resource limits (max_execution_time, max_memory_usage) @@ -22,13 +21,13 @@ public static async Task Run() { Console.WriteLine("Custom Settings Examples\n"); - // Example 1: Connection-level settings - Console.WriteLine("1. Connection-level custom settings:"); - await Example1_ConnectionLevelSettings(); + // Example 1: Client-level settings + Console.WriteLine("1. Client-level custom settings:"); + await Example1_ClientLevelSettings(); - // Example 2: Command-level settings - Console.WriteLine("\n2. Command-level custom settings:"); - await Example2_CommandLevelSettings(); + // Example 2: Query-level settings + Console.WriteLine("\n2. Query-level custom settings:"); + await Example2_QueryLevelSettings(); // Example 3: Execution time limits Console.WriteLine("\n3. Setting execution time limits:"); @@ -37,25 +36,24 @@ public static async Task Run() Console.WriteLine("\nAll custom settings examples completed!"); } - private static async Task Example1_ConnectionLevelSettings() + private static async Task Example1_ClientLevelSettings() { - // Settings applied at the connection level affect all queries + // Settings applied at the client level affect all queries var settings = new ClickHouseClientSettings("Host=localhost"); // Add custom ClickHouse settings settings.CustomSettings.Add("max_threads", 4); settings.CustomSettings.Add("max_block_size", 65536); - using var connection = new ClickHouseConnection(settings); - await connection.OpenAsync(); + using var client = new ClickHouseClient(settings); - Console.WriteLine(" Connection-level settings applied:"); + Console.WriteLine(" Client-level settings applied:"); Console.WriteLine(" max_threads = 4"); Console.WriteLine(" max_block_size = 65536"); // Verify the settings are actually configured by querying system.settings Console.WriteLine("\n Verifying settings from system.settings:"); - using (var reader = await connection.ExecuteReaderAsync(@" + using (var reader = await client.ExecuteReaderAsync(@" SELECT name, value FROM system.settings WHERE name IN ('max_threads', 'max_block_size') @@ -71,31 +69,31 @@ ORDER BY name } } - private static async Task Example2_CommandLevelSettings() + private static async Task Example2_QueryLevelSettings() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); - Console.WriteLine(" Applying settings to a specific command:"); + Console.WriteLine(" Applying settings to a specific query:"); - using var command = connection.CreateCommand(); - command.CommandText = "SELECT number FROM numbers(10)"; - - // Command-level settings override connection-level settings - command.CustomSettings.Add("max_execution_time", 5); // 5 seconds - command.CustomSettings.Add("result_overflow_mode", "break"); - - Console.WriteLine(" max_execution_time = 5"); - Console.WriteLine(" result_overflow_mode = 'break'"); + // Query-level settings override client-level settings + var options = new QueryOptions + { + CustomSettings = new Dictionary + { + ["max_execution_time"] = 5, + ["result_overflow_mode"] = "break", + }, + }; // Verify the settings are actually configured by querying system.settings Console.WriteLine("\n Verifying settings from system.settings:"); - using (var reader = await connection.ExecuteReaderAsync(@" + string sql = @" SELECT name, value FROM system.settings - WHERE name IN ('max_threads', 'max_block_size') + WHERE name IN ('max_execution_time', 'result_overflow_mode') ORDER BY name - ")) + "; + using (var reader = await client.ExecuteReaderAsync(sql, options: options)) { while (reader.Read()) { @@ -108,17 +106,17 @@ ORDER BY name private static async Task Example3_ExecutionTimeLimits() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); Console.WriteLine(" Setting max_execution_time to limit query duration:"); - using var command = connection.CreateCommand(); - command.CommandText = @" - SELECT sleep(0.1), number - FROM numbers(100) - "; - command.CustomSettings.Add("max_execution_time", 1); // 1 second + var options = new QueryOptions + { + CustomSettings = new Dictionary + { + ["max_execution_time"] = 1, + }, + }; try { @@ -126,7 +124,11 @@ FROM numbers(100) var startTime = DateTime.UtcNow; var rowCount = 0; - using var reader = await command.ExecuteReaderAsync(); + using var reader = await client.ExecuteReaderAsync(@" + SELECT sleep(0.1), number + FROM numbers(100) + ", options: options); + while (reader.Read()) { rowCount++; diff --git a/examples/Advanced/Advanced_005_QueryStatistics.cs b/examples/Advanced/Advanced_005_QueryStatistics.cs index 3dba2bd3..95389525 100644 --- a/examples/Advanced/Advanced_005_QueryStatistics.cs +++ b/examples/Advanced/Advanced_005_QueryStatistics.cs @@ -63,5 +63,4 @@ public static async Task Run() Console.WriteLine("\nAll query statistics examples completed!"); } - } diff --git a/examples/Advanced/Advanced_007_CustomHeaders.cs b/examples/Advanced/Advanced_007_CustomHeaders.cs index f22da4cc..3d90b038 100644 --- a/examples/Advanced/Advanced_007_CustomHeaders.cs +++ b/examples/Advanced/Advanced_007_CustomHeaders.cs @@ -1,11 +1,10 @@ -using System.Collections.Generic; using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; /// -/// Demonstrates how to use custom HTTP headers with ClickHouse connections. +/// Demonstrates how to use custom HTTP headers with ClickHouse. /// Custom headers can be used for proxy authentication, distributed tracing via correlation ids, etc. /// /// The Authorization, User-Agent, and Connection headers cannot be overridden. @@ -35,10 +34,8 @@ public static async Task Run() try { - using var connection = new ClickHouseConnection(settings); - await connection.OpenAsync(); - - var result = await connection.ExecuteScalarAsync("SELECT 42"); + using var client = new ClickHouseClient(settings); + var result = await client.ExecuteScalarAsync("SELECT 42"); Console.WriteLine($" Query executed successfully: {result}"); } catch diff --git a/examples/Advanced/Advanced_008_QueryCancellation.cs b/examples/Advanced/Advanced_008_QueryCancellation.cs index e4ea942e..1a7c87b2 100644 --- a/examples/Advanced/Advanced_008_QueryCancellation.cs +++ b/examples/Advanced/Advanced_008_QueryCancellation.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -26,13 +25,7 @@ private static async Task CancelWithTimeout() { Console.WriteLine("1. Cancel query after timeout:"); - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); - - using var command = connection.CreateCommand(); - - // This query would take 3 seconds to complete, but we'll cancel it after 1 second - command.CommandText = "SELECT sleep(3)"; + using var client = new ClickHouseClient("Host=localhost"); // Create a cancellation token that will cancel after 1 second using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); @@ -42,7 +35,8 @@ private static async Task CancelWithTimeout() try { - await command.ExecuteNonQueryAsync(cts.Token); + // This query would take 3 seconds to complete, but we'll cancel it after 1 second + await client.ExecuteNonQueryAsync("SELECT sleep(3)", cancellationToken: cts.Token); Console.WriteLine(" Query completed (unexpected)"); } catch (OperationCanceledException) @@ -66,14 +60,7 @@ private static async Task CancelManually() { Console.WriteLine("\n2. Cancel query manually from another task:"); - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); - - using var command = connection.CreateCommand(); - - // This query would take 3 seconds to complete - command.CommandText = "SELECT sleep(3)"; - + using var client = new ClickHouseClient("Host=localhost"); using var cts = new CancellationTokenSource(); Console.WriteLine(" Starting a 3-second query..."); @@ -91,7 +78,8 @@ private static async Task CancelManually() try { - await command.ExecuteNonQueryAsync(cts.Token); + // This query would take 3 seconds to complete + await client.ExecuteNonQueryAsync("SELECT sleep(3)", cancellationToken: cts.Token); Console.WriteLine(" Query completed (unexpected)"); } catch (OperationCanceledException) diff --git a/examples/Advanced/Advanced_009_ReadOnlyUsers.cs b/examples/Advanced/Advanced_009_ReadOnlyUsers.cs index ee9a66cf..57703b49 100644 --- a/examples/Advanced/Advanced_009_ReadOnlyUsers.cs +++ b/examples/Advanced/Advanced_009_ReadOnlyUsers.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -22,8 +21,7 @@ public static async Task Run() Console.WriteLine("This example demonstrates the limitations of READONLY = 1 users.\n"); // Setup using the default (non-read-only) user - using var defaultClient = new ClickHouseConnection("Host=localhost"); - await defaultClient.OpenAsync(); + using var defaultClient = new ClickHouseClient("Host=localhost"); // Create a unique read-only user for this example var guid = Guid.NewGuid().ToString("N"); @@ -60,16 +58,16 @@ public static async Task Run() { // Cleanup Console.WriteLine("\nCleaning up..."); - await defaultClient.ExecuteStatementAsync($"DROP TABLE IF EXISTS {TestTableName}"); - await defaultClient.ExecuteStatementAsync($"DROP USER IF EXISTS {readOnlyUsername}"); + await defaultClient.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {TestTableName}"); + await defaultClient.ExecuteNonQueryAsync($"DROP USER IF EXISTS {readOnlyUsername}"); Console.WriteLine("Cleanup complete."); } } - private static async Task SetupReadOnlyUser(ClickHouseConnection client, string username, string password) + private static async Task SetupReadOnlyUser(ClickHouseClient client, string username, string password) { // Create a read-only user with READONLY = 1 - await client.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE USER {username} IDENTIFIED WITH sha256_password BY '{password}' DEFAULT DATABASE default @@ -77,23 +75,23 @@ IDENTIFIED WITH sha256_password BY '{password}' "); // Grant access only to SHOW TABLES and SELECT on the test table - await client.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" GRANT SHOW TABLES, SELECT ON {TestTableName} TO {username} "); } - private static async Task SetupTestTable(ClickHouseConnection client) + private static async Task SetupTestTable(ClickHouseClient client) { - await client.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE OR REPLACE TABLE {TestTableName} (id UInt64, name String) ENGINE MergeTree() ORDER BY (id) "); - await client.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" INSERT INTO {TestTableName} VALUES (12, 'foo'), (42, 'bar') @@ -107,8 +105,7 @@ private static async Task DemonstrateAllowedSelect(string connectionString) { Console.WriteLine("1. Read-only user CAN query granted tables:"); - using var client = new ClickHouseConnection(connectionString); - await client.OpenAsync(); + using var client = new ClickHouseClient(connectionString); using var reader = await client.ExecuteReaderAsync($"SELECT * FROM {TestTableName}"); Console.WriteLine(" Query result:"); @@ -127,12 +124,11 @@ private static async Task DemonstrateInsertBlocked(string connectionString) { Console.WriteLine("2. Read-only user CANNOT insert data:"); - using var client = new ClickHouseConnection(connectionString); - await client.OpenAsync(); + using var client = new ClickHouseClient(connectionString); try { - await client.ExecuteStatementAsync($"INSERT INTO {TestTableName} VALUES (100, 'blocked')"); + await client.ExecuteNonQueryAsync($"INSERT INTO {TestTableName} VALUES (100, 'blocked')"); Console.WriteLine(" Unexpected success!"); } catch (ClickHouseServerException ex) @@ -150,8 +146,7 @@ private static async Task DemonstrateUnauthorizedTableBlocked(string connectionS { Console.WriteLine("3. Read-only user CANNOT query non-granted tables (e.g., system.users):"); - using var client = new ClickHouseConnection(connectionString); - await client.OpenAsync(); + using var client = new ClickHouseClient(connectionString); try { @@ -173,16 +168,19 @@ private static async Task DemonstrateSettingsBlocked(string connectionString) { Console.WriteLine("4. Read-only user CANNOT use custom ClickHouse settings:"); - using var client = new ClickHouseConnection(connectionString); - await client.OpenAsync(); + using var client = new ClickHouseClient(connectionString); - using var command = client.CreateCommand(); - command.CommandText = $"SELECT * FROM {TestTableName}"; - command.CustomSettings.Add("send_progress_in_http_headers", 1); + var options = new QueryOptions + { + CustomSettings = new Dictionary + { + ["send_progress_in_http_headers"] = 1, + }, + }; try { - using var reader = await command.ExecuteReaderAsync(); + using var reader = await client.ExecuteReaderAsync($"SELECT * FROM {TestTableName}", options: options); Console.WriteLine(" Unexpected success!"); } catch (ClickHouseServerException ex) diff --git a/examples/Advanced/Advanced_010_RetriesAndDeduplication.cs b/examples/Advanced/Advanced_010_RetriesAndDeduplication.cs index 347a2548..5c48a722 100644 --- a/examples/Advanced/Advanced_010_RetriesAndDeduplication.cs +++ b/examples/Advanced/Advanced_010_RetriesAndDeduplication.cs @@ -1,4 +1,4 @@ -using ClickHouse.Driver.ADO; +using ClickHouse.Driver.ADO.Parameters; using ClickHouse.Driver.Utility; using Polly; using Polly.Retry; @@ -23,30 +23,29 @@ public static class RetriesAndDeduplication public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); // Create a ReplacingMergeTree table for deduplication - await SetupReplacingMergeTreeTable(connection); + await SetupReplacingMergeTreeTable(client); // Demonstrate retry with simulated random failures - await InsertWithRetryAndSimulatedFailures(connection); + await InsertWithRetryAndSimulatedFailures(client); // Show how duplicates are handled - await DemonstrateDuplicateHandling(connection); + await DemonstrateDuplicateHandling(client); - await Cleanup(connection); + await Cleanup(client); } /// /// Creates a ReplacingMergeTree table that automatically deduplicates rows. /// The 'version' column determines which row to keep (highest version wins). /// - private static async Task SetupReplacingMergeTreeTable(ClickHouseConnection connection) + private static async Task SetupReplacingMergeTreeTable(ClickHouseClient client) { Console.WriteLine("1. Creating ReplacingMergeTree table:"); - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {TableName} ( id UInt64, data String, @@ -64,7 +63,7 @@ ORDER BY (id) /// Demonstrates using Polly to retry inserts with simulated random failures. /// Uses ClickHouse's throwIf() function to randomly fail ~33% of inserts. /// - private static async Task InsertWithRetryAndSimulatedFailures(ClickHouseConnection connection) + private static async Task InsertWithRetryAndSimulatedFailures(ClickHouseClient client) { Console.WriteLine("2. Insert with Polly retry policy (simulating ~33% failure rate):"); @@ -103,16 +102,19 @@ await retryPipeline.ExecuteAsync(async ct => { // Use throwIf() to simulate random failures (~33% chance) // This throws FUNCTION_THROW_IF_VALUE_IS_NON_ZERO (error code 395) - await connection.ExecuteStatementAsync($@" - SELECT throwIf(rand() % 3 = 0, 'Simulated transient failure for record {record.Id}!')"); + await client.ExecuteNonQueryAsync($@" + SELECT throwIf(rand() % 3 = 0, 'Simulated transient failure for record {record.Id}!')", cancellationToken: ct); // If we get here, the "pre-check" passed - do the actual insert - using var command = connection.CreateCommand(); - command.CommandText = $"INSERT INTO {TableName} (id, data, version) VALUES ({{id:UInt64}}, {{data:String}}, {{version:UInt64}})"; - command.AddParameter("id", record.Id); - command.AddParameter("data", record.Data); - command.AddParameter("version", record.Version); - await command.ExecuteNonQueryAsync(ct); + var parameters = new ClickHouseParameterCollection(); + parameters.AddParameter("id", record.Id); + parameters.AddParameter("data", record.Data); + parameters.AddParameter("version", record.Version); + + await client.ExecuteNonQueryAsync( + $"INSERT INTO {TableName} (id, data, version) VALUES ({{id:UInt64}}, {{data:String}}, {{version:UInt64}})", + parameters, + cancellationToken: ct); }); } @@ -122,22 +124,22 @@ await connection.ExecuteStatementAsync($@" /// /// Shows how ReplacingMergeTree handles duplicate inserts. /// - private static async Task DemonstrateDuplicateHandling(ClickHouseConnection connection) + private static async Task DemonstrateDuplicateHandling(ClickHouseClient client) { Console.WriteLine("3. Demonstrating duplicate handling:"); // Insert a "duplicate" with the same id but higher version (simulating a retry) Console.WriteLine(" Inserting duplicate of id=1 with version=2 (simulating retry)..."); - await connection.ExecuteStatementAsync( + await client.ExecuteNonQueryAsync( $"INSERT INTO {TableName} (id, data, version) VALUES (1, 'Record A - Updated', 2)"); // Both versions exist - var countBefore = await connection.ExecuteScalarAsync($"SELECT count() FROM {TableName} WHERE id = 1"); + var countBefore = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName} WHERE id = 1"); Console.WriteLine($" Rows with id=1: {countBefore}"); // Show the final state Console.WriteLine("\n Final table contents (FINAL modifier ensures deduplicated view):"); - using var reader = await connection.ExecuteReaderAsync( + using var reader = await client.ExecuteReaderAsync( $"SELECT id, data, version FROM {TableName} FINAL ORDER BY id"); Console.WriteLine(" ID Data Version"); @@ -151,9 +153,9 @@ await connection.ExecuteStatementAsync( } } - private static async Task Cleanup(ClickHouseConnection connection) + private static async Task Cleanup(ClickHouseClient client) { - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {TableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {TableName}"); Console.WriteLine($"Table '{TableName}' dropped"); } } diff --git a/examples/Advanced/Advanced_011_Compression.cs b/examples/Advanced/Advanced_011_Compression.cs index 29e27878..22ae7c95 100644 --- a/examples/Advanced/Advanced_011_Compression.cs +++ b/examples/Advanced/Advanced_011_Compression.cs @@ -41,9 +41,9 @@ namespace ClickHouse.Driver.Examples; /// /// The driver's default HttpClient already has this configured. /// -/// ## ClickHouseBulkCopy +/// ## InsertBinaryAsync /// -/// Note: ClickHouseBulkCopy always uses GZip compression for uploads regardless +/// Note: InsertBinaryAsync always uses GZip compression for uploads regardless /// of the UseCompression setting. This is because bulk inserts benefit significantly /// from compression due to the large data volumes involved. /// @@ -55,28 +55,24 @@ public static async Task Run() // Default: compression enabled Console.WriteLine("1. Default behavior (compression enabled):"); - using (var connection = new ClickHouseConnection("Host=localhost")) + using (var client = new ClickHouseClient("Host=localhost")) { - await connection.OpenAsync(); - // The driver will: // - Compress request bodies with GZip // - Request compressed responses via enable_http_compression=true - var result = await connection.ExecuteScalarAsync("SELECT 'Compressed request and response'"); + var result = await client.ExecuteScalarAsync("SELECT 'Compressed request and response'"); Console.WriteLine($" Result: {result}"); Console.WriteLine(" Request was GZip compressed, response was GZip compressed\n"); } // Compression disabled Console.WriteLine("2. Compression disabled:"); - using (var connection = new ClickHouseConnection("Host=localhost;Compression=false")) + using (var client = new ClickHouseClient("Host=localhost;Compression=false")) { - await connection.OpenAsync(); - // The driver will: // - Send uncompressed request bodies // - Set enable_http_compression=false (uncompressed responses) - var result = await connection.ExecuteScalarAsync("SELECT 'Uncompressed request and response'"); + var result = await client.ExecuteScalarAsync("SELECT 'Uncompressed request and response'"); Console.WriteLine($" Result: {result}"); Console.WriteLine(" Request was uncompressed, response was uncompressed\n"); } @@ -88,10 +84,9 @@ public static async Task Run() Host = "localhost", UseCompression = false // Disable compression }; - using (var connection = new ClickHouseConnection(settings)) + using (var client = new ClickHouseClient(settings)) { - await connection.OpenAsync(); - var result = await connection.ExecuteScalarAsync("SELECT 1"); + var result = await client.ExecuteScalarAsync("SELECT 1"); Console.WriteLine($" UseCompression = {settings.UseCompression}"); Console.WriteLine($" Result: {result}\n"); } diff --git a/examples/AspNet/AspNet_001_HealthChecks.cs b/examples/AspNet/AspNet_001_HealthChecks.cs index 5e110fd5..309e75a9 100644 --- a/examples/AspNet/AspNet_001_HealthChecks.cs +++ b/examples/AspNet/AspNet_001_HealthChecks.cs @@ -1,5 +1,6 @@ #if NET7_0_OR_GREATER using ClickHouse.Driver.ADO; +using ClickHouse.Driver.Utility; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; @@ -8,7 +9,13 @@ namespace ClickHouse.Driver.Examples; /// /// Demonstrates how to implement ASP.NET health checks for ClickHouse. -/// Shows the health check implementation and how to register it in an ASP.NET application. +/// Shows BOTH the ClickHouseClient and ClickHouseConnection approaches for health checks. +/// +/// API CHOICE GUIDE: +/// - ClickHouseClient: Recommended for direct database operations (queries, inserts) +/// - ClickHouseDataSource + ClickHouseConnection: Required for ORMs (Dapper, EF Core, linq2db) +/// +/// For health checks, either approach works. This example shows both patterns. /// public static class AspNetHealthChecks { @@ -16,21 +23,59 @@ public static async Task Run() { Console.WriteLine("ClickHouse ASP.NET Health Checks Example\n"); + var connectionString = "Host=localhost;Port=8123;Protocol=http;Username=default;Password=;Database=default"; + + // ======================================================================= + // OPTION 1: Using ClickHouseClient (recommended for direct operations) + // ======================================================================= + Console.WriteLine("1. Health check using ClickHouseClient:"); + await DemoClientHealthCheck(connectionString); + + // ======================================================================= + // OPTION 2: Using ClickHouseDataSource + ClickHouseConnection (for ORMs) + // ======================================================================= + Console.WriteLine("\n2. Health check using ClickHouseDataSource (for ADO.NET/ORM compatibility):"); + await DemoDataSourceHealthCheck(connectionString); + + Console.WriteLine("\nAll health check examples completed!"); + } + + private static async Task DemoClientHealthCheck(string connectionString) + { var services = new ServiceCollection(); services.AddLogging(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Warning)); - var connectionString = "Host=localhost;Port=8123;Protocol=http;Username=default;Password=;Database=default"; + // Register ClickHouseClient as a singleton + services.AddSingleton(_ => new ClickHouseClient(connectionString)); + + // Register health check using ClickHouseClient + services.AddHealthChecks() + .AddClickHouseClient(name: "clickhouse-client", tags: ["database", "clickhouse"]); + + var serviceProvider = services.BuildServiceProvider(); + var healthCheckService = serviceProvider.GetRequiredService(); + var report = await healthCheckService.CheckHealthAsync(); + + Console.WriteLine($" Overall status: {report.Status}"); + foreach (var entry in report.Entries) + { + Console.WriteLine($" - {entry.Key}: {entry.Value.Status}"); + } + } + + private static async Task DemoDataSourceHealthCheck(string connectionString) + { + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Warning)); - // Register ClickHouse data source + // Register ClickHouse data source (for ADO.NET/ORM compatibility) services.AddClickHouseDataSource(connectionString); - // Register health checks using the extension method + // Register health check using ClickHouseConnection services.AddHealthChecks() - .AddClickHouse(name: "clickhouse", tags: ["database", "clickhouse"]); + .AddClickHouseConnection(name: "clickhouse-connection", tags: ["database", "clickhouse"]); var serviceProvider = services.BuildServiceProvider(); - - // Resolve and run the health check var healthCheckService = serviceProvider.GetRequiredService(); var report = await healthCheckService.CheckHealthAsync(); @@ -38,34 +83,55 @@ public static async Task Run() foreach (var entry in report.Entries) { Console.WriteLine($" - {entry.Key}: {entry.Value.Status}"); - if (!string.IsNullOrEmpty(entry.Value.Description)) - { - Console.WriteLine($" Description: {entry.Value.Description}"); - } } + } +} - Console.WriteLine("\nAll health check examples completed!"); +/// +/// Health check implementation using ClickHouseClient. +/// Recommended for applications that use ClickHouseClient directly. +/// +public class ClickHouseClientHealthCheck : IHealthCheck +{ + private readonly ClickHouseClient _client; + + public ClickHouseClientHealthCheck(ClickHouseClient client) + { + ArgumentNullException.ThrowIfNull(client); + _client = client; + } + + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + try + { + await _client.ExecuteScalarAsync("SELECT 1", cancellationToken: cancellationToken).ConfigureAwait(false); + return HealthCheckResult.Healthy(); + } + catch (Exception ex) + { + return new HealthCheckResult( + context.Registration.FailureStatus, + description: ex.Message, + exception: ex); + } } } /// -/// A health check for ClickHouse databases. +/// Health check implementation using ClickHouseConnection. +/// Use this when your application uses ClickHouseDataSource for ORM compatibility. /// -public class ClickHouseHealthCheck : IHealthCheck +public class ClickHouseConnectionHealthCheck : IHealthCheck { private readonly ClickHouseConnection _connection; - /// - /// Initializes a new instance of the class. - /// - /// The ClickHouse connection to use for health checks. - public ClickHouseHealthCheck(ClickHouseConnection connection) + public ClickHouseConnectionHealthCheck(ClickHouseConnection connection) { ArgumentNullException.ThrowIfNull(connection); _connection = connection; } - /// public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { try @@ -94,15 +160,29 @@ public async Task CheckHealthAsync(HealthCheckContext context public static class ClickHouseHealthCheckBuilderExtensions { /// - /// Adds a health check for ClickHouse using a ClickHouseConnection from DI. + /// Adds a health check for ClickHouse using ClickHouseClient from DI. + /// Recommended for applications that use ClickHouseClient directly. + /// + public static IHealthChecksBuilder AddClickHouseClient( + this IHealthChecksBuilder builder, + string name = "clickhouse", + HealthStatus? failureStatus = null, + IEnumerable? tags = null, + TimeSpan? timeout = null) + { + return builder.Add(new HealthCheckRegistration( + name, + sp => new ClickHouseClientHealthCheck(sp.GetRequiredService()), + failureStatus, + tags, + timeout)); + } + + /// + /// Adds a health check for ClickHouse using ClickHouseConnection from DI. + /// Use this when your application uses ClickHouseDataSource for ORM compatibility. /// - /// The health check builder. - /// The name of the health check. Defaults to "clickhouse". - /// The status to report when the health check fails. Defaults to Unhealthy. - /// Optional tags for the health check. - /// Optional timeout for the health check. - /// The health check builder for chaining. - public static IHealthChecksBuilder AddClickHouse( + public static IHealthChecksBuilder AddClickHouseConnection( this IHealthChecksBuilder builder, string name = "clickhouse", HealthStatus? failureStatus = null, @@ -111,7 +191,7 @@ public static IHealthChecksBuilder AddClickHouse( { return builder.Add(new HealthCheckRegistration( name, - sp => new ClickHouseHealthCheck(sp.GetRequiredService()), + sp => new ClickHouseConnectionHealthCheck(sp.GetRequiredService()), failureStatus, tags, timeout)); diff --git a/examples/Core/Auth_001_JwtAuthentication.cs b/examples/Core/Auth_001_JwtAuthentication.cs index c2abd23e..43e9794f 100644 --- a/examples/Core/Auth_001_JwtAuthentication.cs +++ b/examples/Core/Auth_001_JwtAuthentication.cs @@ -20,8 +20,8 @@ public static async Task Run() return; } - // 1. Basic JWT authentication at connection level - Console.WriteLine("1. JWT authentication at connection level:"); + // 1. Basic JWT authentication + Console.WriteLine("1. JWT authentication:"); var settings = new ClickHouseClientSettings { Host = host, @@ -30,10 +30,9 @@ public static async Task Run() BearerToken = jwt, }; - using (var connection = new ClickHouseConnection(settings)) + using (var client = new ClickHouseClient(settings)) { - await connection.OpenAsync(); - var version = await connection.ExecuteScalarAsync("SELECT version()"); + var version = await client.ExecuteScalarAsync("SELECT version()"); Console.WriteLine($" Connected to ClickHouse version: {version}"); } @@ -47,33 +46,24 @@ public static async Task Run() Protocol = "https", Username = "ignored_user", // These are ignored when BearerToken is set Password = "ignored_password", - BearerToken = jwt, // Bearer token is used instead + BearerToken = jwt, // Bearer token is used instead }; - using (var connection = new ClickHouseConnection(settingsWithBoth)) + using (var client = new ClickHouseClient(settingsWithBoth)) { - await connection.OpenAsync(); + await client.ExecuteNonQueryAsync("SELECT 1"); Console.WriteLine(" Connected successfully using Bearer token (Username/Password ignored)"); } - // 3. Command-level token override - // Useful when you need different tokens for different operations - Console.WriteLine("\n3. Command-level token override:"); - using (var connection = new ClickHouseConnection(settings)) + // 3. Per-query token override via QueryOptions + Console.WriteLine("\n3. Per-query token override:"); + using (var client = new ClickHouseClient(settings)) { - await connection.OpenAsync(); - - using var command = connection.CreateCommand(); - command.BearerToken = jwt; // Override connection-level token - command.CommandText = "SELECT currentUser()"; - - var user = await command.ExecuteScalarAsync(); + var options = new QueryOptions { BearerToken = jwt }; + var user = await client.ExecuteScalarAsync("SELECT currentUser()", options: options); Console.WriteLine($" Current user: {user}"); } - // 4. Token refresh pattern (conceptual) - // Not implemented yet - Console.WriteLine("\nJWT authentication examples completed successfully!"); } } diff --git a/examples/Core/Core_001_BasicUsage.cs b/examples/Core/Core_001_BasicUsage.cs index 77d590cc..a6f9f68e 100644 --- a/examples/Core/Core_001_BasicUsage.cs +++ b/examples/Core/Core_001_BasicUsage.cs @@ -6,25 +6,88 @@ namespace ClickHouse.Driver.Examples; /// /// A simple example demonstrating the basic usage of the ClickHouse C# driver. /// This example shows how to: -/// - Create a connection to ClickHouse /// - Create a table /// - Insert data /// - Query data +/// +/// +/// Two APIs are available: +/// +/// - Recommended for new code. Thread-safe, singleton-friendly. +/// - For ADO.NET compatibility (Dapper, EF Core, etc.). +/// +/// /// public static class BasicUsage { public static async Task Run() { - // Create a connection to ClickHouse using ClickHouseClientSettings - // By default, connects to localhost:8123 with user 'default' and no password - var settings = new ClickHouseClientSettings("Host=localhost;Port=8123;Protocol=http;Username=default;Password=;Database=default"); + Console.WriteLine("=== Using ClickHouseClient (recommended) ===\n"); + await UsingClickHouseClient(); + + Console.WriteLine("\n=== Using ClickHouseConnection (ADO.NET) ===\n"); + await UsingClickHouseConnection(); + } + + private static async Task UsingClickHouseClient() + { + // ClickHouseClient is thread-safe and designed for singleton usage + var settings = new ClickHouseClientSettings("Host=localhost"); + using var client = new ClickHouseClient(settings); + + var version = await client.ExecuteScalarAsync("SELECT version()"); + Console.WriteLine($"Connected to ClickHouse version: {version}"); + + // Create a table + var tableName = "example_basic_client"; + await client.ExecuteNonQueryAsync($@" + CREATE TABLE IF NOT EXISTS {tableName} + ( + id UInt64, + name String, + timestamp DateTime + ) + ENGINE = MergeTree() + ORDER BY (id) + "); + Console.WriteLine($"Table '{tableName}' created"); + + // Insert data using bulk insert + var rows = new List + { + new object[] { 1UL, "Alice", DateTime.UtcNow }, + new object[] { 2UL, "Bob", DateTime.UtcNow }, + }; + await client.InsertBinaryAsync(tableName, new[] { "id", "name", "timestamp" }, rows); + Console.WriteLine("Data inserted"); + + // Query data + using (var reader = await client.ExecuteReaderAsync($"SELECT * FROM {tableName} ORDER BY id")) + { + Console.WriteLine("\nID\tName\tTimestamp"); + Console.WriteLine("--\t----\t---------"); + while (reader.Read()) + { + Console.WriteLine($"{reader.GetFieldValue(0)}\t{reader.GetString(1)}\t{reader.GetDateTime(2):yyyy-MM-dd HH:mm:ss}"); + } + } + + // Clean up + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); + Console.WriteLine($"Table '{tableName}' dropped"); + } + + private static async Task UsingClickHouseConnection() + { + // ClickHouseConnection provides ADO.NET compatibility for Dapper, EF Core, etc. + var settings = new ClickHouseClientSettings("Host=localhost"); using var connection = new ClickHouseConnection(settings); await connection.OpenAsync(); Console.WriteLine($"Connection state: {connection.State}"); // Create a table - var tableName = "example_basic_usage"; + var tableName = "example_basic_ado"; await connection.ExecuteStatementAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( @@ -35,10 +98,9 @@ timestamp DateTime ENGINE = MergeTree() ORDER BY (id) "); - Console.WriteLine($"Table '{tableName}' created"); - // Insert data using a parameterized query + // Insert data using parameterized query using (var command = connection.CreateCommand()) { command.CommandText = $"INSERT INTO {tableName} (id, name, timestamp) VALUES ({{id:UInt64}}, {{name:String}}, {{timestamp:DateTime}})"; @@ -53,28 +115,21 @@ ORDER BY (id) command.AddParameter("timestamp", DateTime.UtcNow); await command.ExecuteNonQueryAsync(); } - Console.WriteLine("Data inserted"); // Query data using (var reader = await connection.ExecuteReaderAsync($"SELECT * FROM {tableName} ORDER BY id")) { - Console.WriteLine("\nQuerying data from table:"); - Console.WriteLine("ID\tName\tTimestamp"); + Console.WriteLine("\nID\tName\tTimestamp"); Console.WriteLine("--\t----\t---------"); - while (reader.Read()) { - var id = reader.GetFieldValue(0); - var name = reader.GetString(1); - var timestamp = reader.GetDateTime(2); - - Console.WriteLine($"{id}\t{name}\t{timestamp:yyyy-MM-dd HH:mm:ss}"); + Console.WriteLine($"{reader.GetFieldValue(0)}\t{reader.GetString(1)}\t{reader.GetDateTime(2):yyyy-MM-dd HH:mm:ss}"); } } // Clean up await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); - Console.WriteLine($"\nTable '{tableName}' dropped"); + Console.WriteLine($"Table '{tableName}' dropped"); } } diff --git a/examples/Core/Core_002_ConnectionStringConfiguration.cs b/examples/Core/Core_002_ConnectionStringConfiguration.cs index 8fff484c..df83a220 100644 --- a/examples/Core/Core_002_ConnectionStringConfiguration.cs +++ b/examples/Core/Core_002_ConnectionStringConfiguration.cs @@ -4,7 +4,7 @@ namespace ClickHouse.Driver.Examples; /// -/// Examples of different ways to configure a connection to ClickHouse. +/// Examples of different ways to configure a ClickHouseClient. /// Shows various connection string formats and options. /// public static class ConnectionStringConfiguration @@ -13,11 +13,10 @@ public static async Task Run() { // 1: Connection string with named parameters Console.WriteLine("1. Connection string with named parameters:"); - using (var connection = new ClickHouseConnection( + using (var client = new ClickHouseClient( "Host=localhost;Port=8123;Protocol=http;Username=default;Password=;Database=default")) { - await connection.OpenAsync(); - var version = await connection.ExecuteScalarAsync("SELECT version()"); + var version = await client.ExecuteScalarAsync("SELECT version()"); Console.WriteLine($" Connected to ClickHouse version: {version}"); } @@ -32,10 +31,9 @@ public static async Task Run() Database = "default", Protocol = "http", }; - using (var connection = new ClickHouseConnection(settings)) + using (var client = new ClickHouseClient(settings)) { - await connection.OpenAsync(); - var version = await connection.ExecuteScalarAsync("SELECT version()"); + var version = await client.ExecuteScalarAsync("SELECT version()"); Console.WriteLine($" Connected to ClickHouse version: {version}"); } @@ -52,19 +50,15 @@ public static async Task Run() }; Console.WriteLine($" Settings: Host={secureSettings.Host}, Port={secureSettings.Port}, Protocol={secureSettings.Protocol}"); - // 4: Connection with custom settings Console.WriteLine("\n4. Connection with custom ClickHouse settings:"); var settingsWithCustom = new ClickHouseClientSettings("Host=localhost"); settingsWithCustom.CustomSettings.Add("max_execution_time", 10); settingsWithCustom.CustomSettings.Add("max_memory_usage", 10000000000); - using (var connection = new ClickHouseConnection(settingsWithCustom)) + using (var client = new ClickHouseClient(settingsWithCustom)) { - await connection.OpenAsync(); - using var command = connection.CreateCommand(); - command.CommandText = "SELECT 1"; - await command.ExecuteScalarAsync(); + await client.ExecuteNonQueryAsync("SELECT 1"); Console.WriteLine(" Executed query with custom ClickHouse settings"); } diff --git a/examples/Core/Core_003_DependencyInjection.cs b/examples/Core/Core_003_DependencyInjection.cs index 5133a66e..04e37a65 100644 --- a/examples/Core/Core_003_DependencyInjection.cs +++ b/examples/Core/Core_003_DependencyInjection.cs @@ -10,43 +10,31 @@ namespace ClickHouse.Driver.Examples; /// /// Demonstrates how to use ClickHouse with dependency injection in .NET 7+ applications. -/// Shows proper integration with IServiceCollection, ClickHouseDataSource, and IHttpClientFactory. -/// Also, loading options using the ConfigurationBuilder. /// /// -/// IMPORTANT: Connection Pooling and Socket Exhaustion +/// Recommended Approach: ClickHouseClient as Singleton /// /// -/// If you create multiple instances without passing a shared -/// , each connection will create its own with its -/// own connection pool. In high-throughput scenarios, this can lead to socket exhaustion and poor performance -/// as each pool maintains separate TCP connections to the server. +/// is the recommended API for new code. It is thread-safe, manages +/// HTTP connection pooling internally, and is designed for singleton usage. Register it as a singleton +/// in your DI container and inject it wherever you need to interact with ClickHouse. +/// +/// +/// +/// ADO.NET Compatibility: ClickHouseDataSource /// /// -/// To avoid this: -/// -/// -/// -/// Use a singleton HttpClient: Pass a shared instance to all -/// objects via . -/// This maintains a single connection pool and reuses TCP connections efficiently. If using an IHttpClientFactory, -/// make sure you are not constantly recreating HttpClients. -/// -/// -/// -/// -/// Use a singleton ClickHouseConnection: A single long-lived connection instance -/// achieves the same result since the underlying is shared. -/// -/// -/// -/// -/// Using : Call AddClickHouseDataSource() which registers -/// as a singleton by default. All -/// instances resolved from DI will share the same and connection pool. -/// -/// -/// +/// For ADO.NET compatibility (e.g., with ORMs like Dapper or EF Core), use . +/// The DataSource owns a internally and creates connections that share it. +/// +/// +/// +/// Connection Pooling +/// +/// +/// Both and manage HTTP connection pooling. +/// The "connection" in ClickHouse is a logical concept - the actual HTTP connections are pooled and reused. +/// You don't need to worry about connection exhaustion as long as you use a singleton client or data source. /// /// public static class DependencyInjection @@ -55,20 +43,22 @@ public static async Task Run() { Console.WriteLine("ClickHouse Dependency Injection Examples (.NET 7+)\n"); - // Example 1: Basic registration with connection string - Console.WriteLine("1. Basic registration with connection string:"); - await Example1_BasicRegistration(); + // Example 1: ClickHouseClient as singleton (RECOMMENDED) + Console.WriteLine("1. ClickHouseClient as singleton (recommended):"); + await Example1_ClientAsSingleton(); - Console.WriteLine("\n2. Registration with ClickHouseClientSettings:"); - await Example2_SettingsRegistration(); + // Example 2: ClickHouseDataSource for ADO.NET compatibility + Console.WriteLine("\n2. ClickHouseDataSource for ADO.NET compatibility:"); + await Example2_DataSourceRegistration(); + // Example 3: With IHttpClientFactory Console.WriteLine("\n3. Registration with IHttpClientFactory:"); await Example3_HttpClientFactoryRegistration(); Console.WriteLine("\nAll dependency injection examples completed!"); } - private static async Task Example1_BasicRegistration() + private static async Task Example1_ClientAsSingleton() { // Create a service collection (this would normally be done by the framework) var services = new ServiceCollection(); @@ -76,40 +66,41 @@ private static async Task Example1_BasicRegistration() // Add logging (optional but recommended) services.AddLogging(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Warning)); - // Register ClickHouse with a connection string. - // - ClickHouseDataSource is registered as a SINGLETON by default - // - All ClickHouseConnection instances share the same underlying HttpClient - services.AddClickHouseDataSource("Host=localhost;Port=8123;Protocol=http;Username=default;Password=;Database=default"); + // Register ClickHouseClient as a singleton. + // This is the recommended approach for new code. + services.AddSingleton(sp => + { + var loggerFactory = sp.GetService(); + return new ClickHouseClient(new ClickHouseClientSettings + { + Host = "localhost", + LoggerFactory = loggerFactory, + }); + }); // Build the service provider var serviceProvider = services.BuildServiceProvider(); - // Resolve and use a ClickHouseConnection. - // Note: ClickHouseConnection is registered as Transient by default, but all - // connections share the same HttpClient from the singleton ClickHouseDataSource. - using (var scope = serviceProvider.CreateScope()) - { - var connection = scope.ServiceProvider.GetRequiredService(); - await connection.OpenAsync(); + // Resolve and use the ClickHouseClient directly + var client = serviceProvider.GetRequiredService(); - var version = await connection.ExecuteScalarAsync("SELECT version()"); - Console.WriteLine($" Connected to ClickHouse version: {version}"); - } + var version = (string)await client.ExecuteScalarAsync("SELECT version()"); + Console.WriteLine($" Connected to ClickHouse version: {version}"); + + // The client can be injected anywhere in your application + // It's thread-safe and designed for concurrent use } - private static async Task Example2_SettingsRegistration() + private static async Task Example2_DataSourceRegistration() { // Build configuration from appsettings.example.json - // Use AppContext.BaseDirectory to get the directory where the assembly is located var configuration = new ConfigurationBuilder() .SetBasePath(AppContext.BaseDirectory) .AddJsonFile("appsettings.example.json", optional: false, reloadOnChange: false) .Build(); var services = new ServiceCollection(); - - // Add logging, this will pull in the logging settings from our configuration file - services.AddLogging(builder => builder.AddConsole()); + services.AddLogging(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Warning)); // Load ClickHouse settings from configuration var settings = configuration.GetSection("ClickHouse").Get(); @@ -121,15 +112,16 @@ private static async Task Example2_SettingsRegistration() Console.WriteLine($" Loaded settings from configuration: Host={settings.Host}, Database={settings.Database}"); + // Register ClickHouseDataSource for ADO.NET compatibility (e.g., with ORMs) + // The DataSource owns a ClickHouseClient internally services.AddClickHouseDataSource(settings); var serviceProvider = services.BuildServiceProvider(); using (var scope = serviceProvider.CreateScope()) { - // You can resolve either ClickHouseConnection, ClickHouseDataSource, or the interfaces - var dataSource = scope.ServiceProvider.GetRequiredService(); - using var connection = dataSource.CreateConnection(); + // Resolve ClickHouseConnection - all connections share the same underlying client + var connection = scope.ServiceProvider.GetRequiredService(); await connection.OpenAsync(); var version = await connection.ExecuteScalarAsync("SELECT version()"); @@ -143,20 +135,33 @@ private static async Task Example3_HttpClientFactoryRegistration() services.AddLogging(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Warning)); // Register a named HttpClient with custom configuration. - // This is important for production scenarios where you want to control HttpClient - // or HttpHandler settings. + // This is important for production scenarios where you want to control + // connection pooling, timeouts, and other HTTP settings. services.AddHttpClient("ClickHouseClient", client => { client.Timeout = TimeSpan.FromMinutes(5); }).ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, // Required for compression support + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, MaxConnectionsPerServer = 100, - PooledConnectionIdleTimeout = TimeSpan.FromSeconds(5), // Make sure to set this to a value lower than the server idle timeout (10s for Cloud) + PooledConnectionIdleTimeout = TimeSpan.FromSeconds(5), // Keep lower than server idle timeout (10s for Cloud) }); - // Register ClickHouse with a settings factory that resolves IHttpClientFactory from DI. - // The underlying SocketsHandler will be shared. + // Option A: Register ClickHouseClient directly (recommended for new code) + services.AddSingleton(sp => + { + var factory = sp.GetRequiredService(); + var loggerFactory = sp.GetService(); + return new ClickHouseClient(new ClickHouseClientSettings + { + Host = "localhost", + HttpClientFactory = factory, + HttpClientName = "ClickHouseClient", + LoggerFactory = loggerFactory, + }); + }); + + // Option B: Also register DataSource for ADO.NET compatibility services.AddClickHouseDataSource(sp => { var factory = sp.GetRequiredService(); @@ -170,13 +175,18 @@ private static async Task Example3_HttpClientFactoryRegistration() var serviceProvider = services.BuildServiceProvider(); + // Use ClickHouseClient directly + var client = serviceProvider.GetRequiredService(); + var version = (string)await client.ExecuteScalarAsync("SELECT version()"); + Console.WriteLine($" Using ClickHouseClient: {version}"); + + // Or use ClickHouseConnection for ADO.NET compatibility using (var scope = serviceProvider.CreateScope()) { var connection = scope.ServiceProvider.GetRequiredService(); await connection.OpenAsync(); - - var version = await connection.ExecuteScalarAsync("SELECT version()"); - Console.WriteLine($" Connected to ClickHouse version: {version}"); + version = (string)await connection.ExecuteScalarAsync("SELECT version()"); + Console.WriteLine($" Using ClickHouseConnection: {version}"); } } } diff --git a/examples/Core/Core_004_HttpClientConfiguration.cs b/examples/Core/Core_004_HttpClientConfiguration.cs index 4a5bbcc0..c11ae883 100644 --- a/examples/Core/Core_004_HttpClientConfiguration.cs +++ b/examples/Core/Core_004_HttpClientConfiguration.cs @@ -7,7 +7,7 @@ namespace ClickHouse.Driver.Examples; /// -/// Demonstrates how to provide your own HttpClient or IHttpClientFactory to ClickHouse connections. +/// Demonstrates how to provide your own HttpClient or IHttpClientFactory to ClickHouseClient. /// This is important for: /// - Custom SSL/TLS configuration (certificates, validation) /// - Proxy configuration @@ -15,21 +15,22 @@ namespace ClickHouse.Driver.Examples; /// - Connection pooling control /// - Integration with dependency injection /// -/// IMPORTANT LIMITATIONS: -/// - When providing your own HttpClient, YOU are responsible for: -/// * Enabling automatic decompression (required if compression is not disabled) -/// * Setting appropriate timeouts -/// * Certificate validation -/// * Disposal (if not using DI). -/// It is recommended that you reuse and do not dispose HttpSocketsHandler/HttpClientHandler, -/// as they maintain internal connection pools. +/// +/// IMPORTANT: When providing your own HttpClient, YOU are responsible for: +/// +/// Enabling automatic decompression (required if compression is enabled) +/// Setting appropriate timeouts +/// Certificate validation +/// Disposal (if not using DI) +/// +/// /// -/// IMPORTANT: Connection Idle Timeout -/// - If you provide your own HttpClient/HttpClientFactory, ensure that the client-side -/// idle connection timeout is SMALLER than the server's keep_alive_timeout setting. -/// - ClickHouse Cloud default keep_alive_timeout: 10 seconds -/// - If client timeout >= server timeout, you may encounter half-open connections that -/// fail with "connection was closed" errors +/// +/// Connection Idle Timeout: +/// If you provide your own HttpClient/HttpClientFactory, ensure that the client-side +/// idle connection timeout is SMALLER than the server's keep_alive_timeout setting. +/// ClickHouse Cloud default keep_alive_timeout: 10 seconds. +/// /// public static class HttpClientConfiguration { @@ -37,8 +38,8 @@ public static async Task Run() { Console.WriteLine("HttpClient Configuration Examples\n"); - // Example 1: Basic custom HttpClient - Console.WriteLine("1. Providing a custom HttpClient:"); + // Example 1: ClickHouseClient with custom HttpClient + Console.WriteLine("1. ClickHouseClient with custom HttpClient:"); await Example1_CustomHttpClient(); // Example 2: SSL/TLS configuration @@ -79,18 +80,23 @@ private static async Task Example1_CustomHttpClient() Timeout = TimeSpan.FromMinutes(5), }; - // Pass the HttpClient to the connection + // Pass the HttpClient via settings var settings = new ClickHouseClientSettings { Host = "localhost", HttpClient = httpClient, }; + // Option A: Use ClickHouseClient directly (recommended for new code) + using var client = new ClickHouseClient(settings); + var version = await client.ExecuteScalarAsync("SELECT version()"); + Console.WriteLine($" Using ClickHouseClient: {version}"); + + // Option B: Use ClickHouseConnection for ADO.NET compatibility using var connection = new ClickHouseConnection(settings); await connection.OpenAsync(); - - var version = await connection.ExecuteScalarAsync("SELECT version()"); - Console.WriteLine($" Connected to ClickHouse version: {version}"); + version = await connection.ExecuteScalarAsync("SELECT version()"); + Console.WriteLine($" Using ClickHouseConnection: {version}"); } private static async Task Example2_SslConfiguration() @@ -207,11 +213,16 @@ private static async Task Example4_HttpClientFactory() HttpClientName = "ClickHouseClient", // Optional: factory can use this name }; + // Option A: Use ClickHouseClient directly (recommended for new code) + using var client = new ClickHouseClient(settings); + var version = await client.ExecuteScalarAsync("SELECT version()"); + Console.WriteLine($" Using ClickHouseClient: {version}"); + + // Option B: Use ClickHouseConnection for ADO.NET compatibility using var connection = new ClickHouseConnection(settings); await connection.OpenAsync(); - - var version = await connection.ExecuteScalarAsync("SELECT version()"); - Console.WriteLine($" Connected using IHttpClientFactory: {version}"); + version = await connection.ExecuteScalarAsync("SELECT version()"); + Console.WriteLine($" Using ClickHouseConnection: {version}"); // Factory handles disposal } diff --git a/examples/DataTypes/DataTypes_001_SimpleTypes.cs b/examples/DataTypes/DataTypes_001_SimpleTypes.cs index ecd9660b..fbbb1e60 100644 --- a/examples/DataTypes/DataTypes_001_SimpleTypes.cs +++ b/examples/DataTypes/DataTypes_001_SimpleTypes.cs @@ -1,6 +1,4 @@ -using System.Net; using System.Numerics; -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Numerics; using ClickHouse.Driver.Utility; @@ -14,59 +12,58 @@ public static class SimpleTypes { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); Console.WriteLine("Simple Data Types Examples\n"); - await IntegerTypes(connection); - await FloatingPointTypes(connection); - await DecimalTypes(connection); - await BooleanType(connection); + await IntegerTypes(client); + await FloatingPointTypes(client); + await DecimalTypes(client); + await BooleanType(client); } /// /// Demonstrates all integer types from 8-bit to 256-bit, signed and unsigned. /// - private static async Task IntegerTypes(ClickHouseConnection connection) + private static async Task IntegerTypes(ClickHouseClient client) { Console.WriteLine("1. Integer Types:"); Console.WriteLine(" ClickHouse Type .NET Type Example Value"); Console.WriteLine(" -------------- --------- -------------"); // Signed integers - var int8 = await connection.ExecuteScalarAsync("SELECT toInt8(-128)"); + var int8 = await client.ExecuteScalarAsync("SELECT toInt8(-128)"); Console.WriteLine($" Int8 sbyte {int8}"); - var int16 = await connection.ExecuteScalarAsync("SELECT toInt16(-32768)"); + var int16 = await client.ExecuteScalarAsync("SELECT toInt16(-32768)"); Console.WriteLine($" Int16 short {int16}"); - var int32 = await connection.ExecuteScalarAsync("SELECT toInt32(-2147483648)"); + var int32 = await client.ExecuteScalarAsync("SELECT toInt32(-2147483648)"); Console.WriteLine($" Int32 int {int32}"); - var int64 = await connection.ExecuteScalarAsync("SELECT toInt64(-9223372036854775808)"); + var int64 = await client.ExecuteScalarAsync("SELECT toInt64(-9223372036854775808)"); Console.WriteLine($" Int64 long {int64}"); - var int128 = await connection.ExecuteScalarAsync("SELECT toInt128(-170141183460469231731687303715884105728)"); + var int128 = await client.ExecuteScalarAsync("SELECT toInt128(-170141183460469231731687303715884105728)"); Console.WriteLine($" Int128 BigInteger {int128}"); - var int256 = await connection.ExecuteScalarAsync("SELECT toInt256(-57896044618658097711785492504343953926634992332820282019728792003956564819968)"); + var int256 = await client.ExecuteScalarAsync("SELECT toInt256(-57896044618658097711785492504343953926634992332820282019728792003956564819968)"); Console.WriteLine($" Int256 BigInteger {((BigInteger)int256!).ToString().Substring(0, 20)}..."); // Unsigned integers - var uint8 = await connection.ExecuteScalarAsync("SELECT toUInt8(255)"); + var uint8 = await client.ExecuteScalarAsync("SELECT toUInt8(255)"); Console.WriteLine($" UInt8 byte {uint8}"); - var uint16 = await connection.ExecuteScalarAsync("SELECT toUInt16(65535)"); + var uint16 = await client.ExecuteScalarAsync("SELECT toUInt16(65535)"); Console.WriteLine($" UInt16 ushort {uint16}"); - var uint32 = await connection.ExecuteScalarAsync("SELECT toUInt32(4294967295)"); + var uint32 = await client.ExecuteScalarAsync("SELECT toUInt32(4294967295)"); Console.WriteLine($" UInt32 uint {uint32}"); - var uint64 = await connection.ExecuteScalarAsync("SELECT toUInt64(18446744073709551615)"); + var uint64 = await client.ExecuteScalarAsync("SELECT toUInt64(18446744073709551615)"); Console.WriteLine($" UInt64 ulong {uint64}"); - var uint128 = await connection.ExecuteScalarAsync("SELECT toUInt128(340282366920938463463374607431768211455)"); + var uint128 = await client.ExecuteScalarAsync("SELECT toUInt128(340282366920938463463374607431768211455)"); Console.WriteLine($" UInt128 BigInteger {((BigInteger)uint128!).ToString().Substring(0, 20)}..."); Console.WriteLine(); @@ -75,23 +72,23 @@ private static async Task IntegerTypes(ClickHouseConnection connection) /// /// Demonstrates floating point types: Float32, Float64. /// - private static async Task FloatingPointTypes(ClickHouseConnection connection) + private static async Task FloatingPointTypes(ClickHouseClient client) { Console.WriteLine("2. Floating Point Types:"); Console.WriteLine(" ClickHouse Type .NET Type Example Value"); Console.WriteLine(" -------------- --------- -------------"); - var float32 = await connection.ExecuteScalarAsync("SELECT toFloat32(3.14159)"); + var float32 = await client.ExecuteScalarAsync("SELECT toFloat32(3.14159)"); Console.WriteLine($" Float32 float {float32}"); - var float64 = await connection.ExecuteScalarAsync("SELECT toFloat64(3.141592653589793)"); + var float64 = await client.ExecuteScalarAsync("SELECT toFloat64(3.141592653589793)"); Console.WriteLine($" Float64 double {float64}"); // Special values - var inf = await connection.ExecuteScalarAsync("SELECT toFloat64(1) / toFloat64(0)"); + var inf = await client.ExecuteScalarAsync("SELECT toFloat64(1) / toFloat64(0)"); Console.WriteLine($" Float64 double {inf} (toFloat64(1) / toFloat64(0))"); - var nan = await connection.ExecuteScalarAsync("SELECT toFloat64(0) / toFloat64(0)"); + var nan = await client.ExecuteScalarAsync("SELECT toFloat64(0) / toFloat64(0)"); Console.WriteLine($" Float64 double {nan} (toFloat64(0) / toFloat64(0))"); Console.WriteLine(); @@ -101,22 +98,22 @@ private static async Task FloatingPointTypes(ClickHouseConnection connection) /// Demonstrates decimal types with various precisions. /// ClickHouseDecimal preserves full precision for large decimals. /// - private static async Task DecimalTypes(ClickHouseConnection connection) + private static async Task DecimalTypes(ClickHouseClient client) { Console.WriteLine("3. Decimal Types:"); Console.WriteLine(" ClickHouse Type .NET Type Example Value"); Console.WriteLine(" -------------- --------- -------------"); // Decimal32 - up to 9 digits of precision - var decimal32 = await connection.ExecuteScalarAsync("SELECT toDecimal32(123.456, 3)"); + var decimal32 = await client.ExecuteScalarAsync("SELECT toDecimal32(123.456, 3)"); Console.WriteLine($" Decimal32(3) ClickHouseDecimal {decimal32}"); // Decimal64 - up to 18 digits of precision - var decimal64 = await connection.ExecuteScalarAsync("SELECT toDecimal64(123456789.123456789, 9)"); + var decimal64 = await client.ExecuteScalarAsync("SELECT toDecimal64(123456789.123456789, 9)"); Console.WriteLine($" Decimal64(9) ClickHouseDecimal {decimal64}"); // Decimal128 - up to 38 digits of precision - var decimal128 = await connection.ExecuteScalarAsync("SELECT toDecimal128(1234567890123456789.12345678901234567890, 20)"); + var decimal128 = await client.ExecuteScalarAsync("SELECT toDecimal128(1234567890123456789.12345678901234567890, 20)"); Console.WriteLine($" Decimal128(20) ClickHouseDecimal {decimal128}"); // Using ClickHouseDecimal for precise operations @@ -131,20 +128,20 @@ private static async Task DecimalTypes(ClickHouseConnection connection) /// /// Demonstrates the Bool type. /// - private static async Task BooleanType(ClickHouseConnection connection) + private static async Task BooleanType(ClickHouseClient client) { Console.WriteLine("4. Boolean Type:"); Console.WriteLine(" ClickHouse Type .NET Type Example Value"); Console.WriteLine(" -------------- --------- -------------"); - var boolTrue = await connection.ExecuteScalarAsync("SELECT true"); + var boolTrue = await client.ExecuteScalarAsync("SELECT true"); Console.WriteLine($" Bool bool {boolTrue}"); - var boolFalse = await connection.ExecuteScalarAsync("SELECT false"); + var boolFalse = await client.ExecuteScalarAsync("SELECT false"); Console.WriteLine($" Bool bool {boolFalse}"); // Bool from expression - var boolExpr = await connection.ExecuteScalarAsync("SELECT (1 > 0)::Bool"); + var boolExpr = await client.ExecuteScalarAsync("SELECT (1 > 0)::Bool"); Console.WriteLine($" Bool (expr) bool {boolExpr} (1 > 0)"); Console.WriteLine(); diff --git a/examples/DataTypes/DataTypes_002_DateTimeHandling.cs b/examples/DataTypes/DataTypes_002_DateTimeHandling.cs index 84192bb8..d04e7a09 100644 --- a/examples/DataTypes/DataTypes_002_DateTimeHandling.cs +++ b/examples/DataTypes/DataTypes_002_DateTimeHandling.cs @@ -1,6 +1,5 @@ using ClickHouse.Driver.ADO; using ClickHouse.Driver.ADO.Readers; -using ClickHouse.Driver.Copy; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -49,9 +48,9 @@ public static async Task Run() Console.WriteLine("\n6. Writing DateTime Via Parameters:"); await Example6_WriteViaParameters(connection); - // Example 7: Writing DateTime via bulk copy - Console.WriteLine("\n7. Writing DateTime Via Bulk Copy:"); - await Example7_WriteViaBulkCopy(connection); + // Example 7: Writing DateTime via InsertBinaryAsync + Console.WriteLine("\n7. Writing DateTime Via InsertBinaryAsync:"); + await Example7_WriteViaInsertBinaryAsync(connection); // Example 8: Working with DateTimeOffset Console.WriteLine("\n8. Working With DateTimeOffset:"); @@ -285,11 +284,11 @@ dt_no_tz DateTime } /// - /// Writing DateTime values via bulk copy. - /// Bulk copy knows the target column's timezone, so it can correctly + /// Writing DateTime values via InsertBinaryAsync. + /// InsertBinaryAsync knows the target column's timezone, so it can correctly /// interpret DateTime.Kind=Unspecified as wall-clock time in that timezone. /// - private static async Task Example7_WriteViaBulkCopy(ClickHouseConnection connection) + private static async Task Example7_WriteViaInsertBinaryAsync(ClickHouseConnection connection) { var tableName = "example_datetime_bulk"; @@ -303,29 +302,26 @@ dt_amsterdam DateTime('Europe/Amsterdam') ENGINE = Memory "); - using (var bulkCopy = new ClickHouseBulkCopy(connection) - { - DestinationTableName = tableName, - }) + using var client = new ClickHouseClient("Host=localhost"); + + // Unspecified DateTime values are treated as wall-clock time in the column's timezone + var columns = new[] { "id", "dt_utc", "dt_amsterdam" }; + var data = new List { - // Unspecified DateTime values are treated as wall-clock time in the column's timezone - var data = new List - { - // Row 1: Same wall-clock time (12:00) in both columns - new object[] { 1u, - new DateTime(2024, 6, 15, 12, 0, 0, DateTimeKind.Unspecified), // 12:00 UTC - new DateTime(2024, 6, 15, 12, 0, 0, DateTimeKind.Unspecified) // 12:00 Amsterdam - }, - // Row 2: UTC DateTime - instant is preserved - new object[] { 2u, - new DateTime(2024, 6, 15, 12, 0, 0, DateTimeKind.Utc), // 12:00 UTC - new DateTime(2024, 6, 15, 12, 0, 0, DateTimeKind.Utc) // 12:00 UTC = 14:00 Amsterdam - }, - }; - - await bulkCopy.WriteToServerAsync(data); - Console.WriteLine($" Inserted {bulkCopy.RowsWritten} rows via bulk copy"); - } + // Row 1: Same wall-clock time (12:00) in both columns + new object[] { 1u, + new DateTime(2024, 6, 15, 12, 0, 0, DateTimeKind.Unspecified), // 12:00 UTC + new DateTime(2024, 6, 15, 12, 0, 0, DateTimeKind.Unspecified) // 12:00 Amsterdam + }, + // Row 2: UTC DateTime - instant is preserved + new object[] { 2u, + new DateTime(2024, 6, 15, 12, 0, 0, DateTimeKind.Utc), // 12:00 UTC + new DateTime(2024, 6, 15, 12, 0, 0, DateTimeKind.Utc) // 12:00 UTC = 14:00 Amsterdam + }, + }; + + await client.InsertBinaryAsync(tableName, columns, data); + Console.WriteLine($" Inserted {data.Count} rows via InsertBinaryAsync"); // Read back var reader = (ClickHouseDataReader)await connection.ExecuteReaderAsync( diff --git a/examples/DataTypes/DataTypes_003_ComplexTypes.cs b/examples/DataTypes/DataTypes_003_ComplexTypes.cs index 48fd202d..36b5df80 100644 --- a/examples/DataTypes/DataTypes_003_ComplexTypes.cs +++ b/examples/DataTypes/DataTypes_003_ComplexTypes.cs @@ -1,5 +1,4 @@ using System.Net; -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -18,39 +17,38 @@ public static class ComplexTypes { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); Console.WriteLine("Complex Data Types Examples\n"); // Example 1: Arrays Console.WriteLine("1. Working with Arrays:"); - await Example1_Arrays(connection); + await Example1_Arrays(client); // Example 2: Maps Console.WriteLine("\n2. Working with Maps:"); - await Example2_Maps(connection); + await Example2_Maps(client); // Example 3: Tuples Console.WriteLine("\n3. Working with Tuples:"); - await Example3_Tuples(connection); + await Example3_Tuples(client); // Example 4: IP Addresses Console.WriteLine("\n4. Working with IP Addresses:"); - await Example4_IPAddresses(connection); + await Example4_IPAddresses(client); // Example 5: Nested structures Console.WriteLine("\n5. Working with Nested structures:"); - await Example5_Nested(connection); + await Example5_Nested(client); Console.WriteLine("\nAll complex data types examples completed!"); } - private static async Task Example1_Arrays(ClickHouseConnection connection) + private static async Task Example1_Arrays(ClickHouseClient client) { var tableName = "example_arrays"; - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( id UInt32, @@ -64,25 +62,18 @@ ORDER BY id Console.WriteLine($" Created table '{tableName}' with Array columns"); - // Insert data with arrays - using (var command = connection.CreateCommand()) + // Insert data with arrays using InsertBinaryAsync + var rows = new List { - command.CommandText = $@" - INSERT INTO {tableName} (id, tags, numbers, scores) - VALUES ({{id:UInt32}}, {{tags:Array(String)}}, {{numbers:Array(Int32)}}, {{scores:Array(Float64)}}) - "; - - command.AddParameter("id", 1); - command.AddParameter("tags", new[] { "important", "urgent", "review" }); - command.AddParameter("numbers", new List { 10, 20, 30, 40 }); // Lists are transformed to Arrays in ClickHouse - command.AddParameter("scores", new[] { 95.5, 87.3, 92.1 }); - await command.ExecuteNonQueryAsync(); - } + new object[] { 1u, new[] { "important", "urgent", "review" }, new[] { 10, 20, 30, 40 }, new[] { 95.5, 87.3, 92.1 } } + }; + var columns = new[] { "id", "tags", "numbers", "scores" }; + await client.InsertBinaryAsync(tableName, columns, rows); Console.WriteLine(" Inserted rows with array data"); // Query and read arrays - using (var reader = await connection.ExecuteReaderAsync($"SELECT id, tags, numbers, scores FROM {tableName} ORDER BY id")) + using (var reader = await client.ExecuteReaderAsync($"SELECT id, tags, numbers, scores FROM {tableName} ORDER BY id")) { Console.WriteLine("\n Reading array data:"); while (reader.Read()) @@ -101,20 +92,20 @@ ORDER BY id // Array functions Console.WriteLine("\n Using array functions:"); - var arrayLength = await connection.ExecuteScalarAsync($"SELECT length(tags) FROM {tableName} WHERE id = 1"); + var arrayLength = await client.ExecuteScalarAsync($"SELECT length(tags) FROM {tableName} WHERE id = 1"); Console.WriteLine($" Length of tags array for ID=1: {arrayLength}"); - var hasElement = await connection.ExecuteScalarAsync($"SELECT has(tags, 'urgent') FROM {tableName} WHERE id = 1"); + var hasElement = await client.ExecuteScalarAsync($"SELECT has(tags, 'urgent') FROM {tableName} WHERE id = 1"); Console.WriteLine($" Does ID=1 have 'urgent' tag? {hasElement}"); - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } - private static async Task Example2_Maps(ClickHouseConnection connection) + private static async Task Example2_Maps(ClickHouseClient client) { var tableName = "example_maps"; - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( user_id UInt32, @@ -126,28 +117,23 @@ ORDER BY user_id Console.WriteLine($" Created table '{tableName}' with Map columns"); - // Insert data with maps - using (var command = connection.CreateCommand()) + // Insert data with maps using InsertBinaryAsync + var rows = new List { - command.CommandText = $@" - INSERT INTO {tableName} (user_id, preferences) - VALUES ({{user_id:UInt32}}, {{preferences:Map(String, String)}}) - "; - - command.AddParameter("user_id", 1); - command.AddParameter("preferences", new Dictionary + new object[] { 1u, new Dictionary { { "theme", "dark" }, { "language", "en" }, { "timezone", "UTC" }, - }); - await command.ExecuteNonQueryAsync(); - } + }} + }; + var columns = new[] { "user_id", "preferences" }; + await client.InsertBinaryAsync(tableName, columns, rows); Console.WriteLine(" Inserted row with map data"); // Query and read maps - using (var reader = await connection.ExecuteReaderAsync($"SELECT user_id, preferences FROM {tableName}")) + using (var reader = await client.ExecuteReaderAsync($"SELECT user_id, preferences FROM {tableName}")) { Console.WriteLine("\n Reading map data:"); while (reader.Read()) @@ -166,18 +152,18 @@ ORDER BY user_id // Map functions Console.WriteLine("\n Using map functions:"); - var prefValue = await connection.ExecuteScalarAsync($"SELECT preferences['theme'] FROM {tableName} WHERE user_id = 1"); + var prefValue = await client.ExecuteScalarAsync($"SELECT preferences['theme'] FROM {tableName} WHERE user_id = 1"); Console.WriteLine($" Theme preference: {prefValue}"); - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } - private static async Task Example3_Tuples(ClickHouseConnection connection) + private static async Task Example3_Tuples(ClickHouseClient client) { var tableName = "example_tuples"; // Tuples can be named, or not - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( id UInt32, @@ -191,30 +177,23 @@ ORDER BY id Console.WriteLine($" Created table '{tableName}' with Tuple columns"); - // Insert data with tuples - using (var command = connection.CreateCommand()) + // Insert data with tuples using InsertBinaryAsync + var rows = new List { - command.CommandText = $@" - INSERT INTO {tableName} (id, person, coordinates, point) - VALUES ( - {{id:UInt32}}, - {{person:Tuple(String, UInt8, String)}}, - {{coordinates:Tuple(Float64, Float64, Float64)}}, - {{point:Tuple(Int32, Int32)}} - ) - "; - - command.AddParameter("id", 1); - command.AddParameter("person", Tuple.Create("Alice Johnson", (byte)30, "alice@example.com")); - command.AddParameter("coordinates", Tuple.Create(12.5, 34.7, 56.9)); - command.AddParameter("point", new List { 1, 2 }); - await command.ExecuteNonQueryAsync(); - } + new object[] { + 1u, + Tuple.Create("Alice Johnson", (byte)30, "alice@example.com"), + Tuple.Create(12.5, 34.7, 56.9), + Tuple.Create(1, 2) + } + }; + var columns = new[] { "id", "person", "coordinates", "point" }; + await client.InsertBinaryAsync(tableName, columns, rows); Console.WriteLine(" Inserted row with tuple data"); // Query and read tuples - using (var reader = await connection.ExecuteReaderAsync($"SELECT id, person, coordinates, point FROM {tableName}")) + using (var reader = await client.ExecuteReaderAsync($"SELECT id, person, coordinates, point FROM {tableName}")) { Console.WriteLine("\n Reading tuple data:"); while (reader.Read()) @@ -231,14 +210,14 @@ ORDER BY id } } - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } - private static async Task Example4_IPAddresses(ClickHouseConnection connection) + private static async Task Example4_IPAddresses(ClickHouseClient client) { var tableName = "example_ip_addresses"; - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( id UInt32, @@ -252,32 +231,19 @@ ORDER BY id Console.WriteLine($" Created table '{tableName}' with IPv4 and IPv6 columns"); - // Insert data with IP addresses - using (var command = connection.CreateCommand()) + // Insert data with IP addresses using InsertBinaryAsync + var rows = new List { - command.CommandText = $@" - INSERT INTO {tableName} (id, ipv4_addr, ipv6_addr, request_time) - VALUES ({{id:UInt32}}, {{ipv4:IPv4}}, {{ipv6:IPv6}}, {{time:DateTime}}) - "; - - command.AddParameter("id", 1); - command.AddParameter("ipv4", IPAddress.Parse("192.168.1.100")); - command.AddParameter("ipv6", IPAddress.Parse("2001:0db8:85a3:0000:0000:8a2e:0370:7334")); - command.AddParameter("time", DateTime.UtcNow); - await command.ExecuteNonQueryAsync(); - - command.Parameters.Clear(); - command.AddParameter("id", 2); - command.AddParameter("ipv4", IPAddress.Parse("10.0.0.1")); - command.AddParameter("ipv6", IPAddress.Parse("fe80::1")); - command.AddParameter("time", DateTime.UtcNow); - await command.ExecuteNonQueryAsync(); - } + new object[] { 1u, IPAddress.Parse("192.168.1.100"), IPAddress.Parse("2001:0db8:85a3:0000:0000:8a2e:0370:7334"), DateTime.UtcNow }, + new object[] { 2u, IPAddress.Parse("10.0.0.1"), IPAddress.Parse("fe80::1"), DateTime.UtcNow } + }; + var columns = new[] { "id", "ipv4_addr", "ipv6_addr", "request_time" }; + await client.InsertBinaryAsync(tableName, columns, rows); Console.WriteLine(" Inserted rows with IP address data"); // Query and read IP addresses - using (var reader = await connection.ExecuteReaderAsync($"SELECT id, ipv4_addr, ipv6_addr FROM {tableName} ORDER BY id")) + using (var reader = await client.ExecuteReaderAsync($"SELECT id, ipv4_addr, ipv6_addr FROM {tableName} ORDER BY id")) { Console.WriteLine("\n Reading IP address data:"); while (reader.Read()) @@ -292,15 +258,15 @@ ORDER BY id } } - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } - private static async Task Example5_Nested(ClickHouseConnection connection) + private static async Task Example5_Nested(ClickHouseClient client) { var tableName = "example_nested"; // Nested is a special type representing an array of tuples - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( order_id UInt32, @@ -317,17 +283,19 @@ ORDER BY order_id Console.WriteLine($" Created table '{tableName}' with Nested structure"); // Insert data - Nested columns are stored as separate arrays - await connection.ExecuteStatementAsync($@" - INSERT INTO {tableName} (order_id, items.name, items.quantity, items.price) - VALUES - (1, ['Widget', 'Gadget', 'Tool'], [2, 1, 3], [19.99, 49.99, 9.99]), - (2, ['Book', 'Pen'], [5, 10], [12.50, 1.25]) - "); + // Using InsertBinaryAsync with nested arrays + var rows = new List + { + new object[] { 1u, new[] { "Widget", "Gadget", "Tool" }, new uint[] { 2, 1, 3 }, new[] { 19.99, 49.99, 9.99 } }, + new object[] { 2u, new[] { "Book", "Pen" }, new uint[] { 5, 10 }, new[] { 12.50, 1.25 } } + }; + var columns = new[] { "order_id", "items.name", "items.quantity", "items.price" }; + await client.InsertBinaryAsync(tableName, columns, rows); Console.WriteLine(" Inserted rows with nested data"); // Query nested data - using (var reader = await connection.ExecuteReaderAsync($@" + using (var reader = await client.ExecuteReaderAsync($@" SELECT order_id, items.name, @@ -354,6 +322,6 @@ ORDER BY order_id } } - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } } diff --git a/examples/DataTypes/DataTypes_004_StringHandling.cs b/examples/DataTypes/DataTypes_004_StringHandling.cs index af2e0361..da27c59e 100644 --- a/examples/DataTypes/DataTypes_004_StringHandling.cs +++ b/examples/DataTypes/DataTypes_004_StringHandling.cs @@ -1,6 +1,5 @@ using System.Text; using ClickHouse.Driver.ADO; -using ClickHouse.Driver.Copy; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -110,17 +109,17 @@ fixed_str FixedString(5) ENGINE = Memory "); - using var bulkCopy = new ClickHouseBulkCopy(connection) { DestinationTableName = tableName }; - await bulkCopy.InitAsync(); + using var client = new ClickHouseClient("Host=localhost"); // Insert strings of different lengths + var columns = new[] { "fixed_str" }; var data = new List { new object[] { "AB" }, // 2 bytes -> padded to 5 new object[] { "ABCDE" }, // 5 bytes -> exact fit }; - await bulkCopy.WriteToServerAsync(data); + await client.InsertBinaryAsync(tableName, columns, data); // Read back and show the actual bytes var cb = new ClickHouseConnectionStringBuilder("Host=localhost") @@ -158,12 +157,12 @@ data String ENGINE = Memory "); - using var bulkCopy = new ClickHouseBulkCopy(connection) { DestinationTableName = tableName }; - await bulkCopy.InitAsync(); + using var client = new ClickHouseClient("Host=localhost"); // Write binary data that is NOT valid UTF-8 var binaryData = new byte[] { 0xFF, 0xFE, 0x00, 0x01, 0x02 }; + var columns = new[] { "id", "data" }; var data = new List { new object[] { 1u, binaryData }, // Write as byte[] @@ -171,8 +170,8 @@ data String new object[] { 3u, new MemoryStream(binaryData) }, // Write as Stream }; - await bulkCopy.WriteToServerAsync(data); - Console.WriteLine($" Inserted {bulkCopy.RowsWritten} rows with binary data"); + await client.InsertBinaryAsync(tableName, columns, data); + Console.WriteLine($" Inserted {data.Count} rows with binary data"); // Read back as byte[] to preserve the binary data var cb = new ClickHouseConnectionStringBuilder("Host=localhost") diff --git a/examples/DataTypes/DataTypes_005_JsonType.cs b/examples/DataTypes/DataTypes_005_JsonType.cs index c5808183..4b5e804b 100644 --- a/examples/DataTypes/DataTypes_005_JsonType.cs +++ b/examples/DataTypes/DataTypes_005_JsonType.cs @@ -1,7 +1,6 @@ using System.Text.Json; using System.Text.Json.Nodes; using ClickHouse.Driver.ADO; -using ClickHouse.Driver.Copy; using ClickHouse.Driver.Json; using ClickHouse.Driver.Utility; @@ -49,18 +48,15 @@ private static async Task Example1_InsertStringMode() Console.WriteLine("-".PadRight(50, '-')); // JsonWriteMode.String is the default, so we can omit it - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); - connection.CustomSettings["allow_experimental_json_type"] = 1; + using var client = new ClickHouseClient("Host=localhost;set_allow_experimental_json_type=1"); var tableName = "example_insert_string_mode"; - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($@" CREATE TABLE {tableName} (id UInt32, data Json) ENGINE = Memory"); - using var bulkCopy = new ClickHouseBulkCopy(connection) { DestinationTableName = tableName }; - // All these input types work with String mode + var columns = new[] { "id", "data" }; var rows = new List { // 1. Raw JSON string - passed through directly @@ -79,11 +75,11 @@ await connection.ExecuteStatementAsync($@" new object[] { 5u, new SimpleEvent { Source = "typed_poco", Value = 500 } }, }; - await bulkCopy.WriteToServerAsync(rows); - Console.WriteLine($" Inserted {bulkCopy.RowsWritten} rows with various input types\n"); + await client.InsertBinaryAsync(tableName, columns, rows); + Console.WriteLine($" Inserted {rows.Count} rows with various input types\n"); // Verify the data - using var reader = await connection.ExecuteReaderAsync( + using var reader = await client.ExecuteReaderAsync( $"SELECT id, data FROM {tableName} ORDER BY id"); Console.WriteLine(" Results:"); @@ -94,13 +90,13 @@ await connection.ExecuteStatementAsync($@" Console.WriteLine($" ID {id}: source={data["source"]}, value={data["value"]}"); } - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } /// /// Writing in Binary mode writes POCOs using ClickHouse's native binary format. /// This preserves type information and supports custom path mappings. - /// + /// /// For paths that have types defined in the column, the type information will be used for appropriate serialization. /// For paths that are not typed, the client will use type inference to pick a suitable ClickHouse type depending on the .NET type. /// @@ -119,21 +115,18 @@ private static async Task Example2_InsertBinaryMode() Console.WriteLine("-".PadRight(50, '-')); // Must explicitly set Binary mode - using var connection = new ClickHouseConnection("Host=localhost;JsonWriteMode=Binary"); - await connection.OpenAsync(); - connection.CustomSettings["allow_experimental_json_type"] = 1; + using var client = new ClickHouseClient("Host=localhost;JsonWriteMode=Binary;set_allow_experimental_json_type=1"); // Register POCO types before using them - connection.RegisterJsonSerializationType(); + client.RegisterJsonSerializationType(); Console.WriteLine(" Registered POCO types for binary serialization\n"); var tableName = "example_insert_binary_mode"; - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($@" CREATE TABLE {tableName} (id UInt32, data Json) ENGINE = Memory"); - using var bulkCopy = new ClickHouseBulkCopy(connection) { DestinationTableName = tableName }; - + var columns = new[] { "id", "data" }; var rows = new List { // Custom path mapping: EventType -> "type", XCoordinate -> "position.x" @@ -146,11 +139,11 @@ await connection.ExecuteStatementAsync($@" }}, }; - await bulkCopy.WriteToServerAsync(rows); - Console.WriteLine($" Inserted {bulkCopy.RowsWritten} rows with binary serialization\n"); + await client.InsertBinaryAsync(tableName, columns, rows); + Console.WriteLine($" Inserted {rows.Count} rows with binary serialization\n"); // Verify the data - using var reader = await connection.ExecuteReaderAsync( + using var reader = await client.ExecuteReaderAsync( $"SELECT id, data FROM {tableName} ORDER BY id"); Console.WriteLine(" Results:"); @@ -161,7 +154,7 @@ await connection.ExecuteStatementAsync($@" Console.WriteLine($" ID {id}: {data.ToJsonString()}"); } - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } /// @@ -276,14 +269,16 @@ tags Array(String), Console.WriteLine(" - metadata.created: DateTime"); // Insert data - hints ensure proper type handling - connection.CustomSettings["date_time_input_format"] = "best_effort"; - using var bulkCopy2 = new ClickHouseBulkCopy(connection) { DestinationTableName = typedTable }; + using var client = new ClickHouseClient("Host=localhost;set_allow_experimental_json_type=1;set_date_time_input_format=best_effort"); - await bulkCopy2.WriteToServerAsync(new[] + var columns = new[] { "id", "data" }; + var rows = new[] { new object[] { 1u, """{"name": "Alice", "age": 30, "tags": ["admin", "active"], "metadata": {"created": "2024-01-15T10:30:00Z"}}""" }, new object[] { 2u, """{"name": "Bob", "age": 25, "tags": ["user"], "metadata": {"created": "2024-02-20T14:45:00Z"}}""" }, - }); + }; + + await client.InsertBinaryAsync(typedTable, columns, rows); // Query paths Console.WriteLine("\n Querying typed paths:"); diff --git a/examples/DataTypes/DataTypes_006_Geometry.cs b/examples/DataTypes/DataTypes_006_Geometry.cs index 22f41ffd..e48c449e 100644 --- a/examples/DataTypes/DataTypes_006_Geometry.cs +++ b/examples/DataTypes/DataTypes_006_Geometry.cs @@ -1,4 +1,4 @@ -using ClickHouse.Driver.ADO; +using ClickHouse.Driver.ADO.Parameters; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -16,30 +16,29 @@ public static class GeometryTypes { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); Console.WriteLine("Geometry Types Examples\n"); // Example 1: Basic geometry types Console.WriteLine("1. Basic Geometry Types (Point, Polygon):"); - await Example1_BasicGeometryTypes(connection); + await Example1_BasicGeometryTypes(client); // Example 2: WKT support Console.WriteLine("\n2. WKT (Well-Known Text) Support:"); - await Example2_WktSupport(connection); + await Example2_WktSupport(client); // Example 3: H3 indexing Console.WriteLine("\n3. H3 Geospatial Indexing:"); - await Example3_H3Indexing(connection); + await Example3_H3Indexing(client); // Example 4: Point in polygon Console.WriteLine("\n4. Point-in-Polygon Containment:"); - await Example4_PointInPolygon(connection); + await Example4_PointInPolygon(client); // Example 5: Great circle distance Console.WriteLine("\n5. Great Circle Distance:"); - await Example5_GreatCircleDistance(connection); + await Example5_GreatCircleDistance(client); Console.WriteLine("\nAll geometry examples completed!"); } @@ -49,11 +48,11 @@ public static async Task Run() /// Points are represented as Tuple<double, double> (x, y). /// Polygons are arrays of rings, where each ring is an array of points. /// - private static async Task Example1_BasicGeometryTypes(ClickHouseConnection connection) + private static async Task Example1_BasicGeometryTypes(ClickHouseClient client) { var tableName = "example_geometry_basic"; - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( id UInt32, @@ -66,64 +65,59 @@ ORDER BY id Console.WriteLine($" Created table '{tableName}' with Point and Polygon columns"); - // Insert data with geometry types - using (var command = connection.CreateCommand()) + // Insert data with geometry types using InsertBinaryAsync + // Point is represented as Tuple (x, y) or (longitude, latitude) + var location = Tuple.Create(4.9041, 52.3676); // Amsterdam coordinates (lon, lat) + + // Polygon is an array of rings. First ring is outer boundary, subsequent rings are holes. + // Each ring is an array of points, with the last point equal to the first (closed ring). + // This polygon roughly covers central Amsterdam + var outerRing = new[] { - command.CommandText = $@" - INSERT INTO {tableName} (id, location, boundary) - VALUES ({{id:UInt32}}, {{location:Point}}, {{boundary:Polygon}}) - "; - - // Point is represented as Tuple (x, y) or (longitude, latitude) - command.AddParameter("id", 1); - command.AddParameter("location", Tuple.Create(4.9041, 52.3676)); // Amsterdam coordinates (lon, lat) - - // Polygon is an array of rings. First ring is outer boundary, subsequent rings are holes. - // Each ring is an array of points, with the last point equal to the first (closed ring). - // This polygon roughly covers central Amsterdam - var outerRing = new[] - { - Tuple.Create(4.85, 52.35), - Tuple.Create(4.95, 52.35), - Tuple.Create(4.95, 52.40), - Tuple.Create(4.85, 52.40), - Tuple.Create(4.85, 52.35) // Close the ring - }; - command.AddParameter("boundary", new[] { outerRing }); - - await command.ExecuteNonQueryAsync(); - } + Tuple.Create(4.85, 52.35), + Tuple.Create(4.95, 52.35), + Tuple.Create(4.95, 52.40), + Tuple.Create(4.85, 52.40), + Tuple.Create(4.85, 52.35) // Close the ring + }; + + var rows = new List + { + new object[] { 1u, location, new[] { outerRing } } + }; + var columns = new[] { "id", "location", "boundary" }; + await client.InsertBinaryAsync(tableName, columns, rows); Console.WriteLine(" Inserted row with Point and Polygon data"); // Read back the geometry data - using (var reader = await connection.ExecuteReaderAsync($"SELECT id, location, boundary FROM {tableName}")) + using (var reader = await client.ExecuteReaderAsync($"SELECT id, location, boundary FROM {tableName}")) { Console.WriteLine("\n Reading geometry data:"); while (reader.Read()) { var id = reader.GetFieldValue(0); - var location = reader.GetFieldValue>(1); + var loc = reader.GetFieldValue>(1); var boundary = reader.GetFieldValue[][]>(2); Console.WriteLine($"\n ID: {id}"); - Console.WriteLine($" Location (Point): ({location.Item1}, {location.Item2})"); + Console.WriteLine($" Location (Point): ({loc.Item1}, {loc.Item2})"); Console.WriteLine($" Boundary (Polygon): {boundary.Length} ring(s), outer ring has {boundary[0].Length} points"); } } - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } /// /// Demonstrates WKT (Well-Known Text) parsing functions for creating geometry from text. /// WKT is a standard text format for representing geometry objects. /// - private static async Task Example2_WktSupport(ClickHouseConnection connection) + private static async Task Example2_WktSupport(ClickHouseClient client) { // Parse Point from WKT Console.WriteLine(" Parsing Point from WKT:"); - using (var reader = await connection.ExecuteReaderAsync("SELECT readWKTPoint('POINT (37.6173 55.7558)')")) + using (var reader = await client.ExecuteReaderAsync("SELECT readWKTPoint('POINT (37.6173 55.7558)')")) { if (reader.Read()) { @@ -134,7 +128,7 @@ private static async Task Example2_WktSupport(ClickHouseConnection connection) // Parse Polygon from WKT Console.WriteLine("\n Parsing Polygon from WKT:"); - using (var reader = await connection.ExecuteReaderAsync(@" + using (var reader = await client.ExecuteReaderAsync(@" SELECT readWKTPolygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))') ")) { @@ -148,7 +142,7 @@ SELECT readWKTPolygon('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))') // Parse Ring from WKT - Ring uses POLYGON format Console.WriteLine("\n Parsing Ring from WKT:"); - using (var reader = await connection.ExecuteReaderAsync(@" + using (var reader = await client.ExecuteReaderAsync(@" SELECT readWKTRing('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))') ")) { @@ -161,7 +155,7 @@ SELECT readWKTRing('POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))') // Parse MultiPolygon from WKT Console.WriteLine("\n Parsing MultiPolygon from WKT:"); - using (var reader = await connection.ExecuteReaderAsync(@" + using (var reader = await client.ExecuteReaderAsync(@" SELECT readWKTMultiPolygon('MULTIPOLYGON (((0 0, 10 0, 10 10, 0 10, 0 0)), ((20 20, 30 20, 30 30, 20 30, 20 20)))') ")) { @@ -178,7 +172,7 @@ SELECT readWKTMultiPolygon('MULTIPOLYGON (((0 0, 10 0, 10 10, 0 10, 0 0)), ((20 /// H3 is a hierarchical hexagonal grid system for indexing geographic coordinates. /// Note: As of ClickHouse 25.5, geoToH3() uses (lat, lon) order. /// - private static async Task Example3_H3Indexing(ClickHouseConnection connection) + private static async Task Example3_H3Indexing(ClickHouseClient client) { // Basic geoToH3 conversion Console.WriteLine(" Converting coordinates to H3 index:"); @@ -187,37 +181,34 @@ private static async Task Example3_H3Indexing(ClickHouseConnection connection) var lat = 52.33676; var lon = 4.9041; - using (var command = connection.CreateCommand()) + // geoToH3(lat, lon, resolution) - note: lat, lon order as of ClickHouse 25.5 + var parameters = new ClickHouseParameterCollection(); + parameters.AddParameter("lat", lat); + parameters.AddParameter("lon", lon); + + using (var reader = await client.ExecuteReaderAsync(@" + SELECT + geoToH3({lat:Float64}, {lon:Float64}, 5) AS h3_res5, + geoToH3({lat:Float64}, {lon:Float64}, 10) AS h3_res10, + geoToH3({lat:Float64}, {lon:Float64}, 15) AS h3_res15 + ", parameters)) { - // geoToH3(lat, lon, resolution) - note: lat, lon order as of ClickHouse 25.5 - command.CommandText = @" - SELECT - geoToH3({lat:Float64}, {lon:Float64}, 5) AS h3_res5, - geoToH3({lat:Float64}, {lon:Float64}, 10) AS h3_res10, - geoToH3({lat:Float64}, {lon:Float64}, 15) AS h3_res15 - "; - command.AddParameter("lat", lat); - command.AddParameter("lon", lon); - - using (var reader = await command.ExecuteReaderAsync()) + if (reader.Read()) { - if (reader.Read()) - { - var h3Res5 = reader.GetFieldValue(0); - var h3Res10 = reader.GetFieldValue(1); - var h3Res15 = reader.GetFieldValue(2); - - Console.WriteLine($" Coordinates: ({lat}, {lon}) - Amsterdam"); - Console.WriteLine($" H3 Resolution 5: {h3Res5}"); - Console.WriteLine($" H3 Resolution 10: {h3Res10}"); - Console.WriteLine($" H3 Resolution 15: {h3Res15}"); - } + var h3Res5 = reader.GetFieldValue(0); + var h3Res10 = reader.GetFieldValue(1); + var h3Res15 = reader.GetFieldValue(2); + + Console.WriteLine($" Coordinates: ({lat}, {lon}) - Amsterdam"); + Console.WriteLine($" H3 Resolution 5: {h3Res5}"); + Console.WriteLine($" H3 Resolution 10: {h3Res10}"); + Console.WriteLine($" H3 Resolution 15: {h3Res15}"); } } // Convert H3 index back to coordinates Console.WriteLine("\n Converting H3 index back to coordinates:"); - using (var reader = await connection.ExecuteReaderAsync($@" + using (var reader = await client.ExecuteReaderAsync($@" SELECT h3ToGeo(geoToH3({lat}, {lon}, 10)) AS center_point ")) { @@ -230,7 +221,7 @@ SELECT h3ToGeo(geoToH3({lat}, {lon}, 10)) AS center_point // Get H3 cell boundary as polygon Console.WriteLine("\n Getting H3 cell boundary:"); - using (var reader = await connection.ExecuteReaderAsync($@" + using (var reader = await client.ExecuteReaderAsync($@" SELECT h3ToGeoBoundary(geoToH3({lat}, {lon}, 8)) AS boundary ")) { @@ -245,7 +236,7 @@ SELECT h3ToGeoBoundary(geoToH3({lat}, {lon}, 8)) AS boundary Console.WriteLine("\n Practical: Grouping locations by H3 cell:"); var tableName = "example_h3_locations"; - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( name String, @@ -256,16 +247,17 @@ lon Float64 ORDER BY name "); - await connection.ExecuteStatementAsync($@" - INSERT INTO {tableName} (name, lat, lon) - VALUES - ('Location A', 52.3731, 4.8936), - ('Location B', 52.3735, 4.8940), - ('Location C', 52.3750, 4.8950), - ('Location D', 52.3900, 4.9100) - "); + var rows = new List + { + new object[] { "Location A", 52.3731, 4.8936 }, + new object[] { "Location B", 52.3735, 4.8940 }, + new object[] { "Location C", 52.3750, 4.8950 }, + new object[] { "Location D", 52.3900, 4.9100 } + }; + var columns = new[] { "name", "lat", "lon" }; + await client.InsertBinaryAsync(tableName, columns, rows); - using (var reader = await connection.ExecuteReaderAsync($@" + using (var reader = await client.ExecuteReaderAsync($@" SELECT geoToH3(lat, lon, 7) AS h3_cell, count() AS location_count, @@ -286,21 +278,21 @@ ORDER BY location_count DESC } } - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } /// /// Demonstrates point-in-polygon containment checks. /// Useful for geofencing, area-based filtering, and spatial queries. /// - private static async Task Example4_PointInPolygon(ClickHouseConnection connection) + private static async Task Example4_PointInPolygon(ClickHouseClient client) { // Define a simple rectangular polygon (e.g., a geofence) Console.WriteLine(" Basic point-in-polygon check:"); // Polygon vertices (clockwise or counter-clockwise) // This represents a rectangular area - using (var reader = await connection.ExecuteReaderAsync(@" + using (var reader = await client.ExecuteReaderAsync(@" SELECT pointInPolygon((5, 5), [(0, 0), (10, 0), (10, 10), (0, 10)]) AS inside, pointInPolygon((15, 15), [(0, 0), (10, 0), (10, 10), (0, 10)]) AS outside @@ -319,7 +311,7 @@ private static async Task Example4_PointInPolygon(ClickHouseConnection connectio Console.WriteLine("\n Practical: Filtering locations within a geofence:"); var tableName = "example_geofence"; - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( device_id String, @@ -331,18 +323,19 @@ ORDER BY device_id "); // Insert some device locations around Amsterdam - await connection.ExecuteStatementAsync($@" - INSERT INTO {tableName} (device_id, lat, lon) - VALUES - ('device_1', 52.3731, 4.8936), - ('device_2', 52.3752, 4.8840), - ('device_3', 52.3105, 4.7683), - ('device_4', 52.3600, 4.8852) - "); + var rows = new List + { + new object[] { "device_1", 52.3731, 4.8936 }, + new object[] { "device_2", 52.3752, 4.8840 }, + new object[] { "device_3", 52.3105, 4.7683 }, + new object[] { "device_4", 52.3600, 4.8852 } + }; + var columns = new[] { "device_id", "lat", "lon" }; + await client.InsertBinaryAsync(tableName, columns, rows); // Define a geofence polygon around central Amsterdam // Check which devices are inside the geofence - using (var reader = await connection.ExecuteReaderAsync($@" + using (var reader = await client.ExecuteReaderAsync($@" SELECT device_id, lat, @@ -367,14 +360,14 @@ ORDER BY device_id } } - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } /// /// Demonstrates great circle distance calculations between geographic coordinates. /// Uses the haversine formula to calculate the shortest distance over the Earth's surface. /// - private static async Task Example5_GreatCircleDistance(ClickHouseConnection connection) + private static async Task Example5_GreatCircleDistance(ClickHouseClient client) { // Calculate distance between two cities Console.WriteLine(" Distance between cities:"); @@ -393,7 +386,7 @@ private static async Task Example5_GreatCircleDistance(ClickHouseConnection conn Console.WriteLine(" From Amsterdam to:"); foreach (var (name, lon, lat) in cities.Skip(1)) { - using (var reader = await connection.ExecuteReaderAsync($@" + using (var reader = await client.ExecuteReaderAsync($@" SELECT greatCircleDistance(4.9041, 52.3676, {lon}, {lat}) AS distance_meters ")) { @@ -411,7 +404,7 @@ SELECT greatCircleDistance(4.9041, 52.3676, {lon}, {lat}) AS distance_meters Console.WriteLine("\n Practical: Finding nearby locations:"); var tableName = "example_locations_distance"; - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( name String, @@ -422,22 +415,23 @@ lat Float64 ORDER BY name "); - await connection.ExecuteStatementAsync($@" - INSERT INTO {tableName} (name, lon, lat) - VALUES - ('Dam Square', 4.8936, 52.3731), - ('Anne Frank House', 4.8840, 52.3752), - ('Rijksmuseum', 4.8852, 52.3600), - ('Amsterdam Centraal', 4.9003, 52.3791), - ('Schiphol Airport', 4.7683, 52.3105) - "); + var rows = new List + { + new object[] { "Dam Square", 4.8936, 52.3731 }, + new object[] { "Anne Frank House", 4.8840, 52.3752 }, + new object[] { "Rijksmuseum", 4.8852, 52.3600 }, + new object[] { "Amsterdam Centraal", 4.9003, 52.3791 }, + new object[] { "Schiphol Airport", 4.7683, 52.3105 } + }; + var columns = new[] { "name", "lon", "lat" }; + await client.InsertBinaryAsync(tableName, columns, rows); // Find locations within 5km of Dam Square var centerLon = 4.8936; var centerLat = 52.3731; var radiusMeters = 5000; - using (var reader = await connection.ExecuteReaderAsync($@" + using (var reader = await client.ExecuteReaderAsync($@" SELECT name, greatCircleDistance({centerLon}, {centerLat}, lon, lat) AS distance @@ -455,6 +449,6 @@ ORDER BY distance } } - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } } diff --git a/examples/DataTypes/Vector_001_QBitSimilaritySearch.cs b/examples/DataTypes/Vector_001_QBitSimilaritySearch.cs index 50cd61a1..589ad4ad 100644 --- a/examples/DataTypes/Vector_001_QBitSimilaritySearch.cs +++ b/examples/DataTypes/Vector_001_QBitSimilaritySearch.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -12,15 +11,14 @@ public static class QBitSimilaritySearch { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); Console.WriteLine("=== QBit Similarity Search with Different Precision Levels ===\n"); var tableName = "example_qbit_similarity"; - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($@" CREATE TABLE {tableName} ( word String, @@ -32,7 +30,7 @@ ORDER BY word // Insert sample word embeddings (simplified 5-dimensional vectors) // In practice, these would come from an embedding model - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" INSERT INTO {tableName} VALUES ('apple', [0.9, 0.1, 0.8, 0.2, 0.7]), ('banana', [0.85, 0.15, 0.75, 0.25, 0.65]), @@ -51,7 +49,7 @@ INSERT INTO {tableName} VALUES Console.WriteLine("=== High Precision Search (32 bits) ==="); Console.WriteLine("Using L2DistanceTransposed with precision=32\n"); - using (var reader = await connection.ExecuteReaderAsync($@" + using (var reader = await client.ExecuteReaderAsync($@" SELECT word, L2DistanceTransposed(vec, {queryVector}, 32) AS distance @@ -74,7 +72,7 @@ ORDER BY distance Console.WriteLine("\n=== Low Precision Search (12 bits) ==="); Console.WriteLine("Using L2DistanceTransposed with precision=12\n"); - using (var reader = await connection.ExecuteReaderAsync($@" + using (var reader = await client.ExecuteReaderAsync($@" SELECT word, L2DistanceTransposed(vec, {queryVector}, 12) AS distance @@ -95,7 +93,7 @@ ORDER BY distance // Read vector data back as float[] Console.WriteLine("\n=== Reading QBit Data ===\n"); - using (var reader = await connection.ExecuteReaderAsync($"SELECT word, vec FROM {tableName} LIMIT 3")) + using (var reader = await client.ExecuteReaderAsync($"SELECT word, vec FROM {tableName} LIMIT 3")) { while (reader.Read()) { @@ -105,7 +103,7 @@ ORDER BY distance } } - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); Console.WriteLine($"\nCleaned up table '{tableName}'"); } } diff --git a/examples/Insert/Insert_001_SimpleDataInsert.cs b/examples/Insert/Insert_001_SimpleDataInsert.cs index d6f12234..351a75a7 100644 --- a/examples/Insert/Insert_001_SimpleDataInsert.cs +++ b/examples/Insert/Insert_001_SimpleDataInsert.cs @@ -1,45 +1,44 @@ using ClickHouse.Driver.ADO; +using ClickHouse.Driver.ADO.Parameters; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; /// -/// Demonstrates simple data insertion methods in ClickHouse. -/// Shows parameterized queries and basic INSERT statements. -/// For performant inserts using the binary format, see the BulkCopy examples instead. +/// Demonstrates three data insertion approaches in ClickHouse: +/// 1. InsertBinaryAsync - High-performance binary format (recommended for bulk data) +/// 2. ExecuteStatementAsync with parameters - SQL with parameterized values +/// 3. ADO.NET Command - Classic ADO.NET pattern with ClickHouseCommand /// public static class SimpleDataInsert { private const string TableName = "example_simple_insert"; - private const string ExceptTableName = "example_insert_except"; public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); - - await SetupTable(connection); + using var client = new ClickHouseClient("Host=localhost"); - // Insert a single row using {name:Type} parameter syntax - await InsertUsingParameterizedQuery(connection); + await SetupTable(client); - // Insert multiple rows by reusing the same command - await InsertMultipleRowsWithCommandReuse(connection); + // Option 1: InsertBinaryAsync (recommended for bulk inserts) + await InsertUsingBinaryAsync(client); - // Simple insert without parameters using extension method - await InsertUsingExecuteStatementAsync(connection); + // Option 2: ExecuteStatementAsync with parameters + await InsertUsingParameterizedStatement(client); - await VerifyInsertedData(connection); + // Option 3: ADO.NET Command pattern + await InsertUsingAdoCommand(); - // Use EXCEPT clause to skip columns with DEFAULT values - await InsertUsingExceptClause(connection); + // Option 4: EXCEPT clause for DEFAULT columns + await InsertUsingExceptClause(client); - await Cleanup(connection); + await VerifyInsertedData(client); + await Cleanup(client); } - private static async Task SetupTable(ClickHouseConnection connection) + private static async Task SetupTable(ClickHouseClient client) { - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {TableName} ( id UInt64, @@ -57,89 +56,92 @@ ORDER BY (id) } /// - /// Demonstrates inserting a single row using parameterized query. - /// Parameters are specified using {name:Type} syntax in the query. + /// Option 1: InsertBinaryAsync - High-performance binary format. + /// Recommended for inserting multiple rows efficiently. /// - private static async Task InsertUsingParameterizedQuery(ClickHouseConnection connection) + private static async Task InsertUsingBinaryAsync(ClickHouseClient client) { - Console.WriteLine("1. Inserting data using parameterized query:"); - using var command = connection.CreateCommand(); + Console.WriteLine("1. InsertBinaryAsync (recommended for bulk inserts):"); - command.CommandText = $@" - INSERT INTO {TableName} (id, name, email, age, score, registered_at) - VALUES ({{id:UInt64}}, {{name:String}}, {{email:String}}, {{age:UInt8}}, {{score:Float32}}, {{timestamp:DateTime}})"; + var rows = new List + { + new object[] { 1UL, "Alice Smith", "alice@example.com", (byte)28, 95.5f, DateTime.UtcNow }, + new object[] { 2UL, "Bob Johnson", "bob@example.com", (byte)35, 87.3f, DateTime.UtcNow }, + new object[] { 3UL, "Carol White", "carol@example.com", (byte)42, 92.1f, DateTime.UtcNow }, + }; - command.AddParameter("id", 1); - command.AddParameter("name", "Alice Smith"); - command.AddParameter("email", "alice@example.com"); - command.AddParameter("age", 28); - command.AddParameter("score", 95.5f); - command.AddParameter("timestamp", DateTime.UtcNow); + var columns = new[] { "id", "name", "email", "age", "score", "registered_at" }; + await client.InsertBinaryAsync(TableName, columns, rows); - await command.ExecuteNonQueryAsync(); - Console.WriteLine(" Inserted 1 row with parameters\n"); + Console.WriteLine($" Inserted {rows.Count} rows using binary format\n"); } /// - /// Demonstrates inserting multiple rows by reusing a command with different parameter values. - /// The Parameters collection is cleared between each insert. + /// Option 2: ExecuteNonQueryAsync with parameters. + /// Uses {name:Type} syntax for parameterized values. /// - private static async Task InsertMultipleRowsWithCommandReuse(ClickHouseConnection connection) + private static async Task InsertUsingParameterizedStatement(ClickHouseClient client) { - Console.WriteLine("2. Inserting multiple rows using parameters:"); - using var command = connection.CreateCommand(); + Console.WriteLine("2. ExecuteNonQueryAsync with parameters:"); - command.CommandText = $@" - INSERT INTO {TableName} (id, name, email, age, score, registered_at) - VALUES ({{id:UInt64}}, {{name:String}}, {{email:String}}, {{age:UInt8}}, {{score:Float32}}, {{timestamp:DateTime}})"; + var parameters = new ClickHouseParameterCollection(); + parameters.AddParameter("id", 4UL); + parameters.AddParameter("name", "David Brown"); + parameters.AddParameter("email", "david@example.com"); + parameters.AddParameter("age", (byte)29); + parameters.AddParameter("score", 88.9f); + parameters.AddParameter("timestamp", DateTime.UtcNow); - var users = new[] - { - new { Id = 2UL, Name = "Bob Johnson", Email = "bob@example.com", Age = (byte)35, Score = 87.3f }, - new { Id = 3UL, Name = "Carol White", Email = "carol@example.com", Age = (byte)42, Score = 92.1f }, - new { Id = 4UL, Name = "David Brown", Email = "david@example.com", Age = (byte)29, Score = 88.9f }, - }; - - foreach (var user in users) - { - command.Parameters.Clear(); - command.AddParameter("id", user.Id); - command.AddParameter("name", user.Name); - command.AddParameter("email", user.Email); - command.AddParameter("age", user.Age); - command.AddParameter("score", user.Score); - command.AddParameter("timestamp", DateTime.UtcNow); - await command.ExecuteNonQueryAsync(); - } + await client.ExecuteNonQueryAsync($@" + INSERT INTO {TableName} (id, name, email, age, score, registered_at) + VALUES ({{id:UInt64}}, {{name:String}}, {{email:String}}, {{age:UInt8}}, {{score:Float32}}, {{timestamp:DateTime}})", + parameters); - Console.WriteLine($" Inserted {users.Length} rows\n"); + Console.WriteLine(" Inserted 1 row using parameterized statement\n"); } /// - /// Demonstrates inserting data using ExecuteStatementAsync extension method. - /// Suitable for simple cases where parameterization is not needed. + /// Option 3: ADO.NET Command pattern. + /// Classic approach using ClickHouseConnection and ClickHouseCommand. + /// Useful when integrating with ORMs or ADO.NET-based tools. /// - private static async Task InsertUsingExecuteStatementAsync(ClickHouseConnection connection) + private static async Task InsertUsingAdoCommand() { - Console.WriteLine("3. Inserting data using ExecuteStatementAsync:"); - await connection.ExecuteStatementAsync($@" + Console.WriteLine("3. ADO.NET Command pattern:"); + + using var connection = new ClickHouseConnection("Host=localhost"); + await connection.OpenAsync(); + + using var command = connection.CreateCommand(); + command.CommandText = $@" INSERT INTO {TableName} (id, name, email, age, score, registered_at) - VALUES (5, 'Eve Davis', 'eve@example.com', 31, 91.7, now()) - "); - Console.WriteLine(" Inserted 1 row using ExecuteStatementAsync\n"); + VALUES ({{id:UInt64}}, {{name:String}}, {{email:String}}, {{age:UInt8}}, {{score:Float32}}, {{timestamp:DateTime}})"; + + command.AddParameter("id", 5UL); + command.AddParameter("name", "Eve Davis"); + command.AddParameter("email", "eve@example.com"); + command.AddParameter("age", (byte)31); + command.AddParameter("score", 91.7f); + command.AddParameter("timestamp", DateTime.UtcNow); + + await command.ExecuteNonQueryAsync(); + + Console.WriteLine(" Inserted 1 row using ADO.NET Command\n"); } /// - /// Demonstrates inserting data using the EXCEPT clause to exclude columns with DEFAULT values. - /// This is useful when you want the server to populate certain columns automatically. + /// Option 4: Using EXCEPT clause to skip columns with DEFAULT values. + /// The server automatically populates columns excluded via EXCEPT. /// - private static async Task InsertUsingExceptClause(ClickHouseConnection connection) + private static async Task InsertUsingExceptClause(ClickHouseClient client) { - Console.WriteLine("4. Inserting data using EXCEPT clause:"); + Console.WriteLine("4. Insert using EXCEPT clause:"); + + var exceptTableName = "example_insert_except"; // Create a table with DEFAULT columns - await connection.ExecuteStatementAsync($@" - CREATE TABLE IF NOT EXISTS {ExceptTableName} ( + await client.ExecuteNonQueryAsync($@" + CREATE TABLE IF NOT EXISTS {exceptTableName} ( id Int32, name String, value Float64, @@ -148,22 +150,20 @@ updated DateTime DEFAULT now() ) ENGINE = Memory "); - using var command = connection.CreateCommand(); - // Use EXCEPT to exclude columns that have DEFAULT values - // The server will automatically populate 'created' and 'updated' columns - command.CommandText = $"INSERT INTO {ExceptTableName} (* EXCEPT (created, updated)) VALUES ({{id:Int32}}, {{name:String}}, {{value:Float64}})"; + var parameters = new ClickHouseParameterCollection(); + parameters.AddParameter("id", 1); + parameters.AddParameter("name", "Test Item"); + parameters.AddParameter("value", 123.45); - command.AddParameter("id", 1); - command.AddParameter("name", "Test Item"); - command.AddParameter("value", 123.45); + await client.ExecuteNonQueryAsync( + $"INSERT INTO {exceptTableName} (* EXCEPT (created, updated)) VALUES ({{id:Int32}}, {{name:String}}, {{value:Float64}})", + parameters); - await command.ExecuteNonQueryAsync(); - Console.WriteLine(" Inserted 1 row using EXCEPT clause (created/updated auto-populated)\n"); + Console.WriteLine(" Inserted 1 row using EXCEPT clause (created/updated auto-populated)"); // Verify the data was inserted with DEFAULT values - using var reader = await connection.ExecuteReaderAsync($"SELECT * FROM {ExceptTableName}"); - Console.WriteLine(" Result:"); + using var reader = await client.ExecuteReaderAsync($"SELECT * FROM {exceptTableName}"); while (reader.Read()) { var id = reader.GetInt32(0); @@ -171,17 +171,17 @@ updated DateTime DEFAULT now() var value = reader.GetDouble(2); var created = reader.GetDateTime(3); var updated = reader.GetDateTime(4); - Console.WriteLine($" id={id}, name={name}, value={value}, created={created:HH:mm:ss}, updated={updated:HH:mm:ss}\n"); + Console.WriteLine($" Result: id={id}, name={name}, value={value}, created={created:HH:mm:ss}, updated={updated:HH:mm:ss}\n"); } - // Clean up the EXCEPT example table - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {ExceptTableName}"); + // Clean up + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {exceptTableName}"); } - private static async Task VerifyInsertedData(ClickHouseConnection connection) + private static async Task VerifyInsertedData(ClickHouseClient client) { Console.WriteLine("Verifying inserted data:"); - using var reader = await connection.ExecuteReaderAsync($"SELECT * FROM {TableName} ORDER BY id"); + using var reader = await client.ExecuteReaderAsync($"SELECT * FROM {TableName} ORDER BY id"); Console.WriteLine("ID\tName\t\t\tEmail\t\t\t\tAge\tScore\tRegistered At"); Console.WriteLine("--\t----\t\t\t-----\t\t\t\t---\t-----\t-------------"); @@ -198,13 +198,13 @@ private static async Task VerifyInsertedData(ClickHouseConnection connection) Console.WriteLine($"{id}\t{name,-20}\t{email,-30}\t{age}\t{score:F1}\t{registeredAt:yyyy-MM-dd HH:mm:ss}"); } - var count = await connection.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + var count = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); Console.WriteLine($"\nTotal rows inserted: {count}"); } - private static async Task Cleanup(ClickHouseConnection connection) + private static async Task Cleanup(ClickHouseClient client) { - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {TableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {TableName}"); Console.WriteLine($"\nTable '{TableName}' dropped"); } } diff --git a/examples/Insert/Insert_002_BulkInsert.cs b/examples/Insert/Insert_002_BulkInsert.cs index 7f9ac667..a8498428 100644 --- a/examples/Insert/Insert_002_BulkInsert.cs +++ b/examples/Insert/Insert_002_BulkInsert.cs @@ -1,24 +1,21 @@ -using ClickHouse.Driver.ADO; -using ClickHouse.Driver.Copy; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; /// -/// Demonstrates high-performance bulk data insertion using ClickHouseBulkCopy. +/// Demonstrates high-performance bulk data insertion using InsertBinaryAsync. /// This is the recommended approach for inserting large amounts of data efficiently. /// public static class BulkInsert { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); var tableName = "example_bulk_insert"; // Create a test table - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( id UInt64, @@ -34,33 +31,30 @@ ORDER BY (id) Console.WriteLine($"Table '{tableName}' created\n"); - // Example 1: Bulk insert for large data sets - Console.WriteLine("1. Bulk inserting:"); - using (var bulkCopy = new ClickHouseBulkCopy(connection) - { - DestinationTableName = tableName, - BatchSize = 1000, // Number of rows per batch. Due to the way the MergeTree table works, it is recommended to insert data in large batches. - MaxDegreeOfParallelism = 4, // Use parallel processing for better performance - }) - { - // Track progress with BatchSent event - bulkCopy.BatchSent += (sender, e) => - { - Console.WriteLine($" Batch sent: {e.RowsWritten} rows written"); - }; + // Example 1: Bulk insert with default options + Console.WriteLine("1. Bulk inserting with InsertBinaryAsync:"); + var columns = new[] { "id", "product_name", "category", "price", "quantity", "sale_date" }; + var data = GenerateSampleData(10000, startId: 1); - // Generate data - var largeData = GenerateSampleData(10000, startId: 6); + var rowsInserted = await client.InsertBinaryAsync(tableName, columns, data); + Console.WriteLine($" Inserted {rowsInserted} rows\n"); - // IMPORTANT: the order of the data in the provided object[] array must match the order of the columns in the table - await bulkCopy.WriteToServerAsync(largeData); - Console.WriteLine($" Total rows inserted: {bulkCopy.RowsWritten}\n"); - } + // Example 2: Bulk insert with custom options (batch size, parallelism) + Console.WriteLine("2. Bulk inserting with custom InsertOptions:"); + var options = new InsertOptions + { + BatchSize = 5000, // Rows per batch + MaxDegreeOfParallelism = 4, // Parallel batch uploads + }; - // Example 2: Bulk insert with specific columns - Console.WriteLine("2. Bulk inserting with specific columns:"); + var moreData = GenerateSampleData(10000, startId: 10001); + rowsInserted = await client.InsertBinaryAsync(tableName, columns, moreData, options); + Console.WriteLine($" Inserted {rowsInserted} rows with custom options\n"); + + // Example 3: Bulk insert with specific columns (others use defaults) + Console.WriteLine("3. Bulk inserting with specific columns:"); var partialTableName = "example_bulk_partial"; - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {partialTableName} ( id UInt64, @@ -72,52 +66,46 @@ created_at DateTime DEFAULT now() ORDER BY (id) "); - using (var bulkCopy = new ClickHouseBulkCopy(connection) - { - DestinationTableName = partialTableName, - ColumnNames = new[] { "id", "name" }, // Only specify these columns, others use defaults - }) + var partialColumns = new[] { "id", "name" }; + var partialData = new List { - var partialData = new List - { - new object[] { 1UL, "Item 1" }, - new object[] { 2UL, "Item 2" }, - new object[] { 3UL, "Item 3" }, - }; + new object[] { 1UL, "Item 1" }, + new object[] { 2UL, "Item 2" }, + new object[] { 3UL, "Item 3" }, + }; - await bulkCopy.WriteToServerAsync(partialData); - Console.WriteLine($" Inserted {bulkCopy.RowsWritten} rows with partial columns\n"); - } + rowsInserted = await client.InsertBinaryAsync(partialTableName, partialColumns, partialData); + Console.WriteLine($" Inserted {rowsInserted} rows with partial columns\n"); // Query and display sample results Console.WriteLine("Sample data from main table:"); - using (var reader = await connection.ExecuteReaderAsync($"SELECT * FROM {tableName} ORDER BY id LIMIT 5")) + using (var reader = await client.ExecuteReaderAsync($"SELECT * FROM {tableName} ORDER BY id LIMIT 5")) { Console.WriteLine("ID\tProduct Name\t\tCategory\tPrice\t\tQuantity"); Console.WriteLine("--\t------------\t\t--------\t-----\t\t--------"); while (reader.Read()) { - var id = reader.GetFieldValue(0); + var id = reader.GetFieldValue(0); var productName = reader.GetString(1); var category = reader.GetString(2); var price = reader.GetFloat(3); - var quantity = reader.GetFieldValue(4); + var quantity = reader.GetFieldValue(4); Console.WriteLine($"{id}\t{productName,-20}\t{category,-15}\t${price,-10:F2}\t{quantity}"); } } // Get total row counts - var totalCount = await connection.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); + var totalCount = await client.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); Console.WriteLine($"\nTotal rows in {tableName}: {totalCount}"); - var partialCount = await connection.ExecuteScalarAsync($"SELECT count() FROM {partialTableName}"); + var partialCount = await client.ExecuteScalarAsync($"SELECT count() FROM {partialTableName}"); Console.WriteLine($"Total rows in {partialTableName}: {partialCount}"); // Clean up - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {partialTableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {partialTableName}"); Console.WriteLine($"\nTables dropped"); } diff --git a/examples/Insert/Insert_003_AsyncInsert.cs b/examples/Insert/Insert_003_AsyncInsert.cs index 9498d1ae..bfc393ee 100644 --- a/examples/Insert/Insert_003_AsyncInsert.cs +++ b/examples/Insert/Insert_003_AsyncInsert.cs @@ -69,7 +69,7 @@ public static async Task Run() /// private static async Task Example1_AsyncInsertWithWait() { - // Configure async inserts at connection level. + // Configure async inserts at client level. // You can also set these directly in the connection string using the "set_" prefix: // // "Host=localhost;set_async_insert=1;set_wait_for_async_insert=1;set_async_insert_busy_timeout_ms=1000" @@ -80,13 +80,12 @@ private static async Task Example1_AsyncInsertWithWait() settings.CustomSettings["async_insert_max_data_size"] = 1_000_000; settings.CustomSettings["async_insert_busy_timeout_ms"] = 1000; - using var connection = new ClickHouseConnection(settings); - await connection.OpenAsync(); + using var client = new ClickHouseClient(settings); var tableName = "example_async_insert_wait"; // Create table - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE OR REPLACE TABLE {tableName} (id Int32, data String) ENGINE MergeTree @@ -103,8 +102,6 @@ ORDER BY id var tasks = Enumerable.Range(0, concurrentInserts).Select(async batchIndex => { - using var command = connection.CreateCommand(); - // Build VALUES clause for batch insert var values = string.Join(",\n", Enumerable.Range(0, rowsPerInsert).Select(_ => @@ -114,11 +111,9 @@ ORDER BY id return $"({id}, '{data}')"; })); - command.CommandText = $"INSERT INTO {tableName} (id, data) VALUES {values}"; - try { - await command.ExecuteNonQueryAsync(); + await client.ExecuteNonQueryAsync($"INSERT INTO {tableName} (id, data) VALUES {values}"); } catch (Exception ex) { @@ -131,11 +126,11 @@ ORDER BY id Console.WriteLine($" {concurrentInserts} concurrent inserts ({rowsPerInsert} rows each) completed"); // Verify data - var count = await connection.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); + var count = await client.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); Console.WriteLine($" Total rows in table: {count} (expected: {concurrentInserts * rowsPerInsert})"); // Cleanup - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } /// @@ -161,13 +156,12 @@ private static async Task Example2_AsyncInsertWithoutWait() settings.CustomSettings["async_insert_max_data_size"] = 1_000_000; settings.CustomSettings["async_insert_busy_timeout_ms"] = 1000; - using var connection = new ClickHouseConnection(settings); - await connection.OpenAsync(); + using var client = new ClickHouseClient(settings); var tableName = "example_async_insert_nowait"; // Create table - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE OR REPLACE TABLE {tableName} (id Int32, name String) ENGINE MergeTree @@ -193,16 +187,13 @@ ORDER BY id var batchSize = random.Next(10, 100); var startId = rowsSent; - using var command = connection.CreateCommand(); var values = string.Join(",\n", Enumerable.Range(0, batchSize).Select(i => $"({startId + i}, 'Name {startId + i}')")); - command.CommandText = $"INSERT INTO {tableName} (id, name) VALUES {values}"; - try { - await command.ExecuteNonQueryAsync(); + await client.ExecuteNonQueryAsync($"INSERT INTO {tableName} (id, name) VALUES {values}"); Interlocked.Add(ref rowsSent, batchSize); Interlocked.Increment(ref insertCount); @@ -228,7 +219,7 @@ ORDER BY id try { - var written = await connection.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); + var written = await client.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); Console.WriteLine($" >> Status: {rowsSent} rows sent, {written} rows written to table"); } catch (Exception ex) @@ -250,10 +241,10 @@ ORDER BY id try { await monitorTask; } catch (OperationCanceledException) { } // Final count - var finalCount = await connection.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); + var finalCount = await client.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); Console.WriteLine($"\n Final: {rowsSent} rows sent, {finalCount} rows written"); // Cleanup - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); } } diff --git a/examples/Insert/Insert_004_RawStreamInsert.cs b/examples/Insert/Insert_004_RawStreamInsert.cs index 9e8c476b..42a8bd3a 100644 --- a/examples/Insert/Insert_004_RawStreamInsert.cs +++ b/examples/Insert/Insert_004_RawStreamInsert.cs @@ -13,22 +13,21 @@ public static class RawStreamInsert { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); - - await InsertFromFile(connection); - await InsertFromMemory(connection); + using var client = new ClickHouseClient("Host=localhost"); + + await InsertFromFile(client); + await InsertFromMemory(client); } /// /// Demonstrates inserting data from a CSV file stream. /// - private static async Task InsertFromFile(ClickHouseConnection connection) + private static async Task InsertFromFile(ClickHouseClient client) { var tableName = "example_raw_stream_file"; // Create a test table matching the CSV structure - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( Id UInt64, @@ -52,7 +51,7 @@ ORDER BY (Id) // For older versions, use "CSVWithNames" or use the setting input_format_csv_skip_first_lines await using (var fileStream = File.OpenRead(csvFile)) { - using var response = await connection.InsertRawStreamAsync( + using var response = await client.InsertRawStreamAsync( table: tableName, stream: fileStream, format: "CSV"); @@ -62,7 +61,7 @@ ORDER BY (Id) // Query and display results Console.WriteLine(" Querying inserted data:"); - using (var reader = await connection.ExecuteReaderAsync($"SELECT * FROM {tableName} ORDER BY Id")) + using (var reader = await client.ExecuteReaderAsync($"SELECT * FROM {tableName} ORDER BY Id")) { Console.WriteLine(" ID\tName\t\tValue"); Console.WriteLine(" --\t----\t\t-----"); @@ -77,23 +76,23 @@ ORDER BY (Id) } } - var totalCount = await connection.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); + var totalCount = await client.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); Console.WriteLine($"\n Total rows: {totalCount}"); // Clean up table - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); Console.WriteLine($"\n Table '{tableName}' dropped\n"); } /// /// Demonstrates inserting data from in-memory streams in various formats. /// - private static async Task InsertFromMemory(ClickHouseConnection connection) + private static async Task InsertFromMemory(ClickHouseClient client) { var tableName = "example_raw_stream_memory"; // Create a test table - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( id UInt64, @@ -118,7 +117,7 @@ ORDER BY (id) using (var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) { - using var response = await connection.InsertRawStreamAsync( + using var response = await client.InsertRawStreamAsync( table: tableName, stream: jsonStream, format: "JSONEachRow", @@ -129,7 +128,7 @@ ORDER BY (id) // Query and display all results Console.WriteLine(" Querying all inserted data:"); - using (var reader = await connection.ExecuteReaderAsync($"SELECT * FROM {tableName} ORDER BY id")) + using (var reader = await client.ExecuteReaderAsync($"SELECT * FROM {tableName} ORDER BY id")) { Console.WriteLine(" ID\tProduct\t\t\tPrice\t\tIn Stock"); Console.WriteLine(" --\t-------\t\t\t-----\t\t--------"); @@ -145,11 +144,11 @@ ORDER BY (id) } } - var totalCount = await connection.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); + var totalCount = await client.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); Console.WriteLine($"\n Total rows: {totalCount}"); // Clean up table - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); Console.WriteLine($"\n Table '{tableName}' dropped"); } } diff --git a/examples/Insert/Insert_005_InsertFromSelect.cs b/examples/Insert/Insert_005_InsertFromSelect.cs index b306aeb1..1ed963e7 100644 --- a/examples/Insert/Insert_005_InsertFromSelect.cs +++ b/examples/Insert/Insert_005_InsertFromSelect.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -40,13 +39,12 @@ public static class InsertFromSelect { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); Console.WriteLine("INSERT FROM SELECT Examples\n"); - await BasicCopy(connection); - await TransformAndAggregate(connection); + await BasicCopy(client); + await TransformAndAggregate(client); Console.WriteLine("All INSERT FROM SELECT examples completed!"); } @@ -54,7 +52,7 @@ public static async Task Run() /// /// Basic example: Copy data from one table to another. /// - private static async Task BasicCopy(ClickHouseConnection connection) + private static async Task BasicCopy(ClickHouseClient client) { Console.WriteLine("1. Basic copy from one table to another:"); @@ -62,7 +60,7 @@ private static async Task BasicCopy(ClickHouseConnection connection) var targetTable = "example_insert_select_target"; // Create source table with data - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {sourceTable} ( id UInt64, @@ -73,7 +71,7 @@ value Float32 ORDER BY id "); - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" INSERT INTO {sourceTable} VALUES (1, 'Alpha', 10.5), (2, 'Beta', 20.3), @@ -83,7 +81,7 @@ INSERT INTO {sourceTable} VALUES "); // Create target table with same schema - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {targetTable} ( id UInt64, @@ -95,24 +93,24 @@ ORDER BY id "); // Copy all data from source to target - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" INSERT INTO {targetTable} SELECT * FROM {sourceTable} "); - var count = await connection.ExecuteScalarAsync($"SELECT count() FROM {targetTable}"); + var count = await client.ExecuteScalarAsync($"SELECT count() FROM {targetTable}"); Console.WriteLine($" Copied {count} rows from {sourceTable} to {targetTable}"); // Cleanup - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {sourceTable}"); - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {targetTable}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {sourceTable}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {targetTable}"); Console.WriteLine($" Tables dropped\n"); } /// /// Transform data during insertion with aggregations. /// - private static async Task TransformAndAggregate(ClickHouseConnection connection) + private static async Task TransformAndAggregate(ClickHouseClient client) { Console.WriteLine("2. Transform and aggregate data during insertion:"); @@ -120,7 +118,7 @@ private static async Task TransformAndAggregate(ClickHouseConnection connection) var summaryTable = "example_order_summary"; // Create orders table with sample data - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {ordersTable} ( order_id UInt64, @@ -134,7 +132,7 @@ order_date Date ORDER BY (order_date, order_id) "); - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" INSERT INTO {ordersTable} VALUES (1, 100, 'Widget', 5, 10.00, '2024-01-15'), (2, 100, 'Gadget', 2, 25.00, '2024-01-16'), @@ -145,7 +143,7 @@ INSERT INTO {ordersTable} VALUES "); // Create summary table for aggregated data - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {summaryTable} ( customer_id UInt32, @@ -158,7 +156,7 @@ ORDER BY customer_id "); // Insert aggregated data - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" INSERT INTO {summaryTable} SELECT customer_id, @@ -172,7 +170,7 @@ ORDER BY customer_id // Display results Console.WriteLine(" Customer order summary:"); - using (var reader = await connection.ExecuteReaderAsync($"SELECT * FROM {summaryTable} ORDER BY customer_id")) + using (var reader = await client.ExecuteReaderAsync($"SELECT * FROM {summaryTable} ORDER BY customer_id")) { Console.WriteLine(" Customer ID | Orders | Quantity | Total Spent"); Console.WriteLine(" ------------|--------|----------|------------"); @@ -183,8 +181,8 @@ ORDER BY customer_id } // Cleanup - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {ordersTable}"); - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {summaryTable}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {ordersTable}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {summaryTable}"); Console.WriteLine($"\n Tables dropped\n"); } } diff --git a/examples/Insert/Insert_006_EphemeralColumns.cs b/examples/Insert/Insert_006_EphemeralColumns.cs index 98b790e7..a9ab7ed1 100644 --- a/examples/Insert/Insert_006_EphemeralColumns.cs +++ b/examples/Insert/Insert_006_EphemeralColumns.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -21,8 +20,7 @@ public static class EphemeralColumns { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); Console.WriteLine("Ephemeral Columns Examples\n"); @@ -31,7 +29,7 @@ public static async Task Run() var tableName = "example_ephemeral_derived"; // Create table where multiple columns are derived from ephemeral inputs - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE OR REPLACE TABLE {tableName} ( id UInt64, @@ -51,38 +49,25 @@ ORDER BY (id) Console.WriteLine(" - first_name: String EPHEMERAL"); Console.WriteLine(" - last_name: String EPHEMERAL\n"); - // Insert with ephemeral columns using parameterized query + // Insert with ephemeral columns using InsertBinaryAsync + // Note: Must specify the ephemeral columns explicitly Console.WriteLine(" Inserting data with first_name and last_name:"); - var people = new[] + var rows = new List { - (Id: 1UL, FirstName: "Alice", LastName: "Smith"), - (Id: 2UL, FirstName: "Bob", LastName: "Johnson"), - (Id: 3UL, FirstName: "Carol", LastName: "Williams") + new object[] { 1UL, "Alice", "Smith" }, + new object[] { 2UL, "Bob", "Johnson" }, + new object[] { 3UL, "Carol", "Williams" } }; - using (var command = connection.CreateCommand()) - { - command.CommandText = $@" - INSERT INTO {tableName} (id, first_name, last_name) - VALUES ({{id:UInt64}}, {{first_name:String}}, {{last_name:String}})"; - - foreach (var person in people) - { - command.Parameters.Clear(); - command.AddParameter("id", person.Id); - command.AddParameter("first_name", person.FirstName); - command.AddParameter("last_name", person.LastName); - await command.ExecuteNonQueryAsync(); - Console.WriteLine($" - Inserted id={person.Id}, first_name='{person.FirstName}', last_name='{person.LastName}'"); - } - } + var columns = new[] { "id", "first_name", "last_name" }; + await client.InsertBinaryAsync(tableName, columns, rows); - Console.WriteLine(); + Console.WriteLine(" - Inserted 3 rows with ephemeral columns\n"); // Query results - derived columns are stored, ephemeral columns are not Console.WriteLine(" Query results (derived columns are stored, ephemeral are not):"); - using (var reader = await connection.ExecuteReaderAsync($"SELECT * FROM {tableName} ORDER BY id")) + using (var reader = await client.ExecuteReaderAsync($"SELECT * FROM {tableName} ORDER BY id")) { Console.WriteLine(" ID\tFull Name\t\tName Length"); Console.WriteLine(" --\t---------\t\t-----------"); @@ -93,7 +78,7 @@ ORDER BY (id) } // Cleanup - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); Console.WriteLine($"\n Table dropped\n"); Console.WriteLine("All ephemeral column examples completed!"); diff --git a/examples/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs b/examples/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs index f815180e..0eb61049 100644 --- a/examples/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs +++ b/examples/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -8,7 +7,9 @@ namespace ClickHouse.Driver.Examples; /// /// ReplacingMergeTree automatically deduplicates rows with the same sorting key (ORDER BY), /// keeping only the row with the highest version. This enables performant "upsert" behavior. -/// +/// +/// Note that FINAL incurs a performance penalty, see the blog here for tips on managing that: https://clickhouse.com/blog/clickhouse-postgresql-change-data-capture-cdc-part-1#final-performance +/// /// Other specialized MergeTree variants exist as well, see https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/ /// /// Key concepts: @@ -23,33 +24,32 @@ public static class UpsertsWithReplacingMergeTree public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); - await SetupTable(connection); + await SetupTable(client); - await InsertInitialData(connection); + await InsertInitialData(client); - await PerformUpserts(connection); + await PerformUpserts(client); - await DemonstrateSoftDeletes(connection); + await DemonstrateSoftDeletes(client); - await ShowQueryingStrategies(connection); + await ShowQueryingStrategies(client); - await ForceMergeAndVerify(connection); + await ForceMergeAndVerify(client); - await Cleanup(connection); + await Cleanup(client); } /// /// Creates a ReplacingMergeTree table with version and deleted columns. /// - private static async Task SetupTable(ClickHouseConnection connection) + private static async Task SetupTable(ClickHouseClient client) { Console.WriteLine("1. Creating ReplacingMergeTree table with version and deleted columns:"); - await connection.ExecuteStatementAsync($@"DROP TABLE IF EXISTS {TableName}"); - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS {TableName}"); + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {TableName} ( user_id UInt64, email String, @@ -72,57 +72,42 @@ ORDER BY (user_id) /// /// Inserts initial user records. /// - private static async Task InsertInitialData(ClickHouseConnection connection) + private static async Task InsertInitialData(ClickHouseClient client) { Console.WriteLine("2. Inserting initial user records:"); - using var command = connection.CreateCommand(); - command.CommandText = $@" - INSERT INTO {TableName} (user_id, email, name, status, version) - VALUES - ({{id1:UInt64}}, {{email1:String}}, {{name1:String}}, {{status1:String}}, {{ver1:UInt64}}), - ({{id2:UInt64}}, {{email2:String}}, {{name2:String}}, {{status2:String}}, {{ver2:UInt64}}), - ({{id3:UInt64}}, {{email3:String}}, {{name3:String}}, {{status3:String}}, {{ver3:UInt64}}) - "; - - command.AddParameter("id1", 1UL); - command.AddParameter("email1", "alice@example.com"); - command.AddParameter("name1", "Alice"); - command.AddParameter("status1", "active"); - command.AddParameter("ver1", 1UL); - - command.AddParameter("id2", 2UL); - command.AddParameter("email2", "bob@example.com"); - command.AddParameter("name2", "Bob"); - command.AddParameter("status2", "active"); - command.AddParameter("ver2", 1UL); - - command.AddParameter("id3", 3UL); - command.AddParameter("email3", "carol@example.com"); - command.AddParameter("name3", "Carol"); - command.AddParameter("status3", "pending"); - command.AddParameter("ver3", 1UL); - - await command.ExecuteNonQueryAsync(); + var rows = new List + { + new object[] { 1UL, "alice@example.com", "Alice", "active", 1UL }, + new object[] { 2UL, "bob@example.com", "Bob", "active", 1UL }, + new object[] { 3UL, "carol@example.com", "Carol", "pending", 1UL } + }; + + var columns = new[] { "user_id", "email", "name", "status", "version" }; + await client.InsertBinaryAsync(TableName, columns, rows); + Console.WriteLine(" Inserted 3 users (Alice, Bob, Carol) with version=1\n"); } /// /// Demonstrates the upsert pattern: insert a new row with higher version to "update". /// - private static async Task PerformUpserts(ClickHouseConnection connection) + private static async Task PerformUpserts(ClickHouseClient client) { Console.WriteLine("3. Performing upserts (updates via insert with higher version):"); // Update Alice's email by inserting a new row with version=2 Console.WriteLine(" Updating Alice's email (user_id=1) by inserting version=2..."); - await connection.ExecuteStatementAsync($@" - INSERT INTO {TableName} (user_id, email, name, status, version) - VALUES (1, 'alice.smith@example.com', 'Alice Smith', 'active', 2) - "); + + var rows = new List + { + new object[] { 1UL, "alice.smith@example.com", "Alice Smith", "active", 2UL } + }; + var columns = new[] { "user_id", "email", "name", "status", "version" }; + await client.InsertBinaryAsync(TableName, columns, rows); // Show that both versions exist before merge - var rowCount = await connection.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + var rowCount = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); Console.WriteLine($"\n Total rows in table (before merge): {rowCount}"); Console.WriteLine(" Note: Both old and new versions coexist until background merge runs\n"); } @@ -130,16 +115,19 @@ await connection.ExecuteStatementAsync($@" /// /// Demonstrates soft deletes using the deleted column. /// - private static async Task DemonstrateSoftDeletes(ClickHouseConnection connection) + private static async Task DemonstrateSoftDeletes(ClickHouseClient client) { Console.WriteLine("4. Performing soft delete:"); // Delete Carol by inserting a row with deleted=1 Console.WriteLine(" Deleting Carol (user_id=3) by inserting version=2 with deleted=1..."); - await connection.ExecuteStatementAsync($@" - INSERT INTO {TableName} (user_id, email, name, status, version, deleted) - VALUES (3, 'carol@example.com', 'Carol', 'pending', 2, 1) - "); + + var rows = new List + { + new object[] { 3UL, "carol@example.com", "Carol", "pending", 2UL, (byte)1 } + }; + var columns = new[] { "user_id", "email", "name", "status", "version", "deleted" }; + await client.InsertBinaryAsync(TableName, columns, rows); Console.WriteLine(" Soft delete inserted. Row will be removed during next merge.\n"); } @@ -147,13 +135,13 @@ await connection.ExecuteStatementAsync($@" /// /// Shows different querying strategies with and without FINAL. /// - private static async Task ShowQueryingStrategies(ClickHouseConnection connection) + private static async Task ShowQueryingStrategies(ClickHouseClient client) { Console.WriteLine("5. Querying strategies:"); // Query WITHOUT FINAL - shows all versions Console.WriteLine("\n a) Query WITHOUT FINAL (shows all row versions):"); - using (var reader = await connection.ExecuteReaderAsync( + using (var reader = await client.ExecuteReaderAsync( $"SELECT user_id, email, name, status, version, deleted FROM {TableName} ORDER BY user_id, version")) { Console.WriteLine(" user_id | email | name | status | ver | del"); @@ -172,7 +160,7 @@ private static async Task ShowQueryingStrategies(ClickHouseConnection connection // Query WITH FINAL - shows deduplicated view Console.WriteLine("\n b) Query WITH FINAL (deduplicated, deleted rows removed):"); - using (var reader = await connection.ExecuteReaderAsync( + using (var reader = await client.ExecuteReaderAsync( $"SELECT user_id, email, name, status, version FROM {TableName} FINAL ORDER BY user_id")) { Console.WriteLine(" user_id | email | name | status | ver"); @@ -194,17 +182,17 @@ private static async Task ShowQueryingStrategies(ClickHouseConnection connection /// /// Forces a merge to physically collapse duplicates. /// - private static async Task ForceMergeAndVerify(ClickHouseConnection connection) + private static async Task ForceMergeAndVerify(ClickHouseClient client) { Console.WriteLine("6. Forcing merge to physically deduplicate:"); - var beforeCount = await connection.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + var beforeCount = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); Console.WriteLine($" Rows before OPTIMIZE: {beforeCount}"); - await connection.ExecuteStatementAsync($"OPTIMIZE TABLE {TableName} FINAL"); + await client.ExecuteNonQueryAsync($"OPTIMIZE TABLE {TableName} FINAL"); Console.WriteLine(" Executed: OPTIMIZE TABLE ... FINAL"); - var afterCount = await connection.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + var afterCount = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); Console.WriteLine($" Rows after OPTIMIZE: {afterCount}"); Console.WriteLine("\n After merge, the table physically contains only the latest versions."); @@ -212,9 +200,9 @@ private static async Task ForceMergeAndVerify(ClickHouseConnection connection) } - private static async Task Cleanup(ClickHouseConnection connection) + private static async Task Cleanup(ClickHouseClient client) { - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {TableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {TableName}"); Console.WriteLine($"Table '{TableName}' dropped"); } } diff --git a/examples/ORM/ORM_001_Dapper.cs b/examples/ORM/ORM_001_Dapper.cs index 47295d92..c8a5d2dc 100644 --- a/examples/ORM/ORM_001_Dapper.cs +++ b/examples/ORM/ORM_001_Dapper.cs @@ -12,6 +12,11 @@ namespace ClickHouse.Driver.Examples; /// Demonstrates using Dapper ORM with the ClickHouse driver. /// Dapper provides a simple object mapping layer over ADO.NET connections. /// +/// Connection Lifetime Pattern: +/// - Use ClickHouseDataSource as a singleton (register in DI, or create once and reuse) +/// - Create short-lived connections per operation using dataSource.CreateConnection() +/// - The DataSource manages connection pooling internally +/// /// NOTE: Dapper's @parameter syntax does NOT work with ClickHouse's {param:Type} syntax. /// The following will NOT work: /// connection.QueryAsync<string>("SELECT {p1:Int32}", new { p1 = 42 }); @@ -29,7 +34,13 @@ static DapperExample() public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); + // Create a DataSource - in a real app, this would be a singleton (register in DI) + // The DataSource manages HttpClient pooling internally + var dataSource = new ClickHouseDataSource("Host=localhost"); + + // Create a connection from the DataSource + // Connections are lightweight - create them per operation + await using var connection = dataSource.CreateConnection(); await connection.OpenAsync(); await SetupTable(connection); diff --git a/examples/README.md b/examples/README.md index 5c494edc..5d72c163 100644 --- a/examples/README.md +++ b/examples/README.md @@ -34,7 +34,7 @@ If something is missing, or you found a mistake in one of these examples, please ### Inserting Data - [Insert_001_SimpleDataInsert.cs](Insert/Insert_001_SimpleDataInsert.cs) - Basic data insertion using parameterized queries -- [Insert_002_BulkInsert.cs](Insert/Insert_002_BulkInsert.cs) - High-performance bulk data insertion using `ClickHouseBulkCopy` +- [Insert_002_BulkInsert.cs](Insert/Insert_002_BulkInsert.cs) - High-performance bulk data insertion using `InsertBinaryAsync()` - [Insert_003_AsyncInsert.cs](Insert/Insert_003_AsyncInsert.cs) - Server-side batching with async inserts for high-concurrency workloads - [Insert_004_RawStreamInsert.cs](Insert/Insert_004_RawStreamInsert.cs) - Inserting raw data streams from files or memory (CSV, JSON, Parquet, etc.) - [Insert_005_InsertFromSelect.cs](Insert/Insert_005_InsertFromSelect.cs) - Using INSERT FROM SELECT for ETL, data transformation, and loading from external sources (S3, URL, remote servers) diff --git a/examples/Select/Select_001_BasicSelect.cs b/examples/Select/Select_001_BasicSelect.cs index 8a6ad0dc..522df48f 100644 --- a/examples/Select/Select_001_BasicSelect.cs +++ b/examples/Select/Select_001_BasicSelect.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -11,13 +10,12 @@ public static class BasicSelect { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); var tableName = "example_select_basic"; // Create and populate a test table - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( id UInt64, @@ -31,23 +29,25 @@ is_active Boolean ORDER BY (id) "); - // Insert sample data - await connection.ExecuteStatementAsync($@" - INSERT INTO {tableName} VALUES - (1, 'Alice Johnson', 'Engineering', 95000, '2020-01-15', true), - (2, 'Bob Smith', 'Sales', 75000, '2019-06-20', true), - (3, 'Carol White', 'Engineering', 105000, '2018-03-10', true), - (4, 'David Brown', 'Marketing', 68000, '2021-09-05', true), - (5, 'Eve Davis', 'Engineering', 88000, '2020-11-12', false), - (6, 'Frank Miller', 'Sales', 82000, '2019-02-28', true), - (7, 'Grace Lee', 'Marketing', 71000, '2022-01-08', true) - "); + // Insert sample data using InsertBinaryAsync + var rows = new List + { + new object[] { 1UL, "Alice Johnson", "Engineering", 95000f, new DateOnly(2020, 1, 15), true }, + new object[] { 2UL, "Bob Smith", "Sales", 75000f, new DateOnly(2019, 6, 20), true }, + new object[] { 3UL, "Carol White", "Engineering", 105000f, new DateOnly(2018, 3, 10), true }, + new object[] { 4UL, "David Brown", "Marketing", 68000f, new DateOnly(2021, 9, 5), true }, + new object[] { 5UL, "Eve Davis", "Engineering", 88000f, new DateOnly(2020, 11, 12), false }, + new object[] { 6UL, "Frank Miller", "Sales", 82000f, new DateOnly(2019, 2, 28), true }, + new object[] { 7UL, "Grace Lee", "Marketing", 71000f, new DateOnly(2022, 1, 8), true } + }; + var columns = new[] { "id", "name", "department", "salary", "hire_date", "is_active" }; + await client.InsertBinaryAsync(tableName, columns, rows); Console.WriteLine($"Created and populated table '{tableName}'\n"); // Example 1: SELECT with WHERE clause Console.WriteLine("\n1. SELECT with WHERE clause (Engineering department only):"); - using (var reader = await connection.ExecuteReaderAsync( + using (var reader = await client.ExecuteReaderAsync( $"SELECT name, salary FROM {tableName} WHERE department = 'Engineering' ORDER BY salary DESC")) { Console.WriteLine("Name\t\t\tSalary"); @@ -62,7 +62,7 @@ INSERT INTO {tableName} VALUES // Example 2: SELECT with aggregations Console.WriteLine("\n2. SELECT with aggregations (average salary by department):"); - using (var reader = await connection.ExecuteReaderAsync($@" + using (var reader = await client.ExecuteReaderAsync($@" SELECT department, count() as employee_count, @@ -86,13 +86,13 @@ ORDER BY avg_salary DESC // Example 3: Using ExecuteScalarAsync for single value Console.WriteLine("\n3. Using ExecuteScalarAsync for single value:"); - var totalEmployees = await connection.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); + var totalEmployees = await client.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); Console.WriteLine($" Total employees: {totalEmployees}"); // Example 4: Reading data with GetFieldValue - Console.WriteLine("\n7. Using GetFieldValue for type-safe reading:"); - using (var reader = await connection.ExecuteReaderAsync( + Console.WriteLine("\n4. Using GetFieldValue for type-safe reading:"); + using (var reader = await client.ExecuteReaderAsync( $"SELECT id, name, salary FROM {tableName} WHERE id = 1")) { if (reader.Read()) @@ -105,7 +105,7 @@ ORDER BY avg_salary DESC } // Clean up - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); Console.WriteLine($"\nTable '{tableName}' dropped"); } } diff --git a/examples/Select/Select_002_SelectMetadata.cs b/examples/Select/Select_002_SelectMetadata.cs index 11d671d0..fcd92380 100644 --- a/examples/Select/Select_002_SelectMetadata.cs +++ b/examples/Select/Select_002_SelectMetadata.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -10,13 +9,12 @@ public static class SelectMetadata { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); var tableName = "example_formats"; // Create and populate a test table - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( id UInt32, @@ -30,16 +28,18 @@ ORDER BY (id) Console.WriteLine($"Created table '{tableName}'\n"); // Insert sample data - await connection.ExecuteStatementAsync($@" - INSERT INTO {tableName} (id, name, values) - VALUES (1, 'Example', [10, 20, 30]) - "); + var rows = new List + { + new object[] { 1u, "Example", new uint[] { 10, 20, 30 } } + }; + var columns = new[] { "id", "name", "values" }; + await client.InsertBinaryAsync(tableName, columns, rows); Console.WriteLine("Inserted sample data\n"); // Example 1: Reading data field by field Console.WriteLine("\n1. Column metadata:"); - using (var reader = await connection.ExecuteReaderAsync($"SELECT * FROM {tableName} LIMIT 1")) + using (var reader = await client.ExecuteReaderAsync($"SELECT * FROM {tableName} LIMIT 1")) { if (reader.Read()) { @@ -55,7 +55,7 @@ await connection.ExecuteStatementAsync($@" } // Clean up - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); Console.WriteLine($"\nTable '{tableName}' dropped"); } } diff --git a/examples/Select/Select_003_SelectWithParameterBinding.cs b/examples/Select/Select_003_SelectWithParameterBinding.cs index 650dc459..c8b3d82e 100644 --- a/examples/Select/Select_003_SelectWithParameterBinding.cs +++ b/examples/Select/Select_003_SelectWithParameterBinding.cs @@ -1,4 +1,4 @@ -using ClickHouse.Driver.ADO; +using ClickHouse.Driver.ADO.Parameters; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -11,13 +11,12 @@ public static class SelectWithParameterBinding { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); var tableName = "example_parameter_binding"; // Create and populate a test table - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( id UInt64, @@ -32,34 +31,35 @@ score Float32 ORDER BY (id) "); - // Insert sample data - await connection.ExecuteStatementAsync($@" - INSERT INTO {tableName} VALUES - (1, 'alice', 'alice@example.com', 28, 'USA', '2020-01-15', 95.5), - (2, 'bob', 'bob@example.com', 35, 'UK', '2019-06-20', 87.3), - (3, 'carol', 'carol@example.com', 42, 'USA', '2018-03-10', 92.1), - (4, 'david', 'david@example.com', 29, 'Canada', '2021-09-05', 88.9), - (5, 'eve', 'eve@example.com', 31, 'USA', '2020-11-12', 91.7), - (6, 'frank', 'frank@example.com', 45, 'UK', '2019-02-28', 79.8), - (7, 'grace', 'grace@example.com', 26, 'Canada', '2022-01-08', 94.2) - "); + // Insert sample data using InsertBinaryAsync + var rows = new List + { + new object[] { 1UL, "alice", "alice@example.com", (byte)28, "USA", new DateOnly(2020, 1, 15), 95.5f }, + new object[] { 2UL, "bob", "bob@example.com", (byte)35, "UK", new DateOnly(2019, 6, 20), 87.3f }, + new object[] { 3UL, "carol", "carol@example.com", (byte)42, "USA", new DateOnly(2018, 3, 10), 92.1f }, + new object[] { 4UL, "david", "david@example.com", (byte)29, "Canada", new DateOnly(2021, 9, 5), 88.9f }, + new object[] { 5UL, "eve", "eve@example.com", (byte)31, "USA", new DateOnly(2020, 11, 12), 91.7f }, + new object[] { 6UL, "frank", "frank@example.com", (byte)45, "UK", new DateOnly(2019, 2, 28), 79.8f }, + new object[] { 7UL, "grace", "grace@example.com", (byte)26, "Canada", new DateOnly(2022, 1, 8), 94.2f } + }; + var columns = new[] { "id", "username", "email", "age", "country", "registration_date", "score" }; + await client.InsertBinaryAsync(tableName, columns, rows); Console.WriteLine($"Created and populated table '{tableName}'\n"); // Example 1: Using parameters. The ClickHouse type is parsed from the query, and used to serialize the parameter value appropriately Console.WriteLine("\n1. Parameters with explicit types:"); - using (var command = connection.CreateCommand()) { // The ClickHouse parameter format is {parameter_name:clickhouse_type}. Below, "Date" specifies the type of the parameter - command.CommandText = $@" + var parameters = new ClickHouseParameterCollection(); + parameters.AddParameter("startDate", new DateOnly(2020, 1, 1)); + + using var reader = await client.ExecuteReaderAsync($@" SELECT username, registration_date FROM {tableName} WHERE registration_date >= {{startDate:Date}} - ORDER BY registration_date"; + ORDER BY registration_date", parameters); - command.AddParameter("startDate", new DateTime(2020, 1, 1)); - - using var reader = await command.ExecuteReaderAsync(); Console.WriteLine(" Users registered since 2020:"); while (reader.Read()) { @@ -67,43 +67,38 @@ INSERT INTO {tableName} VALUES } } - // Example 2: Reusing command with different parameter values - Console.WriteLine("\n2. Reusing command with different parameters:"); - using (var command = connection.CreateCommand()) + // Example 2: Querying with different parameter values + Console.WriteLine("\n2. Querying with different parameters:"); { - command.CommandText = $@" - SELECT username, country, score - FROM {tableName} - WHERE country = {{country:String}} - ORDER BY score DESC - LIMIT 1"; - - var countries = new[] { "USA", "UK", "Canada" }; + var countriesQuery = new[] { "USA", "UK", "Canada" }; - foreach (var country in countries) + foreach (var country in countriesQuery) { - command.Parameters.Clear(); - command.AddParameter("country", country); + var parameters = new ClickHouseParameterCollection(); + parameters.AddParameter("country", country); + + var topUser = await client.ExecuteScalarAsync($@" + SELECT username + FROM {tableName} + WHERE country = {{country:String}} + ORDER BY score DESC + LIMIT 1", parameters); - var topUser = await command.ExecuteScalarAsync(); Console.WriteLine($" Top user in {country}: {topUser}"); } } // Example 3: Parameter binding with IN clause using arrays Console.WriteLine("\n3. Parameter binding with IN clause:"); - using (var command = connection.CreateCommand()) { - // Note: For IN clauses with arrays, we need to format them properly - command.CommandText = $@" + var parameters = new ClickHouseParameterCollection(); + parameters.AddParameter("countries", new[] { "USA", "UK" }); + + using var reader = await client.ExecuteReaderAsync($@" SELECT username, country, age FROM {tableName} WHERE country IN ({{countries:Array(String)}}) - ORDER BY age DESC"; - - command.AddParameter("countries", new[] { "USA", "UK" }); - - using var reader = await command.ExecuteReaderAsync(); + ORDER BY age DESC", parameters); Console.WriteLine(" Users from USA or UK:"); while (reader.Read()) @@ -114,18 +109,17 @@ WHERE country IN ({{countries:Array(String)}}) // Example 4: Parameter binding for tuple comparison Console.WriteLine("\n4. Parameter binding with tuple:"); - using (var command = connection.CreateCommand()) { - command.CommandText = $@" + var parameters = new ClickHouseParameterCollection(); + parameters.AddParameter("comparison", Tuple.Create((byte)30, 85.0f)); + + using var reader = await client.ExecuteReaderAsync($@" SELECT username, age, score FROM {tableName} WHERE (age, score) > {{comparison:Tuple(UInt8, Float32)}} ORDER BY age, score - LIMIT 3"; - - command.AddParameter("comparison", Tuple.Create((byte)30, 85.0f)); + LIMIT 3", parameters); - using var reader = await command.ExecuteReaderAsync(); Console.WriteLine(" Users with (age, score) > (30, 85.0):"); while (reader.Read()) { @@ -134,7 +128,7 @@ WHERE country IN ({{countries:Array(String)}}) } // Clean up - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); Console.WriteLine($"\nTable '{tableName}' dropped"); } } diff --git a/examples/Select/Select_004_ExportToFile.cs b/examples/Select/Select_004_ExportToFile.cs index b92d221a..8a926e4e 100644 --- a/examples/Select/Select_004_ExportToFile.cs +++ b/examples/Select/Select_004_ExportToFile.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -12,13 +11,12 @@ public static class ExportToFile { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); var tableName = "example_export"; // Create and populate a test table - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} ( id UInt64, @@ -30,36 +28,35 @@ salary Float32 ORDER BY (id) "); - await connection.ExecuteStatementAsync($@" - INSERT INTO {tableName} VALUES - (1, 'Alice Johnson', 'Engineering', 95000), - (2, 'Bob Smith', 'Sales', 75000), - (3, 'Carol White', 'Engineering', 105000), - (4, 'David Brown', 'Marketing', 68000), - (5, 'Eve Davis', 'Engineering', 88000) - "); + var rows = new List + { + new object[] { 1UL, "Alice Johnson", "Engineering", 95000f }, + new object[] { 2UL, "Bob Smith", "Sales", 75000f }, + new object[] { 3UL, "Carol White", "Engineering", 105000f }, + new object[] { 4UL, "David Brown", "Marketing", 68000f }, + new object[] { 5UL, "Eve Davis", "Engineering", 88000f } + }; + var columns = new[] { "id", "name", "department", "salary" }; + await client.InsertBinaryAsync(tableName, columns, rows); Console.WriteLine($"Created and populated table '{tableName}'\n"); - await ExportToJsonEachRow(connection, tableName); - await ExportToParquetFile(connection, tableName); + await ExportToJsonEachRow(client, tableName); + await ExportToParquetFile(client, tableName); // Clean up - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); Console.WriteLine($"\nTable '{tableName}' dropped"); } /// /// Demonstrates exporting query results as JSONEachRow format to memory. /// - private static async Task ExportToJsonEachRow(ClickHouseConnection connection, string tableName) + private static async Task ExportToJsonEachRow(ClickHouseClient client, string tableName) { Console.WriteLine("1. Export to JSONEachRow (in memory):"); - using var command = connection.CreateCommand(); - command.CommandText = $"SELECT * FROM {tableName} FORMAT JSONEachRow"; - - using var result = await command.ExecuteRawResultAsync(CancellationToken.None); + using var result = await client.ExecuteRawResultAsync($"SELECT * FROM {tableName} FORMAT JSONEachRow"); // Read the entire response as a string var json = await result.ReadAsStringAsync(); @@ -75,7 +72,7 @@ private static async Task ExportToJsonEachRow(ClickHouseConnection connection, s /// /// Demonstrates exporting query results as Parquet format to a file. /// - private static async Task ExportToParquetFile(ClickHouseConnection connection, string tableName) + private static async Task ExportToParquetFile(ClickHouseClient client, string tableName) { Console.WriteLine("2. Export to Parquet file:"); @@ -83,10 +80,7 @@ private static async Task ExportToParquetFile(ClickHouseConnection connection, s try { - using var command = connection.CreateCommand(); - command.CommandText = $"SELECT * FROM {tableName} FORMAT Parquet"; - - using var result = await command.ExecuteRawResultAsync(CancellationToken.None); + using var result = await client.ExecuteRawResultAsync($"SELECT * FROM {tableName} FORMAT Parquet"); // Stream directly to file await using (var fileStream = File.Create(parquetFile)) diff --git a/examples/Tables/Tables_001_CreateTableSingleNode.cs b/examples/Tables/Tables_001_CreateTableSingleNode.cs index 48e4c1ca..7806a11d 100644 --- a/examples/Tables/Tables_001_CreateTableSingleNode.cs +++ b/examples/Tables/Tables_001_CreateTableSingleNode.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -11,15 +10,14 @@ public static class CreateTableSingleNode { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); Console.WriteLine("Creating tables on a single ClickHouse node\n"); // Example 1: Simple MergeTree table Console.WriteLine("1. Creating a simple MergeTree table:"); var tableName1 = "example_simple_mergetree"; - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName1} ( id UInt64, @@ -34,7 +32,7 @@ ORDER BY (id) // Example 2: MergeTree with partition by Console.WriteLine("2. Creating a MergeTree table with partitioning:"); var tableName2 = "example_partitioned_mergetree"; - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName2} ( event_date Date, @@ -51,7 +49,7 @@ ORDER BY (event_date, user_id) // Example 3: Table with default values Console.WriteLine("3. Creating a table with default values:"); var tableName3 = "example_with_defaults"; - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName3} ( id UInt64, @@ -67,7 +65,7 @@ ORDER BY (id) // Example 4: Memory engine table (for temporary data) Console.WriteLine("4. Creating a Memory engine table:"); var tableName4 = "example_memory"; - await connection.ExecuteStatementAsync($@" + await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName4} ( id UInt64, @@ -79,7 +77,7 @@ value String // Verify tables were created Console.WriteLine("Verifying created tables:"); - using (var reader = await connection.ExecuteReaderAsync( + using (var reader = await client.ExecuteReaderAsync( "SELECT name, engine FROM system.tables WHERE name LIKE 'example_%' AND database = currentDatabase() ORDER BY name")) { Console.WriteLine("Table Name\t\t\t\t\tEngine"); @@ -94,10 +92,10 @@ value String // Clean up - drop all created tables Console.WriteLine("\nCleaning up example tables..."); - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName1}"); - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName2}"); - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName3}"); - await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {tableName4}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName1}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName2}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName3}"); + await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName4}"); Console.WriteLine("All example tables dropped"); } } diff --git a/examples/Tables/Tables_002_CreateTableCluster.cs b/examples/Tables/Tables_002_CreateTableCluster.cs index 440cd37f..cf52a88d 100644 --- a/examples/Tables/Tables_002_CreateTableCluster.cs +++ b/examples/Tables/Tables_002_CreateTableCluster.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -12,8 +11,7 @@ public static class CreateTableCluster public static async Task Run() { // For cluster operations, connect to any node in the cluster - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); Console.WriteLine("Creating tables on a ClickHouse cluster\n"); @@ -37,6 +35,6 @@ created_at DateTime DEFAULT now() ORDER BY (id) "; - await connection.ExecuteStatementAsync(clusterTableDDL); + await client.ExecuteNonQueryAsync(clusterTableDDL); } } diff --git a/examples/Tables/Tables_003_CreateTableCloud.cs b/examples/Tables/Tables_003_CreateTableCloud.cs index 8e8fe5e2..8cdd2c2a 100644 --- a/examples/Tables/Tables_003_CreateTableCloud.cs +++ b/examples/Tables/Tables_003_CreateTableCloud.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -20,32 +19,33 @@ public static async Task Run() // Connect to ClickHouse Cloud var connectionString = $"Host={cloudHost};Port=8443;Protocol=https;Username=default;Password={cloudPassword}"; - using var connection = new ClickHouseConnection(connectionString); - await connection.OpenAsync(); + using var client = new ClickHouseClient(connectionString); Console.WriteLine($"Connected to ClickHouse Cloud: {cloudHost}\n"); Console.WriteLine("Creating a simple table (Cloud handles replication):"); var tableName1 = "example_cloud_simple"; - using (var command = connection.CreateCommand()) + // Note: No ENGINE clause needed - Cloud defaults to ReplicatedMergeTree + // No ON CLUSTER needed - Cloud handles distribution automatically + // Use QueryOptions to add custom settings per query + var options = new QueryOptions { - // Note: No ENGINE clause needed - Cloud defaults to ReplicatedMergeTree - // No ON CLUSTER needed - Cloud handles distribution automatically - command.CommandText = $@" - CREATE TABLE IF NOT EXISTS {tableName1} - ( - id UInt64, - name String, - created_at DateTime DEFAULT now() - ) - ORDER BY (id) - "; - - command.CustomSettings.Add("wait_end_of_query", "1"); - - await command.ExecuteNonQueryAsync(); - } + CustomSettings = new Dictionary + { + ["wait_end_of_query"] = "1" + } + }; + + await client.ExecuteNonQueryAsync($@" + CREATE TABLE IF NOT EXISTS {tableName1} + ( + id UInt64, + name String, + created_at DateTime DEFAULT now() + ) + ORDER BY (id) + ", options: options); Console.WriteLine($" Table '{tableName1}' created\n"); } diff --git a/examples/Testing/Testing_001_Testcontainers.cs b/examples/Testing/Testing_001_Testcontainers.cs index 98941e29..b248198d 100644 --- a/examples/Testing/Testing_001_Testcontainers.cs +++ b/examples/Testing/Testing_001_Testcontainers.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Utility; using Testcontainers.ClickHouse; @@ -32,7 +31,7 @@ namespace ClickHouse.Driver.Examples; /// * GitHub Actions: /// - Testcontainers works out of the box with GitHub Actions /// - No special configuration needed -/// +/// /// * Azure DevOps: /// - Use Linux agents or Windows with Docker Desktop /// - May need to set TESTCONTAINERS_RYUK_DISABLED=true in some cases @@ -76,12 +75,11 @@ public static async Task Run() /// private static async Task RunSimulatedTests(string connectionString) { - using var connection = new ClickHouseConnection(connectionString); - await connection.OpenAsync(); + using var client = new ClickHouseClient(connectionString); // Test 1: Create table Console.WriteLine(" [TEST] CreateTable_ShouldSucceed"); - await connection.ExecuteStatementAsync(@" + await client.ExecuteNonQueryAsync(@" CREATE TABLE test_users ( id UInt64, name String, @@ -92,21 +90,19 @@ ORDER BY id "); Console.WriteLine(" PASSED - Table created\n"); - // Test 2: Insert data + // Test 2: Insert data using InsertBinaryAsync Console.WriteLine(" [TEST] InsertData_ShouldSucceed"); - using var insertCommand = connection.CreateCommand(); - insertCommand.CommandText = @" - INSERT INTO test_users (id, name, email) VALUES - ({id:UInt64}, {name:String}, {email:String})"; - insertCommand.AddParameter("id", 1UL); - insertCommand.AddParameter("name", "Alice"); - insertCommand.AddParameter("email", "alice@example.com"); - await insertCommand.ExecuteNonQueryAsync(); + var rows = new List + { + new object[] { 1UL, "Alice", "alice@example.com" } + }; + var columns = new[] { "id", "name", "email" }; + await client.InsertBinaryAsync("test_users", columns, rows); Console.WriteLine(" PASSED - Data inserted\n"); // Test 3: Query data Console.WriteLine(" [TEST] QueryData_ShouldReturnInsertedRow"); - var name = await connection.ExecuteScalarAsync("SELECT name FROM test_users WHERE id = 1"); + var name = await client.ExecuteScalarAsync("SELECT name FROM test_users WHERE id = 1"); Console.WriteLine($"Name: {name}"); if ((string)name! != "Alice") throw new Exception($"Expected 'Alice' but got '{name}'"); @@ -114,14 +110,14 @@ INSERT INTO test_users (id, name, email) VALUES // Test 4: Verify count Console.WriteLine(" [TEST] Count_ShouldBeOne"); - var count = await connection.ExecuteScalarAsync("SELECT count() FROM test_users"); + var count = await client.ExecuteScalarAsync("SELECT count() FROM test_users"); if ((ulong)count! != 1) throw new Exception($"Expected 1 but got {count}"); Console.WriteLine(" PASSED - Count is correct\n"); // Test 5: Drop table (cleanup within test) Console.WriteLine(" [TEST] DropTable_ShouldSucceed"); - await connection.ExecuteStatementAsync("DROP TABLE test_users"); + await client.ExecuteNonQueryAsync("DROP TABLE test_users"); Console.WriteLine(" PASSED - Table dropped"); Console.WriteLine("\n All tests passed!"); diff --git a/examples/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs b/examples/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs index 86860e13..d34b9df9 100644 --- a/examples/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs +++ b/examples/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs @@ -21,22 +21,19 @@ public static async Task Run() .SetMinimumLevel(LogLevel.Trace); // Set to Trace to see HttpClient configuration }); - Console.WriteLine("Creating connection with Trace-level logging enabled...\n"); + Console.WriteLine("Creating client with Trace-level logging enabled...\n"); - // Create connection settings with logger factory + // Create client settings with logger factory var settings = new ClickHouseClientSettings("Host=localhost;Port=8123;Username=default;Database=default") { LoggerFactory = loggerFactory, }; - using var connection = new ClickHouseConnection(settings); - - Console.WriteLine("Opening connection (watch for HttpClient configuration logs)...\n"); - await connection.OpenAsync(); + using var client = new ClickHouseClient(settings); // Perform a simple query Console.WriteLine("\n\nPerforming a simple query..."); - var result = await connection.ExecuteScalarAsync("SELECT 1"); + var result = await client.ExecuteScalarAsync("SELECT 1"); Console.WriteLine($"Query result: {result}"); } } diff --git a/examples/Troubleshooting/Troubleshooting_002_NetworkTracing.cs b/examples/Troubleshooting/Troubleshooting_002_NetworkTracing.cs index c4a5726b..2dbe203b 100644 --- a/examples/Troubleshooting/Troubleshooting_002_NetworkTracing.cs +++ b/examples/Troubleshooting/Troubleshooting_002_NetworkTracing.cs @@ -1,5 +1,6 @@ using ClickHouse.Driver.ADO; using ClickHouse.Driver.Diagnostic; +using ClickHouse.Driver.Utility; using Microsoft.Extensions.Logging; namespace ClickHouse.Driver.Examples; @@ -28,8 +29,6 @@ public static async Task Run() Console.WriteLine("This example demonstrates low-level .NET network tracing."); Console.WriteLine("You will see detailed HTTP, Socket, DNS, and TLS events.\n"); - - // Step 1: Configure a logger factory with Trace level enabled var loggerFactory = LoggerFactory.Create(builder => { @@ -38,7 +37,7 @@ public static async Task Run() .SetMinimumLevel(LogLevel.Trace); // Must be Trace level to see network events }); - // Step 2: Configure ClickHouse connection with EnableDebugMode + // Step 2: Configure ClickHouse client with EnableDebugMode var settings = new ClickHouseClientSettings("Host=localhost;Port=8123;Username=default;Database=default") { LoggerFactory = loggerFactory, @@ -49,24 +48,18 @@ public static async Task Run() try { - using (var connection = new ClickHouseConnection(settings)) - { - // Open connection - you'll see DNS resolution, socket connection, TLS handshake (if HTTPS) - await connection.OpenAsync(); - Console.WriteLine("\n[Application] Connection opened successfully\n"); + using var client = new ClickHouseClient(settings); - // Execute a simple query - you'll see HTTP request/response details - using var command = connection.CreateCommand(); - command.CommandText = "SELECT version(), 'Hello from ClickHouse!' as message"; - using var reader = await command.ExecuteReaderAsync(); + // Execute a simple query - you'll see DNS resolution, socket connection, HTTP request/response details + Console.WriteLine("\n[Application] Executing query...\n"); - if (await reader.ReadAsync()) - { - Console.WriteLine($"\n[Application] ClickHouse version: {reader.GetString(0)}"); - Console.WriteLine($"[Application] Message: {reader.GetString(1)}\n"); - } - } + using var reader = await client.ExecuteReaderAsync("SELECT version(), 'Hello from ClickHouse!' as message"); + if (await reader.ReadAsync()) + { + Console.WriteLine($"\n[Application] ClickHouse version: {reader.GetString(0)}"); + Console.WriteLine($"[Application] Message: {reader.GetString(1)}\n"); + } } catch (Exception ex) { diff --git a/examples/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs b/examples/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs index 2e843872..f4e48095 100644 --- a/examples/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs +++ b/examples/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs @@ -1,4 +1,3 @@ -using ClickHouse.Driver.ADO; using ClickHouse.Driver.Diagnostic; using ClickHouse.Driver.Utility; using OpenTelemetry; @@ -40,28 +39,27 @@ public static async Task Run() Console.WriteLine($"Listening to ActivitySource: {ClickHouseDiagnosticsOptions.ActivitySourceName}"); Console.WriteLine("SQL in traces: enabled\n"); - using var connection = new ClickHouseConnection("Host=localhost"); - await connection.OpenAsync(); + using var client = new ClickHouseClient("Host=localhost"); // Query with results - shows read statistics - await ExecuteQueryWithResults(connection); + await ExecuteQueryWithResults(client); // Insert operation - shows write statistics - await ExecuteInsert(connection); + await ExecuteInsert(client); // Error trace demonstration - await ExecuteWithError(connection); + await ExecuteWithError(client); // Reset to default ClickHouseDiagnosticsOptions.IncludeSqlInActivityTags = false; } - private static async Task ExecuteQueryWithResults(ClickHouseConnection connection) + private static async Task ExecuteQueryWithResults(ClickHouseClient client) { Console.WriteLine("\n1. Query with results (shows read statistics):"); Console.WriteLine(" Trace will include: db.clickhouse.read_rows, db.clickhouse.read_bytes\n"); - using var reader = await connection.ExecuteReaderAsync( + using var reader = await client.ExecuteReaderAsync( "SELECT number, toString(number) as str FROM system.numbers LIMIT 100"); var count = 0; @@ -69,37 +67,37 @@ private static async Task ExecuteQueryWithResults(ClickHouseConnection connectio Console.WriteLine($" Read {count} rows\n"); } - private static async Task ExecuteInsert(ClickHouseConnection connection) + private static async Task ExecuteInsert(ClickHouseClient client) { Console.WriteLine("\n2. Insert operation (shows write statistics):"); Console.WriteLine(" Trace will include: db.clickhouse.written_rows, db.clickhouse.written_bytes\n"); - await connection.ExecuteStatementAsync(@" + await client.ExecuteNonQueryAsync(@" CREATE TABLE IF NOT EXISTS example_otel_trace ( id UInt32, value String ) ENGINE = Memory "); - using var command = connection.CreateCommand(); - command.CommandText = "INSERT INTO example_otel_trace VALUES ({id:UInt32}, {value:String})"; - command.AddParameter("id", 1); - command.AddParameter("value", "test"); - await command.ExecuteNonQueryAsync(); + var rows = new List + { + new object[] { 1u, "test" } + }; + await client.InsertBinaryAsync("example_otel_trace", new[] { "id", "value" }, rows); Console.WriteLine(" Inserted 1 row\n"); - await connection.ExecuteStatementAsync("DROP TABLE IF EXISTS example_otel_trace"); + await client.ExecuteNonQueryAsync("DROP TABLE IF EXISTS example_otel_trace"); } - private static async Task ExecuteWithError(ClickHouseConnection connection) + private static async Task ExecuteWithError(ClickHouseClient client) { Console.WriteLine("\n3. Error trace (intentional error):"); Console.WriteLine(" Trace will show: otel.status_code=ERROR, Activity.StatusDescription will include exception details\n"); try { - await connection.ExecuteScalarAsync("SELECT * FROM non_existent_table_12345"); + await client.ExecuteScalarAsync("SELECT * FROM non_existent_table_12345"); } catch (Exception ex) {