Skip to content

Commit 9b87e3b

Browse files
committed
Merge branch 'develop' into Appender-Data-Chunk
2 parents 83377b6 + f495d72 commit 9b87e3b

25 files changed

+579
-116
lines changed

.github/workflows/NuGet.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ jobs:
99
steps:
1010

1111
- name: Download Artifacts
12-
uses: dawidd6/action-download-artifact@v2
12+
uses: dawidd6/action-download-artifact@v3
1313
with:
1414
github_token: ${{ secrets.GITHUB_TOKEN }}
1515
workflow: ci.yml

.github/workflows/Sonar.yml

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
name: SonarCloud
2+
23
on:
4+
workflow_dispatch:
5+
36
push:
47
branches:
58
- develop
@@ -67,6 +70,7 @@ jobs:
6770
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
6871
shell: pwsh
6972
run: |
70-
./.sonar/scanner/dotnet-sonarscanner begin /k:"Giorgi_DuckDB.NET" /o:"giorgi" /d:sonar.login="${{ secrets.SONAR_TOKEN }}" /d:sonar.host.url="https://sonarcloud.io"
73+
./.sonar/scanner/dotnet-sonarscanner begin /k:"Giorgi_DuckDB.NET" /o:"giorgi" /d:sonar.token="${{ secrets.SONAR_TOKEN }}" /d:sonar.cs.opencover.reportsPaths=DuckDB.NET.Test/coverage.net8.0.opencover.xml /d:sonar.host.url="https://sonarcloud.io"
7174
dotnet build
72-
./.sonar/scanner/dotnet-sonarscanner end /d:sonar.login="${{ secrets.SONAR_TOKEN }}"
75+
dotnet test --configuration Release --verbosity normal --logger GitHubActions /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:BuildType=Full /p:DoesNotReturnAttribute="DoesNotReturnAttribute"
76+
./.sonar/scanner/dotnet-sonarscanner end /d:sonar.token="${{ secrets.SONAR_TOKEN }}"

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,3 +347,4 @@ healthchecksdb
347347

348348
# Backup folder for Package Reference Convert tool in Visual Studio 2017
349349
MigrationBackup/
350+
ConsoleApplication1

DuckDB.NET.Bindings/Bindings.csproj

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@
22

33
<PropertyGroup>
44
<Description>DuckDB Bindings for C#.</Description>
5-
<PackageReleaseNotes>Update to DuckDB 0.10.1.</PackageReleaseNotes>
5+
<PackageReleaseNotes>Update to DuckDB 0.10.3.</PackageReleaseNotes>
66
<RootNamespace>DuckDB.NET.Native</RootNamespace>
77
<RuntimeIdentifiers>win-x64;linux-x64;linux-arm64;osx</RuntimeIdentifiers>
8-
<DuckDbArtifactRoot Condition=" '$(DuckDbArtifactRoot)' == '' ">https://github.com/duckdb/duckdb/releases/download/v0.10.2</DuckDbArtifactRoot>
8+
<DuckDbArtifactRoot Condition=" '$(DuckDbArtifactRoot)' == '' ">https://github.com/duckdb/duckdb/releases/download/v0.10.3</DuckDbArtifactRoot>
99
<SignAssembly>True</SignAssembly>
1010
<AssemblyOriginatorKeyFile>..\keyPair.snk</AssemblyOriginatorKeyFile>
1111
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>

DuckDB.NET.Data/Data.csproj

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@
33
<PropertyGroup>
44
<Description>DuckDB ADO.NET Provider for C#.</Description>
55
<PackageReleaseNotes>
6-
Update to DuckDB 0.10.1.
6+
Update to DuckDB 0.10.3.
77

8-
Added support for reading TimeTz types.
8+
Added support for writing decimal, Guid and DateTimeOffset values when using managed Appender.
99

10-
Added support for reading Array types.
10+
Implemented DuckDBConnection.GetSchema (By @hazzik)
1111
</PackageReleaseNotes>
1212
<SignAssembly>True</SignAssembly>
1313
<AssemblyOriginatorKeyFile>..\keyPair.snk</AssemblyOriginatorKeyFile>

DuckDB.NET.Data/DuckDBCommand.cs

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,15 @@ public class DuckDBCommand : DbCommand
2424
public override bool DesignTimeVisible { get; set; }
2525
public override UpdateRowSource UpdatedRowSource { get; set; }
2626

27+
/// <summary>
28+
/// A flag to determine whether to use streaming mode or not when executing a query. Defaults to false.
29+
/// In streaming mode DuckDB will use less RAM but query execution might be slower. Applies only to queries that return a result-set.
30+
/// </summary>
31+
/// <remarks>
32+
/// Streaming mode uses `duckdb_execute_prepared_streaming` and `duckdb_stream_fetch_chunk`, non-streaming (materialized) mode uses `duckdb_execute_prepared` and `duckdb_result_get_chunk`.
33+
/// </remarks>
34+
public bool UseStreamingMode { get; set; } = false;
35+
2736
private string commandText = string.Empty;
2837

2938
#if NET6_0_OR_GREATER
@@ -67,7 +76,7 @@ public override int ExecuteNonQuery()
6776
{
6877
EnsureConnectionOpen();
6978

70-
var results = PreparedStatement.PrepareMultiple(connection!.NativeConnection, CommandText, parameters);
79+
var results = PreparedStatement.PrepareMultiple(connection!.NativeConnection, CommandText, parameters, UseStreamingMode);
7180

7281
var count = 0;
7382

@@ -102,7 +111,7 @@ protected override DbDataReader ExecuteDbDataReader(CommandBehavior behavior)
102111
{
103112
EnsureConnectionOpen();
104113

105-
var results = PreparedStatement.PrepareMultiple(connection!.NativeConnection, CommandText, parameters);
114+
var results = PreparedStatement.PrepareMultiple(connection!.NativeConnection, CommandText, parameters, UseStreamingMode);
106115

107116
var reader = new DuckDBDataReader(this, results, behavior);
108117

DuckDB.NET.Data/DuckDBConnection.cs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,4 +215,13 @@ public DuckDBConnection Duplicate()
215215

216216
return duplicatedConnection;
217217
}
218+
219+
public override DataTable GetSchema() =>
220+
GetSchema(DbMetaDataCollectionNames.MetaDataCollections);
221+
222+
public override DataTable GetSchema(string collectionName) =>
223+
GetSchema(collectionName, null);
224+
225+
public override DataTable GetSchema(string collectionName, string?[]? restrictionValues) =>
226+
DuckDBSchema.GetSchema(this, collectionName, restrictionValues);
218227
}

DuckDB.NET.Data/DuckDBDataReader.cs

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public class DuckDBDataReader : DbDataReader
2828
private long currentChunkIndex;
2929

3030
private readonly IEnumerator<DuckDBResult> resultEnumerator;
31-
private VectorDataReaderBase[] vectorReaders = Array.Empty<VectorDataReaderBase>();
31+
private VectorDataReaderBase[] vectorReaders = [];
3232

3333
internal DuckDBDataReader(DuckDBCommand command, IEnumerable<DuckDBResult> queryResults, CommandBehavior behavior)
3434
{
@@ -293,23 +293,36 @@ public override DataTable GetSchemaTable()
293293
{
294294
Columns =
295295
{
296-
{ "ColumnOrdinal", typeof(int) },
297-
{ "ColumnName", typeof(string) },
298-
{ "DataType", typeof(Type) },
299-
{ "ColumnSize", typeof(int) },
300-
{ "AllowDBNull", typeof(bool) }
296+
{ SchemaTableColumn.ColumnName, typeof(string) },
297+
{ SchemaTableColumn.ColumnOrdinal, typeof(int) },
298+
{ SchemaTableColumn.ColumnSize, typeof(int) },
299+
{ SchemaTableColumn.NumericPrecision, typeof(byte)},
300+
{ SchemaTableColumn.NumericScale, typeof(byte) },
301+
{ SchemaTableColumn.DataType, typeof(Type) },
302+
{ SchemaTableColumn.AllowDBNull, typeof(bool) }
301303
}
302304
};
303305

304-
var rowData = new object[5];
306+
var rowData = new object[7];
305307

306308
for (var i = 0; i < FieldCount; i++)
307309
{
308-
rowData[0] = i;
309-
rowData[1] = GetName(i);
310-
rowData[2] = GetFieldType(i);
311-
rowData[3] = -1;
312-
rowData[4] = true;
310+
rowData[0] = GetName(i);
311+
rowData[1] = i;
312+
rowData[2] = -1;
313+
rowData[5] = GetFieldType(i);
314+
rowData[6] = true;
315+
316+
if (vectorReaders[i] is DecimalVectorDataReader decimalVectorDataReader)
317+
{
318+
rowData[4] = decimalVectorDataReader.Scale;
319+
rowData[3] = decimalVectorDataReader.Precision;
320+
}
321+
else
322+
{
323+
rowData[3] = rowData[4] = DBNull.Value;
324+
}
325+
313326
table.Rows.Add(rowData);
314327
}
315328

DuckDB.NET.Data/DuckDBSchema.cs

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
using System;
2+
using System.Data;
3+
using System.Data.Common;
4+
using System.Linq;
5+
using System.Text;
6+
7+
namespace DuckDB.NET.Data;
8+
9+
internal static class DuckDBSchema
10+
{
11+
private static readonly string[] TableRestrictions = ["table_catalog", "table_schema", "table_name", "table_type"];
12+
13+
private static readonly string[] ColumnRestrictions = ["table_catalog", "table_schema", "table_name", "column_name"];
14+
15+
private static readonly string[] ForeignKeyRestrictions = ["constraint_catalog", "constraint_schema", "table_name", "constraint_name"];
16+
17+
private static readonly string[] IndexesRestrictions = ["index_catalog", "index_schema", "table_name", "index_name"];
18+
19+
public static DataTable GetSchema(DuckDBConnection connection, string collectionName, string?[]? restrictionValues) =>
20+
collectionName.ToUpperInvariant() switch
21+
{
22+
"METADATACOLLECTIONS" => GetMetaDataCollections(),
23+
"RESTRICTIONS" => GetRestrictions(),
24+
"RESERVEDWORDS" => GetReservedWords(connection),
25+
"TABLES" => GetTables(connection, restrictionValues),
26+
"COLUMNS" => GetColumns(connection, restrictionValues),
27+
"FOREIGNKEYS" => GetForeignKeys(connection, restrictionValues),
28+
"INDEXES" => GetIndexes(connection, restrictionValues),
29+
_ => throw new ArgumentOutOfRangeException(nameof(collectionName), collectionName, "Invalid collection name.")
30+
};
31+
32+
private static DataTable GetMetaDataCollections() =>
33+
new(DbMetaDataCollectionNames.MetaDataCollections)
34+
{
35+
Columns =
36+
{
37+
{ DbMetaDataColumnNames.CollectionName, typeof(string) },
38+
{ DbMetaDataColumnNames.NumberOfRestrictions, typeof(int) },
39+
{ DbMetaDataColumnNames.NumberOfIdentifierParts, typeof(int) }
40+
},
41+
Rows =
42+
{
43+
{ DbMetaDataCollectionNames.MetaDataCollections, 0, 0 },
44+
{ DbMetaDataCollectionNames.Restrictions, 0, 0 },
45+
{ DbMetaDataCollectionNames.ReservedWords, 0, 0 },
46+
{ DuckDbMetaDataCollectionNames.Tables, TableRestrictions.Length, 3 },
47+
{ DuckDbMetaDataCollectionNames.Columns, ColumnRestrictions.Length, 4 },
48+
{ DuckDbMetaDataCollectionNames.ForeignKeys, ForeignKeyRestrictions.Length, 3 },
49+
{ DuckDbMetaDataCollectionNames.Indexes, IndexesRestrictions.Length, 3 },
50+
}
51+
};
52+
53+
private static DataTable GetRestrictions() =>
54+
new(DbMetaDataCollectionNames.Restrictions)
55+
{
56+
Columns =
57+
{
58+
{ "CollectionName", typeof(string) },
59+
{ "RestrictionName", typeof(string) },
60+
{ "RestrictionDefault", typeof(string) },
61+
{ "RestrictionNumber", typeof(int) }
62+
},
63+
Rows =
64+
{
65+
{ DuckDbMetaDataCollectionNames.Tables, "Catalog", "table_catalog", 1 },
66+
{ DuckDbMetaDataCollectionNames.Tables, "Schema", "table_schema", 2 },
67+
{ DuckDbMetaDataCollectionNames.Tables, "Table", "table_name", 3 },
68+
{ DuckDbMetaDataCollectionNames.Tables, "TableType", "table_type", 4 },
69+
70+
{ DuckDbMetaDataCollectionNames.Columns, "Catalog", "table_catalog", 1 },
71+
{ DuckDbMetaDataCollectionNames.Columns, "Schema", "table_schema", 2 },
72+
{ DuckDbMetaDataCollectionNames.Columns, "Table", "table_name", 3 },
73+
{ DuckDbMetaDataCollectionNames.Columns, "Column", "column_name", 4 },
74+
75+
{ DuckDbMetaDataCollectionNames.ForeignKeys, "Catalog", "constraint_catalog", 1 },
76+
{ DuckDbMetaDataCollectionNames.ForeignKeys, "Schema", "constraint_schema", 2 },
77+
{ DuckDbMetaDataCollectionNames.ForeignKeys, "Table", "table_name", 3 },
78+
{ DuckDbMetaDataCollectionNames.ForeignKeys, "Constraint", "constraint_name", 4 },
79+
80+
{ DuckDbMetaDataCollectionNames.Indexes, "Catalog", "constraint_catalog", 1 },
81+
{ DuckDbMetaDataCollectionNames.Indexes, "Schema", "constraint_schema", 2 },
82+
{ DuckDbMetaDataCollectionNames.Indexes, "Table", "table_name", 3 },
83+
{ DuckDbMetaDataCollectionNames.Indexes, "Constraint", "constraint_name", 4 },
84+
},
85+
};
86+
87+
private static DataTable GetReservedWords(DuckDBConnection connection)
88+
{
89+
using var command = connection.CreateCommand();
90+
command.CommandText = "SELECT keyword_name as ReservedWord FROM duckdb_keywords() WHERE keyword_category = 'reserved'";
91+
return GetDataTable(DbMetaDataCollectionNames.ReservedWords, command);
92+
}
93+
94+
private static DataTable GetTables(DuckDBConnection connection, string?[]? restrictionValues)
95+
{
96+
if (restrictionValues?.Length > TableRestrictions.Length)
97+
{
98+
throw new ArgumentException("Too many restrictions", nameof(restrictionValues));
99+
}
100+
101+
const string query = "SELECT table_catalog, table_schema, table_name, table_type FROM information_schema.tables";
102+
103+
using var command = BuildCommand(connection, query, restrictionValues, true, TableRestrictions);
104+
105+
return GetDataTable(DuckDbMetaDataCollectionNames.Tables, command);
106+
}
107+
108+
private static DataTable GetColumns(DuckDBConnection connection, string?[]? restrictionValues)
109+
{
110+
if (restrictionValues?.Length > ColumnRestrictions.Length)
111+
{
112+
throw new ArgumentException("Too many restrictions", nameof(restrictionValues));
113+
}
114+
115+
const string query =
116+
"""
117+
SELECT
118+
table_catalog, table_schema, table_name, column_name,
119+
ordinal_position, column_default, is_nullable, data_type,
120+
character_maximum_length, character_octet_length,
121+
numeric_precision, numeric_precision_radix,
122+
numeric_scale, datetime_precision,
123+
character_set_catalog, character_set_schema, character_set_name, collation_catalog
124+
FROM information_schema.columns
125+
""";
126+
using var command = BuildCommand(connection, query, restrictionValues, true, ColumnRestrictions);
127+
128+
return GetDataTable(DuckDbMetaDataCollectionNames.Columns, command);
129+
}
130+
131+
private static DataTable GetForeignKeys(DuckDBConnection connection, string?[]? restrictionValues)
132+
{
133+
if (restrictionValues?.Length > ForeignKeyRestrictions.Length)
134+
{
135+
throw new ArgumentException("Too many restrictions", nameof(restrictionValues));
136+
}
137+
138+
const string query =
139+
"""
140+
SELECT
141+
constraint_catalog, constraint_schema, constraint_name,
142+
table_catalog, table_schema, table_name, constraint_type,
143+
is_deferrable, initially_deferred
144+
FROM information_schema.table_constraints
145+
WHERE constraint_type = 'FOREIGN KEY'
146+
""";
147+
using var command = BuildCommand(connection, query, restrictionValues, false, ForeignKeyRestrictions);
148+
149+
return GetDataTable(DuckDbMetaDataCollectionNames.ForeignKeys, command);
150+
}
151+
152+
private static DataTable GetIndexes(DuckDBConnection connection, string?[]? restrictionValues)
153+
{
154+
if (restrictionValues?.Length > IndexesRestrictions.Length)
155+
{
156+
throw new ArgumentException("Too many restrictions", nameof(restrictionValues));
157+
}
158+
159+
const string query =
160+
"""
161+
SELECT
162+
database_name as index_catalog,
163+
schema_name as index_schema,
164+
index_name,
165+
table_name,
166+
is_unique,
167+
is_primary
168+
FROM duckdb_indexes()
169+
""";
170+
using var command = BuildCommand(connection, query, restrictionValues, true, IndexesRestrictions);
171+
172+
return GetDataTable(DuckDbMetaDataCollectionNames.Indexes, command);
173+
}
174+
175+
private static DuckDBCommand BuildCommand(DuckDBConnection connection, string query, string?[]? restrictions, bool addWhere, string[]? restrictionNames)
176+
{
177+
var command = connection.CreateCommand();
178+
if (restrictions is not { Length: > 0 } || restrictionNames == null)
179+
{
180+
command.CommandText = query;
181+
return command;
182+
}
183+
184+
var builder = new StringBuilder(query);
185+
foreach (var (name, restriction) in restrictionNames.Zip(restrictions, Tuple.Create))
186+
{
187+
if (restriction?.Length > 0)
188+
{
189+
if (addWhere)
190+
{
191+
builder.Append(" WHERE ");
192+
addWhere = false;
193+
}
194+
else
195+
{
196+
builder.Append(" AND ");
197+
}
198+
199+
builder.Append($"{name} = ${name}");
200+
command.Parameters.Add(new DuckDBParameter(name, restriction));
201+
}
202+
}
203+
204+
command.CommandText = builder.ToString();
205+
return command;
206+
}
207+
208+
private static DataTable GetDataTable(string tableName, DuckDBCommand command)
209+
{
210+
var table = new DataTable(tableName);
211+
try
212+
{
213+
using var reader = command.ExecuteReader();
214+
table.BeginLoadData();
215+
table.Load(reader);
216+
}
217+
finally
218+
{
219+
table.EndLoadData();
220+
}
221+
222+
return table;
223+
}
224+
}

0 commit comments

Comments
 (0)