Skip to content

Commit 69e7119

Browse files
GiorgiGiorgi
committed
Migrate appender implementation to data chunks.
1 parent 01f0534 commit 69e7119

9 files changed

+143
-81
lines changed

‎DuckDB.NET.Data/DuckDBAppender.cs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public DuckDBAppenderRow CreateRow()
5656
}
5757

5858
rowCount++;
59-
return new DuckDBAppenderRow(nativeAppender, qualifiedTableName, vectorWriters, rowCount - 1);
59+
return new DuckDBAppenderRow(qualifiedTableName, vectorWriters, rowCount - 1);
6060
}
6161

6262
public void Close()
@@ -99,9 +99,8 @@ private unsafe void InitVectorWriters()
9999
for (long index = 0; index < vectorWriters.LongLength; index++)
100100
{
101101
var vector = NativeMethods.DataChunks.DuckDBDataChunkGetVector(dataChunk, index);
102-
var vectorData = NativeMethods.Vectors.DuckDBVectorGetData(vector);
103102

104-
vectorWriters[index] = VectorDataWriterFactory.CreateWriter(vector, vectorData, logicalTypes[index]);
103+
vectorWriters[index] = VectorDataWriterFactory.CreateWriter(vector, logicalTypes[index]);
105104
}
106105
}
107106

‎DuckDB.NET.Data/DuckDBAppenderRow.cs

Lines changed: 27 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,12 @@ namespace DuckDB.NET.Data;
88
public class DuckDBAppenderRow
99
{
1010
private int columnIndex = 0;
11-
private readonly Native.DuckDBAppender appender;
1211
private readonly string qualifiedTableName;
1312
private readonly VectorDataWriterBase[] vectorWriters;
1413
private readonly ulong rowIndex;
1514

16-
internal DuckDBAppenderRow(Native.DuckDBAppender appender, string qualifiedTableName, VectorDataWriterBase[] vectorWriters, ulong rowIndex)
15+
internal DuckDBAppenderRow(string qualifiedTableName, VectorDataWriterBase[] vectorWriters, ulong rowIndex)
1716
{
18-
this.appender = appender;
1917
this.qualifiedTableName = qualifiedTableName;
2018
this.vectorWriters = vectorWriters;
2119
this.rowIndex = rowIndex;
@@ -29,9 +27,9 @@ public void EndRow()
2927
}
3028
}
3129

32-
public DuckDBAppenderRow AppendNullValue() => Append<int>(null); //Doesn't matter what type T we pass to Append when passing null.
30+
public DuckDBAppenderRow AppendNullValue() => AppendValueInternal<int?>(null); //Doesn't matter what type T we pass to Append when passing null.
3331

34-
public DuckDBAppenderRow AppendValue(bool? value) => AppendValue2(value);
32+
public DuckDBAppenderRow AppendValue(bool? value) => AppendValueInternal(value);
3533

3634
#if NET6_0_OR_GREATER
3735

@@ -40,84 +38,68 @@ public void EndRow()
4038
public DuckDBAppenderRow AppendValue(Span<byte> value) => AppendSpan(value);
4139
#endif
4240

43-
public DuckDBAppenderRow AppendValue(string? value) => AppendValue2(value);
41+
public DuckDBAppenderRow AppendValue(string? value) => AppendValueInternal(value);
4442

45-
public DuckDBAppenderRow AppendValue(decimal? value) => AppendValue2(value);
43+
public DuckDBAppenderRow AppendValue(decimal? value) => AppendValueInternal(value);
4644

47-
public DuckDBAppenderRow AppendValue(Guid? value) => AppendValue2(value);
45+
public DuckDBAppenderRow AppendValue(Guid? value) => AppendValueInternal(value);
4846

49-
public DuckDBAppenderRow AppendValue(BigInteger? value, bool unsigned = false)
50-
{
51-
if (value == null)
52-
{
53-
return AppendNullValue();
54-
}
55-
56-
if (unsigned)
57-
{
58-
Append<DuckDBUHugeInt>(new DuckDBUHugeInt(value.Value));
59-
}
60-
else
61-
{
62-
Append<DuckDBHugeInt>(new DuckDBHugeInt(value.Value));
63-
}
64-
65-
return this;
66-
}
47+
public DuckDBAppenderRow AppendValue(BigInteger? value) => AppendValueInternal(value);
6748

6849
#region Append Signed Int
6950

70-
public DuckDBAppenderRow AppendValue(sbyte? value) => Append(value);
51+
public DuckDBAppenderRow AppendValue(sbyte? value) => AppendValueInternal(value);
7152

72-
public DuckDBAppenderRow AppendValue(short? value) => Append(value);
53+
public DuckDBAppenderRow AppendValue(short? value) => AppendValueInternal(value);
7354

74-
public DuckDBAppenderRow AppendValue(int? value) => Append(value);
55+
public DuckDBAppenderRow AppendValue(int? value) => AppendValueInternal(value);
7556

76-
public DuckDBAppenderRow AppendValue(long? value) => Append(value);
57+
public DuckDBAppenderRow AppendValue(long? value) => AppendValueInternal(value);
7758

7859
#endregion
7960

8061
#region Append Unsigned Int
8162

82-
public DuckDBAppenderRow AppendValue(byte? value) => Append(value);
63+
public DuckDBAppenderRow AppendValue(byte? value) => AppendValueInternal(value);
8364

84-
public DuckDBAppenderRow AppendValue(ushort? value) => Append(value);
65+
public DuckDBAppenderRow AppendValue(ushort? value) => AppendValueInternal(value);
8566

86-
public DuckDBAppenderRow AppendValue(uint? value) => Append(value);
67+
public DuckDBAppenderRow AppendValue(uint? value) => AppendValueInternal(value);
8768

88-
public DuckDBAppenderRow AppendValue(ulong? value) => Append(value);
69+
public DuckDBAppenderRow AppendValue(ulong? value) => AppendValueInternal(value);
8970

9071
#endregion
9172

9273
#region Append Float
9374

94-
public DuckDBAppenderRow AppendValue(float? value) => Append(value);
75+
public DuckDBAppenderRow AppendValue(float? value) => AppendValueInternal(value);
9576

96-
public DuckDBAppenderRow AppendValue(double? value) => Append(value);
77+
public DuckDBAppenderRow AppendValue(double? value) => AppendValueInternal(value);
9778

9879
#endregion
9980

10081
#region Append Temporal
10182
#if NET6_0_OR_GREATER
102-
public DuckDBAppenderRow AppendValue(DateOnly? value) => Append(value == null ? (DuckDBDate?)null : NativeMethods.DateTimeHelpers.DuckDBToDate(value.Value));
83+
public DuckDBAppenderRow AppendValue(DateOnly? value) => AppendValueInternal(value);
10384

104-
public DuckDBAppenderRow AppendValue(TimeOnly? value) => Append(value == null ? (DuckDBTime?)null : NativeMethods.DateTimeHelpers.DuckDBToTime(value.Value));
105-
#else
106-
public DuckDBAppenderRow AppendValue(DuckDBDateOnly? value) => Append(value == null ? (DuckDBDate?)null : NativeMethods.DateTimeHelpers.DuckDBToDate(value.Value));
107-
108-
public DuckDBAppenderRow AppendValue(DuckDBTimeOnly? value) => Append(value == null ? (DuckDBTime?)null : NativeMethods.DateTimeHelpers.DuckDBToTime(value.Value));
85+
public DuckDBAppenderRow AppendValue(TimeOnly? value) => AppendValueInternal(value);
10986
#endif
11087

111-
public DuckDBAppenderRow AppendValue(DateTime? value) => Append(value == null ? (DuckDBTimestampStruct?)null : NativeMethods.DateTimeHelpers.DuckDBToTimestamp(DuckDBTimestamp.FromDateTime(value.Value)));
88+
public DuckDBAppenderRow AppendValue(DuckDBDateOnly? value) => AppendValueInternal(value);
89+
90+
public DuckDBAppenderRow AppendValue(DuckDBTimeOnly? value) => AppendValueInternal(value);
91+
92+
93+
public DuckDBAppenderRow AppendValue(DateTime? value) => AppendValueInternal(value);
11294

11395
public DuckDBAppenderRow AppendValue(TimeSpan? value)
11496
{
115-
return AppendValue2(value);
97+
return AppendValueInternal(value);
11698
}
11799

118100
#endregion
119101

120-
private DuckDBAppenderRow AppendValue2<T>(T? value)
102+
private DuckDBAppenderRow AppendValueInternal<T>(T? value)
121103
{
122104
CheckColumnAccess();
123105

@@ -128,24 +110,6 @@ private DuckDBAppenderRow AppendValue2<T>(T? value)
128110
return this;
129111
}
130112

131-
private DuckDBAppenderRow Append<T>(T? value) where T : unmanaged
132-
{
133-
CheckColumnAccess();
134-
135-
if (value == null)
136-
{
137-
vectorWriters[columnIndex].AppendNull(rowIndex);
138-
}
139-
else
140-
{
141-
vectorWriters[columnIndex].AppendValueInternal(value.Value, rowIndex);
142-
}
143-
144-
columnIndex++;
145-
146-
return this;
147-
}
148-
149113
#if NET6_0_OR_GREATER
150114
private unsafe DuckDBAppenderRow AppendSpan(Span<byte> val)
151115
{

‎DuckDB.NET.Data/Internal/Writer/BooleanVectorDataWriter.cs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
using System;
2+
using DuckDB.NET.Native;
3+
4+
namespace DuckDB.NET.Data.Internal.Writer;
5+
6+
internal sealed unsafe class BooleanVectorDataWriter(IntPtr vector, void* vectorData, DuckDBType columnType) : VectorDataWriterBase(vector, vectorData, columnType)
7+
{
8+
internal override bool AppendBool(bool value, ulong rowIndex) => AppendValueInternal(value, rowIndex);
9+
}

‎DuckDB.NET.Data/Internal/Writer/DateTimeVectorDataWriter.cs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
using System;
2+
using DuckDB.NET.Native;
3+
4+
namespace DuckDB.NET.Data.Internal.Writer;
5+
6+
internal sealed unsafe class DateTimeVectorDataWriter(IntPtr vector, void* vectorData, DuckDBType columnType) : VectorDataWriterBase(vector, vectorData, columnType)
7+
{
8+
internal override bool AppendDateTime(DateTime value, ulong rowIndex) => AppendValueInternal(NativeMethods.DateTimeHelpers.DuckDBToTimestamp(DuckDBTimestamp.FromDateTime(value)), rowIndex);
9+
10+
#if NET6_0_OR_GREATER
11+
internal override bool AppendDateOnly(DateOnly value, ulong rowIndex) => AppendValueInternal(NativeMethods.DateTimeHelpers.DuckDBToDate(value), rowIndex);
12+
13+
internal override bool AppendTimeOnly(TimeOnly value, ulong rowIndex) => AppendValueInternal(NativeMethods.DateTimeHelpers.DuckDBToTime(value), rowIndex);
14+
#endif
15+
16+
internal override bool AppendDateOnly(DuckDBDateOnly value, ulong rowIndex) => AppendValueInternal(NativeMethods.DateTimeHelpers.DuckDBToDate(value), rowIndex);
17+
18+
internal override bool AppendTimeOnly(DuckDBTimeOnly value, ulong rowIndex) => AppendValueInternal(NativeMethods.DateTimeHelpers.DuckDBToTime(value), rowIndex);
19+
}

‎DuckDB.NET.Data/Internal/Writer/DecimalVectorDataWriter.cs

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ internal sealed unsafe class DecimalVectorDataWriter(IntPtr vector, void* vector
99
private readonly byte scale = NativeMethods.LogicalType.DuckDBDecimalScale(logicalType);
1010
private readonly DuckDBType decimalType = NativeMethods.LogicalType.DuckDBDecimalInternalType(logicalType);
1111

12-
public void AppendValue(decimal value, ulong rowIndex)
12+
internal override bool AppendDecimal(decimal value, ulong rowIndex)
1313
{
1414
var power = Math.Pow(10, scale);
1515

@@ -30,10 +30,12 @@ public void AppendValue(decimal value, ulong rowIndex)
3030

3131
var result = BigInteger.Multiply(new BigInteger(integralPart), new BigInteger(power));
3232

33-
result += new BigInteger(decimal.Multiply(fractionalPart, (decimal)power));
34-
33+
result += new BigInteger(decimal.Multiply(fractionalPart, (decimal)power));
34+
3535
AppendValueInternal(new DuckDBHugeInt(result), rowIndex);
3636
break;
3737
}
38+
39+
return true;
3840
}
39-
}
41+
}

‎DuckDB.NET.Data/Internal/Writer/NumericVectorDataWriter.cs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
using System;
2+
using System.Numerics;
3+
using DuckDB.NET.Native;
4+
5+
namespace DuckDB.NET.Data.Internal.Writer;
6+
7+
internal sealed unsafe class NumericVectorDataWriter(IntPtr vector, void* vectorData, DuckDBType columnType) : VectorDataWriterBase(vector, vectorData, columnType)
8+
{
9+
internal override bool AppendNumeric<T>(T value, ulong rowIndex) => AppendValueInternal(value, rowIndex);
10+
11+
internal override bool AppendBigInteger(BigInteger value, ulong rowIndex) => AppendValueInternal<DuckDBHugeInt>(new DuckDBHugeInt(value), rowIndex);
12+
}

‎DuckDB.NET.Data/Internal/Writer/VectorDataWriterBase.cs

Lines changed: 54 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Numerics;
23
using DuckDB.NET.Native;
34

45
namespace DuckDB.NET.Data.Internal.Writer;
@@ -29,24 +30,70 @@ public unsafe void AppendValue<T>(T value, ulong rowIndex)
2930

3031
_ = value switch
3132
{
33+
bool val => AppendBool(val, rowIndex),
34+
35+
sbyte val => AppendNumeric(val, rowIndex),
36+
short val => AppendNumeric(val, rowIndex),
37+
int val => AppendNumeric(val, rowIndex),
38+
long val => AppendNumeric(val, rowIndex),
39+
byte val => AppendNumeric(val, rowIndex),
40+
ushort val => AppendNumeric(val, rowIndex),
41+
uint val => AppendNumeric(val, rowIndex),
42+
ulong val => AppendNumeric(val, rowIndex),
43+
float val => AppendNumeric(val, rowIndex),
44+
double val => AppendNumeric(val, rowIndex),
45+
46+
decimal val => AppendDecimal(val, rowIndex),
47+
BigInteger val => AppendBigInteger(val, rowIndex),
48+
3249
string val => AppendString(val, rowIndex),
3350
Guid val => AppendGuid(val, rowIndex),
51+
DateTime val => AppendDateTime(val, rowIndex),
3452
TimeSpan val => AppendTimeSpan(val, rowIndex),
35-
_ => throw new InvalidOperationException($"Cannot write ${typeof(T).Name} to {columnType} column")
53+
DuckDBDateOnly val => AppendDateOnly(val, rowIndex),
54+
DuckDBTimeOnly val => AppendTimeOnly(val, rowIndex),
55+
#if NET6_0_OR_GREATER
56+
DateOnly val => AppendDateOnly(val, rowIndex),
57+
TimeOnly val => AppendTimeOnly(val, rowIndex),
58+
#endif
59+
_ => ThrowException<T>()
3660
};
3761
}
3862

39-
internal virtual bool AppendTimeSpan(TimeSpan value, ulong rowIndex) => throw new InvalidOperationException($"Cannot write timespan to {columnType} column");
63+
internal virtual bool AppendBool(bool value, ulong rowIndex) => ThrowException<bool>();
64+
65+
internal virtual bool AppendDecimal(decimal value, ulong rowIndex) => ThrowException<decimal>();
66+
67+
internal virtual bool AppendTimeSpan(TimeSpan value, ulong rowIndex) => ThrowException<TimeSpan>();
68+
69+
internal virtual bool AppendGuid(Guid value, ulong rowIndex) => ThrowException<Guid>();
70+
71+
internal virtual bool AppendBlob(byte* value, int length, ulong rowIndex) => ThrowException<byte[]>();
72+
73+
internal virtual bool AppendString(string value, ulong rowIndex) => ThrowException<string>();
4074

41-
internal virtual bool AppendGuid(Guid value, ulong rowIndex) => throw new InvalidOperationException($"Cannot write guid to {columnType} column");
75+
internal virtual bool AppendDateTime(DateTime value, ulong rowIndex) => ThrowException<DateTime>();
4276

43-
internal virtual bool AppendBlob(byte* value, int length, ulong rowIndex) => throw new InvalidOperationException($"Cannot write blob to {columnType} column");
77+
#if NET6_0_OR_GREATER
78+
internal virtual bool AppendDateOnly(DateOnly value, ulong rowIndex) => ThrowException<DateOnly>();
4479

45-
internal virtual bool AppendString(string value, ulong rowIndex) => throw new InvalidOperationException($"Cannot write string to {columnType} column");
80+
internal virtual bool AppendTimeOnly(TimeOnly value, ulong rowIndex) => ThrowException<TimeOnly>();
81+
#endif
4682

47-
internal virtual bool AppendNumeric<T>(T value, ulong rowIndex) where T : unmanaged => throw new InvalidOperationException($"Cannot write {typeof(T).Name} to {columnType} column");
83+
internal virtual bool AppendDateOnly(DuckDBDateOnly value, ulong rowIndex) => ThrowException<DuckDBDateOnly>();
84+
85+
internal virtual bool AppendTimeOnly(DuckDBTimeOnly value, ulong rowIndex) => ThrowException<DuckDBTimeOnly>();
86+
87+
internal virtual bool AppendNumeric<T>(T value, ulong rowIndex) where T : unmanaged => ThrowException<T>();
88+
89+
internal virtual bool AppendBigInteger(BigInteger value, ulong rowIndex) => ThrowException<BigInteger>();
90+
91+
private bool ThrowException<T>()
92+
{
93+
throw new InvalidOperationException($"Cannot write {typeof(T).Name} to {columnType} column");
94+
}
4895

49-
public unsafe bool AppendValueInternal<T>(T value, ulong rowIndex) where T : unmanaged
96+
internal unsafe bool AppendValueInternal<T>(T value, ulong rowIndex) where T : unmanaged
5097
{
5198
((T*)vectorData)[rowIndex] = value;
5299
return true;

‎DuckDB.NET.Data/Internal/Writer/VectorDataWriterFactory.cs

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,31 @@
11
using System;
2+
using DuckDB.NET.Data.Internal.Reader;
23
using DuckDB.NET.Native;
34

45
namespace DuckDB.NET.Data.Internal.Writer
56
{
67
internal static class VectorDataWriterFactory
78
{
8-
public static unsafe VectorDataWriterBase CreateWriter(IntPtr vector, void* dataPointer, DuckDBLogicalType logicalType)
9+
public static unsafe VectorDataWriterBase CreateWriter(IntPtr vector, DuckDBLogicalType logicalType)
910
{
11+
var dataPointer = NativeMethods.Vectors.DuckDBVectorGetData(vector);
1012
var columnType = NativeMethods.LogicalType.DuckDBGetTypeId(logicalType);
13+
1114
return columnType switch
1215
{
1316
DuckDBType.Uuid => new GuidVectorDataWriter(vector, dataPointer, columnType),
17+
DuckDBType.Date => new DateTimeVectorDataWriter(vector, dataPointer, columnType),
18+
DuckDBType.Time => new DateTimeVectorDataWriter(vector, dataPointer, columnType),
19+
DuckDBType.Interval => new IntervalVectorDataWriter(vector, dataPointer, columnType),
20+
DuckDBType.Timestamp => new DateTimeVectorDataWriter(vector, dataPointer, columnType),
21+
22+
DuckDBType.Boolean => new BooleanVectorDataWriter(vector, dataPointer, columnType),
23+
1424
DuckDBType.Blob => new StringVectorDataWriter(vector, dataPointer, columnType),
1525
DuckDBType.Varchar => new StringVectorDataWriter(vector, dataPointer, columnType),
16-
DuckDBType.Interval => new IntervalVectorDataWriter(vector, dataPointer, columnType),
26+
1727
DuckDBType.Decimal => new DecimalVectorDataWriter(vector, dataPointer, logicalType, columnType),
18-
_ => new VectorDataWriterBase(vector, dataPointer, columnType)
28+
_ => new NumericVectorDataWriter(vector, dataPointer, columnType)
1929
};
2030
}
2131
}

‎DuckDB.NET.Test/ManagedAppenderTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public void CommonTypes()
4444
.AppendValue(date.AddDays(i))
4545
.AppendNullValue()
4646
.AppendValue(new BigInteger(ulong.MaxValue) + i)
47-
.AppendValue(new BigInteger(ulong.MaxValue) * 2 + i, true)
47+
.AppendValue(new BigInteger(ulong.MaxValue) * 2 + i)
4848
.AppendValue(i + i / 100m)
4949
.EndRow();
5050
}

0 commit comments

Comments
 (0)