Skip to content

Commit 9cebb77

Browse files
Add MoneyConverter and VectorConverter support
- Added MoneyConverter with NumberStyles.Currency for SQL Server money/smallmoney types - Supports currency symbols ($, €, £, etc.) - Supports thousands separators (1,234.56) - Supports negative parentheses accounting format ((0.00)) - Supports scientific notation - Added VectorConverter for SQL Server 2025 VECTOR data type - Supports JSON array format: [0.1, 0.2, 0.3] - Supports comma-separated format: 0.1, 0.2, 0.3 - Supports scientific notation in vector elements - Handles large embeddings (tested with 100+ dimensions) - Added comprehensive test coverage for both converters - MoneyConverter: currency symbols, thousands separators, scientific notation - VectorConverter: JSON arrays, comma-separated, scientific notation, edge cases Co-authored-by: Chrissy LeMaire <[email protected]>
1 parent abf7f24 commit 9cebb77

File tree

3 files changed

+276
-0
lines changed

3 files changed

+276
-0
lines changed

project/dbatools.Tests/Csv/TypeConverterTest.cs

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Linq;
23
using Microsoft.VisualStudio.TestTools.UnitTesting;
34
using Dataplat.Dbatools.Csv.TypeConverters;
45

@@ -181,6 +182,184 @@ public void TestDecimalConverterScientificNotation()
181182
Assert.AreEqual(-314m, result);
182183
}
183184

185+
[TestMethod]
186+
public void TestMoneyConverter()
187+
{
188+
var converter = MoneyConverter.Default;
189+
190+
// Test basic decimal values
191+
Assert.IsTrue(converter.TryConvert("123.45", out decimal result));
192+
Assert.AreEqual(123.45m, result);
193+
194+
// Test negative values
195+
Assert.IsTrue(converter.TryConvert("-99.99", out result));
196+
Assert.AreEqual(-99.99m, result);
197+
}
198+
199+
[TestMethod]
200+
public void TestMoneyConverterWithCurrencySymbols()
201+
{
202+
var converter = MoneyConverter.Default;
203+
204+
// Test US dollar sign
205+
Assert.IsTrue(converter.TryConvert("$123.45", out decimal result));
206+
Assert.AreEqual(123.45m, result);
207+
208+
// Test negative with dollar sign
209+
Assert.IsTrue(converter.TryConvert("-$99.99", out result));
210+
Assert.AreEqual(-99.99m, result);
211+
212+
// Test parentheses for negative (accounting format)
213+
Assert.IsTrue(converter.TryConvert("($50.00)", out result));
214+
Assert.AreEqual(-50.00m, result);
215+
}
216+
217+
[TestMethod]
218+
public void TestMoneyConverterWithThousandsSeparator()
219+
{
220+
var converter = MoneyConverter.Default;
221+
222+
// Test with thousands separator
223+
Assert.IsTrue(converter.TryConvert("$1,234.56", out decimal result));
224+
Assert.AreEqual(1234.56m, result);
225+
226+
// Test large number with currency
227+
Assert.IsTrue(converter.TryConvert("$1,234,567.89", out result));
228+
Assert.AreEqual(1234567.89m, result);
229+
}
230+
231+
[TestMethod]
232+
public void TestMoneyConverterScientificNotation()
233+
{
234+
var converter = MoneyConverter.Default;
235+
236+
// NumberStyles.Currency includes AllowExponent, so scientific notation should work
237+
Assert.IsTrue(converter.TryConvert("1.5E3", out decimal result));
238+
Assert.AreEqual(1500m, result);
239+
240+
Assert.IsTrue(converter.TryConvert("2.5E-2", out result));
241+
Assert.AreEqual(0.025m, result);
242+
}
243+
244+
[TestMethod]
245+
public void TestMoneyConverterInvalidInput()
246+
{
247+
var converter = MoneyConverter.Default;
248+
249+
Assert.IsFalse(converter.TryConvert("invalid", out _));
250+
Assert.IsFalse(converter.TryConvert("", out _));
251+
Assert.IsFalse(converter.TryConvert(null, out _));
252+
}
253+
254+
#endregion
255+
256+
#region Vector Converter Tests
257+
258+
[TestMethod]
259+
public void TestVectorConverterJsonArrayFormat()
260+
{
261+
var converter = VectorConverter.Default;
262+
263+
// Test JSON array format
264+
Assert.IsTrue(converter.TryConvert("[0.1, 0.2, 0.3]", out float[] result));
265+
Assert.AreEqual(3, result.Length);
266+
Assert.AreEqual(0.1f, result[0], 0.0001f);
267+
Assert.AreEqual(0.2f, result[1], 0.0001f);
268+
Assert.AreEqual(0.3f, result[2], 0.0001f);
269+
}
270+
271+
[TestMethod]
272+
public void TestVectorConverterCommaSeparated()
273+
{
274+
var converter = VectorConverter.Default;
275+
276+
// Test comma-separated format (no brackets)
277+
Assert.IsTrue(converter.TryConvert("0.5, 1.0, 1.5", out float[] result));
278+
Assert.AreEqual(3, result.Length);
279+
Assert.AreEqual(0.5f, result[0], 0.0001f);
280+
Assert.AreEqual(1.0f, result[1], 0.0001f);
281+
Assert.AreEqual(1.5f, result[2], 0.0001f);
282+
}
283+
284+
[TestMethod]
285+
public void TestVectorConverterScientificNotation()
286+
{
287+
var converter = VectorConverter.Default;
288+
289+
// Test scientific notation in vectors
290+
Assert.IsTrue(converter.TryConvert("[1.5e-3, 2.0E2, -3.5e1]", out float[] result));
291+
Assert.AreEqual(3, result.Length);
292+
Assert.AreEqual(0.0015f, result[0], 0.000001f);
293+
Assert.AreEqual(200.0f, result[1], 0.0001f);
294+
Assert.AreEqual(-35.0f, result[2], 0.0001f);
295+
}
296+
297+
[TestMethod]
298+
public void TestVectorConverterNegativeValues()
299+
{
300+
var converter = VectorConverter.Default;
301+
302+
// Test negative values
303+
Assert.IsTrue(converter.TryConvert("[-0.5, -1.0, -1.5]", out float[] result));
304+
Assert.AreEqual(3, result.Length);
305+
Assert.AreEqual(-0.5f, result[0], 0.0001f);
306+
Assert.AreEqual(-1.0f, result[1], 0.0001f);
307+
Assert.AreEqual(-1.5f, result[2], 0.0001f);
308+
}
309+
310+
[TestMethod]
311+
public void TestVectorConverterLargeEmbedding()
312+
{
313+
var converter = VectorConverter.Default;
314+
315+
// Test realistic embedding size (e.g., OpenAI ada-002 uses 1536 dimensions)
316+
// Create a sample with 100 dimensions for testing
317+
string vectorString = "[" + string.Join(", ", Enumerable.Range(0, 100).Select(i => (i * 0.01f).ToString("F3"))) + "]";
318+
319+
Assert.IsTrue(converter.TryConvert(vectorString, out float[] result));
320+
Assert.AreEqual(100, result.Length);
321+
Assert.AreEqual(0.0f, result[0], 0.0001f);
322+
Assert.AreEqual(0.99f, result[99], 0.0001f);
323+
}
324+
325+
[TestMethod]
326+
public void TestVectorConverterWhitespaceHandling()
327+
{
328+
var converter = VectorConverter.Default;
329+
330+
// Test various whitespace scenarios
331+
Assert.IsTrue(converter.TryConvert(" [ 0.1 , 0.2 , 0.3 ] ", out float[] result));
332+
Assert.AreEqual(3, result.Length);
333+
334+
Assert.IsTrue(converter.TryConvert("0.1,0.2,0.3", out result)); // No spaces
335+
Assert.AreEqual(3, result.Length);
336+
}
337+
338+
[TestMethod]
339+
public void TestVectorConverterInvalidInput()
340+
{
341+
var converter = VectorConverter.Default;
342+
343+
// Test invalid inputs
344+
Assert.IsFalse(converter.TryConvert("", out _));
345+
Assert.IsFalse(converter.TryConvert(null, out _));
346+
Assert.IsFalse(converter.TryConvert("[]", out _)); // Empty array
347+
Assert.IsFalse(converter.TryConvert("[not, a, number]", out _));
348+
Assert.IsFalse(converter.TryConvert("[0.1, invalid, 0.3]", out _));
349+
Assert.IsFalse(converter.TryConvert("[", out _)); // Malformed
350+
}
351+
352+
[TestMethod]
353+
public void TestVectorConverterSingleValue()
354+
{
355+
var converter = VectorConverter.Default;
356+
357+
// Test single-value vector
358+
Assert.IsTrue(converter.TryConvert("[42.5]", out float[] result));
359+
Assert.AreEqual(1, result.Length);
360+
Assert.AreEqual(42.5f, result[0], 0.0001f);
361+
}
362+
184363
#endregion
185364

186365
#region Type Converter Registry Tests

project/dbatools/Csv/TypeConverters/NumericConverters.cs

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,33 @@ protected override bool TryParseCore(string value, NumberStyles styles, IFormatP
183183
/// <inheritdoc />
184184
protected override bool TryParseSpan(ReadOnlySpan<char> value, NumberStyles styles, IFormatProvider provider, out byte result)
185185
=> byte.TryParse(value, styles, provider, out result);
186+
#endif
187+
}
188+
189+
/// <summary>
190+
/// Converts string values to Decimal values with currency symbol support.
191+
/// Supports culture-aware parsing for currency symbols, decimal separators, and scientific notation.
192+
/// Suitable for SQL Server money and smallmoney data types.
193+
/// </summary>
194+
public sealed class MoneyConverter : CultureAwareConverterBase<decimal>
195+
{
196+
/// <summary>Gets the default instance of the converter.</summary>
197+
public static MoneyConverter Default { get; } = new MoneyConverter();
198+
199+
/// <summary>Initializes a new instance of the <see cref="MoneyConverter"/> class.</summary>
200+
public MoneyConverter()
201+
{
202+
NumberStyles = NumberStyles.Currency;
203+
}
204+
205+
/// <inheritdoc />
206+
protected override bool TryParseCore(string value, NumberStyles styles, IFormatProvider provider, out decimal result)
207+
=> decimal.TryParse(value, styles, provider, out result);
208+
209+
#if NET8_0_OR_GREATER
210+
/// <inheritdoc />
211+
protected override bool TryParseSpan(ReadOnlySpan<char> value, NumberStyles styles, IFormatProvider provider, out decimal result)
212+
=> decimal.TryParse(value, styles, provider, out result);
186213
#endif
187214
}
188215
}
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
using System;
2+
using System.Globalization;
3+
using System.Linq;
4+
5+
namespace Dataplat.Dbatools.Csv.TypeConverters
6+
{
7+
/// <summary>
8+
/// Converts string values to float arrays for SQL Server 2025 VECTOR data type.
9+
/// Supports JSON array format: "[0.1, 0.2, 0.3, ...]"
10+
/// Supports comma-separated format: "0.1, 0.2, 0.3, ..."
11+
/// </summary>
12+
public sealed class VectorConverter : TypeConverterBase<float[]>
13+
{
14+
/// <summary>Gets the default instance of the converter.</summary>
15+
public static VectorConverter Default { get; } = new VectorConverter();
16+
17+
/// <summary>
18+
/// Gets or sets the format provider to use for parsing individual float values.
19+
/// Defaults to InvariantCulture.
20+
/// </summary>
21+
public IFormatProvider FormatProvider { get; set; } = CultureInfo.InvariantCulture;
22+
23+
/// <summary>
24+
/// Attempts to convert the string value to a float array.
25+
/// Supports both JSON array format "[0.1, 0.2]" and comma-separated format "0.1, 0.2"
26+
/// </summary>
27+
public override bool TryConvert(string value, out float[] result)
28+
{
29+
result = null;
30+
31+
if (string.IsNullOrWhiteSpace(value))
32+
{
33+
return false;
34+
}
35+
36+
// Trim whitespace
37+
value = value.Trim();
38+
39+
// Check for JSON array format and strip brackets
40+
if (value.StartsWith("[") && value.EndsWith("]"))
41+
{
42+
value = value.Substring(1, value.Length - 2);
43+
}
44+
45+
// Split by comma and parse each value
46+
string[] parts = value.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
47+
48+
if (parts.Length == 0)
49+
{
50+
return false;
51+
}
52+
53+
float[] vector = new float[parts.Length];
54+
55+
for (int i = 0; i < parts.Length; i++)
56+
{
57+
string part = parts[i].Trim();
58+
59+
// Try parsing with Float styles to support scientific notation
60+
if (!float.TryParse(part, NumberStyles.Float | NumberStyles.AllowThousands, FormatProvider, out vector[i]))
61+
{
62+
return false;
63+
}
64+
}
65+
66+
result = vector;
67+
return true;
68+
}
69+
}
70+
}

0 commit comments

Comments
 (0)