Skip to content

Commit c8d2430

Browse files
tannergoodingtannergooding
authored
Expose UTF-8 support from FromHexString, TryToHexString, and TryToHexStringLower on the Convert class (#117965)
* Expose UTF-8 support from FromHexString, TryToHexString, and TryToHexStringLower on the Convert class * Fix copy/paste error * Fix an assert * Ensure TryDecodeFrom_Vector128 for UTF8 increments offset correctly
1 parent 92f6dec commit c8d2430

File tree

7 files changed

+584
-140
lines changed

7 files changed

+584
-140
lines changed

‎src/libraries/Common/src/System/HexConverter.cs

Lines changed: 215 additions & 67 deletions
Large diffs are not rendered by default.

‎src/libraries/System.Private.CoreLib/src/System/Convert.cs

Lines changed: 159 additions & 45 deletions
Large diffs are not rendered by default.

‎src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ internal static bool UInt64OrdinalIgnoreCaseAscii(ulong valueA, ulong valueB)
287287
internal static bool AllCharsInVectorAreAscii<TVector>(TVector vec)
288288
where TVector : struct, ISimdVector<TVector, ushort>
289289
{
290-
return (vec & TVector.Create(unchecked((ushort)~0x007F))).Equals(TVector.Zero);
290+
return (vec & TVector.Create(unchecked((ushort)~0x007F))) == TVector.Zero;
291291
}
292292
#endif
293293
}

‎src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -608,6 +608,18 @@ internal static void ThrowFormatException_BadFormatSpecifier()
608608
throw new FormatException(SR.Argument_BadFormatSpecifier);
609609
}
610610

611+
[DoesNotReturn]
612+
internal static void ThrowFormatException_BadHexChar()
613+
{
614+
throw new FormatException(SR.Format_BadHexChar);
615+
}
616+
617+
[DoesNotReturn]
618+
internal static void ThrowFormatException_BadHexLength()
619+
{
620+
throw new FormatException(SR.Format_BadHexLength);
621+
}
622+
611623
[DoesNotReturn]
612624
internal static void ThrowFormatException_NeedSingleChar()
613625
{

‎src/libraries/System.Runtime/ref/System.Runtime.cs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1294,7 +1294,9 @@ public static partial class Convert
12941294
public static object? ChangeType(object? value, System.TypeCode typeCode, System.IFormatProvider? provider) { throw null; }
12951295
public static byte[] FromBase64CharArray(char[] inArray, int offset, int length) { throw null; }
12961296
public static byte[] FromBase64String(string s) { throw null; }
1297+
public static byte[] FromHexString(System.ReadOnlySpan<byte> utf8Source) { throw null; }
12971298
public static byte[] FromHexString(System.ReadOnlySpan<char> chars) { throw null; }
1299+
public static System.Buffers.OperationStatus FromHexString(System.ReadOnlySpan<byte> utf8Source, System.Span<byte> destination, out int bytesConsumed, out int bytesWritten) { throw null; }
12981300
public static System.Buffers.OperationStatus FromHexString(System.ReadOnlySpan<char> source, System.Span<byte> destination, out int charsConsumed, out int bytesWritten) { throw null; }
12991301
public static byte[] FromHexString(string s) { throw null; }
13001302
public static System.Buffers.OperationStatus FromHexString(string source, System.Span<byte> destination, out int charsConsumed, out int bytesWritten) { throw null; }
@@ -1738,7 +1740,9 @@ public static partial class Convert
17381740
public static bool TryFromBase64Chars(System.ReadOnlySpan<char> chars, System.Span<byte> bytes, out int bytesWritten) { throw null; }
17391741
public static bool TryFromBase64String(string s, System.Span<byte> bytes, out int bytesWritten) { throw null; }
17401742
public static bool TryToBase64Chars(System.ReadOnlySpan<byte> bytes, System.Span<char> chars, out int charsWritten, System.Base64FormattingOptions options = System.Base64FormattingOptions.None) { throw null; }
1743+
public static bool TryToHexString(System.ReadOnlySpan<byte> source, System.Span<byte> utf8Destination, out int bytesWritten) { throw null; }
17411744
public static bool TryToHexString(System.ReadOnlySpan<byte> source, System.Span<char> destination, out int charsWritten) { throw null; }
1745+
public static bool TryToHexStringLower(System.ReadOnlySpan<byte> source, System.Span<byte> utf8Destination, out int bytesWritten) { throw null; }
17421746
public static bool TryToHexStringLower(System.ReadOnlySpan<byte> source, System.Span<char> destination, out int charsWritten) { throw null; }
17431747
}
17441748
public delegate TOutput Converter<in TInput, out TOutput>(TInput input) where TInput : allows ref struct where TOutput : allows ref struct;

‎src/libraries/System.Runtime/tests/System.Runtime.Extensions.Tests/System/Convert.FromHexString.cs

Lines changed: 132 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -35,21 +35,52 @@ public static void CompleteValueRange()
3535

3636
private static void TestSequence(byte[] expected, string actual)
3737
{
38-
byte[] fromResult = Convert.FromHexString(actual);
39-
Assert.Equal(expected, fromResult);
40-
41-
Span<byte> tryResult = new byte[actual.Length / 2];
42-
Assert.Equal(OperationStatus.Done, Convert.FromHexString(actual, tryResult, out int consumed, out int written));
43-
Assert.Equal(fromResult.Length, written);
44-
Assert.Equal(actual.Length, consumed);
45-
AssertExtensions.SequenceEqual(expected.AsSpan(), tryResult);
38+
TestSequenceString(expected, actual);
39+
TestSequenceUtf16(expected, actual);
40+
TestSequenceUtf8(expected, Encoding.UTF8.GetBytes(actual));
41+
42+
static void TestSequenceString(byte[] expected, string actual)
43+
{
44+
byte[] fromResult = Convert.FromHexString(actual);
45+
Assert.Equal(expected, fromResult);
46+
47+
Span<byte> tryResult = new byte[actual.Length / 2];
48+
Assert.Equal(OperationStatus.Done, Convert.FromHexString(actual, tryResult, out int consumed, out int written));
49+
Assert.Equal(fromResult.Length, written);
50+
Assert.Equal(actual.Length, consumed);
51+
AssertExtensions.SequenceEqual(expected.AsSpan(), tryResult);
52+
}
53+
54+
static void TestSequenceUtf16(byte[] expected, ReadOnlySpan<char> actual)
55+
{
56+
byte[] fromResult = Convert.FromHexString(actual);
57+
Assert.Equal(expected, fromResult);
58+
59+
Span<byte> tryResult = new byte[actual.Length / 2];
60+
Assert.Equal(OperationStatus.Done, Convert.FromHexString(actual, tryResult, out int consumed, out int written));
61+
Assert.Equal(fromResult.Length, written);
62+
Assert.Equal(actual.Length, consumed);
63+
AssertExtensions.SequenceEqual(expected.AsSpan(), tryResult);
64+
}
65+
66+
static void TestSequenceUtf8(byte[] expected, ReadOnlySpan<byte> actual)
67+
{
68+
byte[] fromResult = Convert.FromHexString(actual);
69+
Assert.Equal(expected, fromResult);
70+
71+
Span<byte> tryResult = new byte[actual.Length / 2];
72+
Assert.Equal(OperationStatus.Done, Convert.FromHexString(actual, tryResult, out int consumed, out int written));
73+
Assert.Equal(fromResult.Length, written);
74+
Assert.Equal(actual.Length, consumed);
75+
AssertExtensions.SequenceEqual(expected.AsSpan(), tryResult);
76+
}
4677
}
4778

4879
[Fact]
4980
public static void InvalidInputString_Null()
5081
{
51-
AssertExtensions.Throws<ArgumentNullException>("s", () => Convert.FromHexString(null));
52-
AssertExtensions.Throws<ArgumentNullException>("source", () => Convert.FromHexString(null, default, out _, out _));
82+
AssertExtensions.Throws<ArgumentNullException>("s", () => Convert.FromHexString((string)null));
83+
AssertExtensions.Throws<ArgumentNullException>("source", () => Convert.FromHexString((string)null, default, out _, out _));
5384
}
5485

5586
[Theory]
@@ -67,30 +98,55 @@ public static void InvalidInputString_FormatException_Or_FalseResult(string inva
6798

6899
Span<byte> buffer = stackalloc byte[invalidInput.Length / 2];
69100
Assert.Equal(OperationStatus.InvalidData, Convert.FromHexString(invalidInput.AsSpan(), buffer, out _, out _));
101+
Assert.Equal(OperationStatus.InvalidData, Convert.FromHexString(Encoding.UTF8.GetBytes(invalidInput), buffer, out _, out _));
70102
}
71103

72104
[Fact]
73105
public static void ZeroLength()
74106
{
75107
Assert.Same(Array.Empty<byte>(), Convert.FromHexString(string.Empty));
108+
Assert.Same(Array.Empty<byte>(), Convert.FromHexString(ReadOnlySpan<char>.Empty));
109+
Assert.Same(Array.Empty<byte>(), Convert.FromHexString(ReadOnlySpan<byte>.Empty));
76110

77111
OperationStatus convertResult = Convert.FromHexString(string.Empty, Span<byte>.Empty, out int consumed, out int written);
78112

79113
Assert.Equal(OperationStatus.Done, convertResult);
80114
Assert.Equal(0, written);
81115
Assert.Equal(0, consumed);
116+
117+
convertResult = Convert.FromHexString(ReadOnlySpan<char>.Empty, Span<byte>.Empty, out consumed, out written);
118+
119+
Assert.Equal(OperationStatus.Done, convertResult);
120+
Assert.Equal(0, written);
121+
Assert.Equal(0, consumed);
122+
123+
convertResult = Convert.FromHexString(ReadOnlySpan<byte>.Empty, Span<byte>.Empty, out consumed, out written);
124+
125+
Assert.Equal(OperationStatus.Done, convertResult);
126+
Assert.Equal(0, written);
127+
Assert.Equal(0, consumed);
82128
}
83129

84130
[Fact]
85131
public static void ToHexFromHexRoundtrip()
86132
{
87133
const int loopCount = 50;
134+
88135
Span<char> buffer = stackalloc char[loopCount * 2];
89136
Span<char> bufferLower = stackalloc char[loopCount * 2];
137+
138+
Span<byte> bufferUtf8 = stackalloc byte[loopCount * 2];
139+
Span<byte> bufferUtf8Lower = stackalloc byte[loopCount * 2];
140+
90141
for (int i = 1; i < loopCount; i++)
91142
{
92143
byte[] data = Security.Cryptography.RandomNumberGenerator.GetBytes(i);
144+
93145
string hex = Convert.ToHexString(data);
146+
string hexLower = hex.ToLowerInvariant();
147+
148+
byte[] hexUtf8 = Encoding.UTF8.GetBytes(hex);
149+
byte[] hexLowerUtf8 = Encoding.UTF8.GetBytes(hexLower);
94150

95151
Span<char> currentBuffer = buffer.Slice(0, i * 2);
96152
bool tryHex = Convert.TryToHexString(data, currentBuffer, out int written);
@@ -101,8 +157,20 @@ public static void ToHexFromHexRoundtrip()
101157
Span<char> currentBufferLower = bufferLower.Slice(0, i * 2);
102158
tryHex = Convert.TryToHexStringLower(data, currentBufferLower, out written);
103159
Assert.True(tryHex);
104-
AssertExtensions.SequenceEqual(hex.ToLowerInvariant().AsSpan(), currentBufferLower);
105-
Assert.Equal(hex.Length, written);
160+
AssertExtensions.SequenceEqual(hexLower.AsSpan(), currentBufferLower);
161+
Assert.Equal(hexLower.Length, written);
162+
163+
Span<byte> currentBufferUtf8 = bufferUtf8.Slice(0, i * 2);
164+
tryHex = Convert.TryToHexString(data, currentBufferUtf8, out written);
165+
Assert.True(tryHex);
166+
AssertExtensions.SequenceEqual(hexUtf8.AsSpan(), currentBufferUtf8);
167+
Assert.Equal(hexUtf8.Length, written);
168+
169+
Span<byte> currentBufferLowerUtf8 = bufferUtf8Lower.Slice(0, i * 2);
170+
tryHex = Convert.TryToHexStringLower(data, currentBufferLowerUtf8, out written);
171+
Assert.True(tryHex);
172+
AssertExtensions.SequenceEqual(hexLowerUtf8.AsSpan(), currentBufferLowerUtf8);
173+
Assert.Equal(hexLowerUtf8.Length, written);
106174

107175
TestSequence(data, hex);
108176
TestSequence(data, hex.ToLowerInvariant());
@@ -134,42 +202,90 @@ public static void TooShortDestination()
134202
Assert.Equal(OperationStatus.DestinationTooSmall, result);
135203
Assert.Equal(destinationSize * 2, charsConsumed);
136204
Assert.Equal(destinationSize, bytesWritten);
205+
206+
result = Convert.FromHexString(hex.AsSpan(), destination, out charsConsumed, out bytesWritten);
207+
208+
Assert.Equal(OperationStatus.DestinationTooSmall, result);
209+
Assert.Equal(destinationSize * 2, charsConsumed);
210+
Assert.Equal(destinationSize, bytesWritten);
211+
212+
result = Convert.FromHexString(Encoding.UTF8.GetBytes(hex), destination, out int bytesConsumed, out bytesWritten);
213+
214+
Assert.Equal(OperationStatus.DestinationTooSmall, result);
215+
Assert.Equal(destinationSize * 2, bytesConsumed);
216+
Assert.Equal(destinationSize, bytesWritten);
137217
}
138218

139219
[Fact]
140220
public static void TooLongDestination()
141221
{
142222
string hex = Convert.ToHexString([255, 255, 255]);
143223
byte[] buffer = new byte[100];
144-
var status = Convert.FromHexString(hex, buffer, out int charsConsumed, out int bytesWritten);
224+
OperationStatus status = Convert.FromHexString(hex, buffer, out int charsConsumed, out int bytesWritten);
225+
226+
Assert.Equal(OperationStatus.Done, status);
227+
Assert.Equal(hex.Length, charsConsumed);
228+
Assert.Equal(hex.Length / 2, bytesWritten);
229+
230+
status = Convert.FromHexString(hex.AsSpan(), buffer, out charsConsumed, out bytesWritten);
145231

146232
Assert.Equal(OperationStatus.Done, status);
147233
Assert.Equal(hex.Length, charsConsumed);
148234
Assert.Equal(hex.Length / 2, bytesWritten);
235+
236+
status = Convert.FromHexString(Encoding.UTF8.GetBytes(hex), buffer, out int bytesConsumed, out bytesWritten);
237+
238+
Assert.Equal(OperationStatus.Done, status);
239+
Assert.Equal(hex.Length, bytesConsumed);
240+
Assert.Equal(hex.Length / 2, bytesWritten);
149241
}
150242

151243
[Fact]
152244
public static void ExactDestination()
153245
{
154246
string hex = "ffffff";
155247
byte[] buffer = new byte[3];
156-
var status = Convert.FromHexString(hex, buffer, out int charsConsumed, out int bytesWritten);
248+
OperationStatus status = Convert.FromHexString(hex, buffer, out int charsConsumed, out int bytesWritten);
249+
250+
Assert.Equal(OperationStatus.Done, status);
251+
Assert.Equal(hex.Length, charsConsumed);
252+
Assert.Equal(hex.Length / 2, bytesWritten);
253+
254+
status = Convert.FromHexString(hex.AsSpan(), buffer, out charsConsumed, out bytesWritten);
157255

158256
Assert.Equal(OperationStatus.Done, status);
159257
Assert.Equal(hex.Length, charsConsumed);
160258
Assert.Equal(hex.Length / 2, bytesWritten);
259+
260+
status = Convert.FromHexString(Encoding.UTF8.GetBytes(hex), buffer, out int bytesConsumed, out bytesWritten);
261+
262+
Assert.Equal(OperationStatus.Done, status);
263+
Assert.Equal(hex.Length, bytesConsumed);
264+
Assert.Equal(hex.Length / 2, bytesWritten);
161265
}
162266

163267
[Fact]
164268
public static void ExactDestination_TrailingCharacter()
165269
{
166-
string hex = "fffff";
270+
string hex = "fffff";
167271
byte[] buffer = new byte[2];
168-
var status = Convert.FromHexString(hex, buffer, out int charsConsumed, out int bytesWritten);
272+
OperationStatus status = Convert.FromHexString(hex, buffer, out int charsConsumed, out int bytesWritten);
273+
274+
Assert.Equal(OperationStatus.NeedMoreData, status);
275+
Assert.Equal(hex.Length - 1, charsConsumed);
276+
Assert.Equal(hex.Length / 2, bytesWritten);
277+
278+
status = Convert.FromHexString(hex.AsSpan(), buffer, out charsConsumed, out bytesWritten);
169279

170280
Assert.Equal(OperationStatus.NeedMoreData, status);
171281
Assert.Equal(hex.Length - 1, charsConsumed);
172282
Assert.Equal(hex.Length / 2, bytesWritten);
283+
284+
status = Convert.FromHexString(Encoding.UTF8.GetBytes(hex), buffer, out int bytesConsumed, out bytesWritten);
285+
286+
Assert.Equal(OperationStatus.NeedMoreData, status);
287+
Assert.Equal(hex.Length - 1, bytesConsumed);
288+
Assert.Equal(hex.Length / 2, bytesWritten);
173289
}
174290

175291
[Fact]

0 commit comments

Comments
 (0)