Skip to content

Commit ff37fa8

Browse files
lemirelemire
committed
added tests, some minor failures
1 parent 7ecc035 commit ff37fa8

File tree

3 files changed

+264
-15
lines changed

3 files changed

+264
-15
lines changed

src/Base64ARMUTF8.cs renamed to src/Base64ARM.cs

Lines changed: 46 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -713,17 +713,15 @@ private unsafe static OperationStatus InnerDecodeFromBase64ARMRegular(ReadOnlySp
713713
{
714714
src -= bufferBytesConsumed;
715715
dst -= bufferBytesWritten;
716-
bytesConsumed = Math.Max(0, (int)(src - srcInit));
717-
bytesWritten = Math.Max(0, (int)(dst - dstInit));
718716

719717
int remainderBytesConsumed = 0;
720718
int remainderBytesWritten = 0;
721719

722720
OperationStatus result =
723721
SimdBase64.Base64.Base64WithWhiteSpaceToBinaryScalar(source.Slice(Math.Max(0, bytesConsumed)), dest.Slice(Math.Max(0, bytesWritten)), out remainderBytesConsumed, out remainderBytesWritten, isUrl);
724-
725722
bytesConsumed += remainderBytesConsumed;
726723
bytesWritten += remainderBytesWritten;
724+
727725
return result;
728726
}
729727
if (badCharMask != 0)
@@ -771,6 +769,22 @@ private unsafe static OperationStatus InnerDecodeFromBase64ARMRegular(ReadOnlySp
771769
int lastBlockSrcCount = 0;
772770
while ((bufferPtr - startOfBuffer) % 64 != 0 && src < srcEnd)
773771
{
772+
773+
if (!SimdBase64.Base64.IsValidBase64Index(*src))
774+
{
775+
bytesConsumed = Math.Max(0, (int)(src - srcInit) - lastBlockSrcCount - (int)bufferBytesConsumed);
776+
bytesWritten = Math.Max(0, (int)(dst - dstInit) - (int)bufferBytesWritten);
777+
778+
int remainderBytesConsumed = 0;
779+
int remainderBytesWritten = 0;
780+
781+
OperationStatus result =
782+
SimdBase64.Base64.Base64WithWhiteSpaceToBinaryScalar(source.Slice(Math.Max(0, bytesConsumed)), dest.Slice(Math.Max(0, bytesWritten)), out remainderBytesConsumed, out remainderBytesWritten, isUrl);
783+
784+
bytesConsumed += remainderBytesConsumed;
785+
bytesWritten += remainderBytesWritten;
786+
return result;
787+
}
774788
byte val = toBase64[(int)*src];
775789
*bufferPtr = val;
776790
if (val > 64)
@@ -834,6 +848,13 @@ private unsafe static OperationStatus InnerDecodeFromBase64ARMRegular(ReadOnlySp
834848

835849
while (leftover < 4 && src < srcEnd)
836850
{
851+
if (!SimdBase64.Base64.IsValidBase64Index(*src))
852+
{
853+
bytesConsumed = (int)(src - srcInit);
854+
bytesWritten = (int)(dst - dstInit);
855+
return OperationStatus.InvalidData;
856+
}
857+
837858
byte val = toBase64[(byte)*src];
838859
if (val > 64)
839860
{
@@ -1015,9 +1036,6 @@ private unsafe static OperationStatus InnerDecodeFromBase64ARMUrl(ReadOnlySpan<b
10151036
src -= bufferBytesConsumed;
10161037
dst -= bufferBytesWritten;
10171038

1018-
bytesConsumed = Math.Max(0, (int)(src - srcInit));
1019-
bytesWritten = Math.Max(0, (int)(dst - dstInit));
1020-
10211039
int remainderBytesConsumed = 0;
10221040
int remainderBytesWritten = 0;
10231041

@@ -1381,6 +1399,22 @@ private unsafe static OperationStatus InnerDecodeFromBase64ARMUrl(ReadOnlySpan<c
13811399
int lastBlockSrcCount = 0;
13821400
while ((bufferPtr - startOfBuffer) % 64 != 0 && src < srcEnd)
13831401
{
1402+
1403+
if (!SimdBase64.Base64.IsValidBase64Index(*src))
1404+
{
1405+
bytesConsumed = Math.Max(0, (int)(src - srcInit) - lastBlockSrcCount - (int)bufferBytesConsumed);
1406+
bytesWritten = Math.Max(0, (int)(dst - dstInit) - (int)bufferBytesWritten);
1407+
1408+
int remainderBytesConsumed = 0;
1409+
int remainderBytesWritten = 0;
1410+
1411+
OperationStatus result =
1412+
SimdBase64.Base64.Base64WithWhiteSpaceToBinaryScalar(source.Slice(Math.Max(0, bytesConsumed)), dest.Slice(Math.Max(0, bytesWritten)), out remainderBytesConsumed, out remainderBytesWritten, isUrl);
1413+
1414+
bytesConsumed += remainderBytesConsumed;
1415+
bytesWritten += remainderBytesWritten;
1416+
return result;
1417+
}
13841418
byte val = toBase64[(int)*src];
13851419
*bufferPtr = val;
13861420
if (val > 64)
@@ -1445,6 +1479,12 @@ private unsafe static OperationStatus InnerDecodeFromBase64ARMUrl(ReadOnlySpan<c
14451479

14461480
while (leftover < 4 && src < srcEnd)
14471481
{
1482+
if (!SimdBase64.Base64.IsValidBase64Index(*src))
1483+
{
1484+
bytesConsumed = (int)(src - srcInit);
1485+
bytesWritten = (int)(dst - dstInit);
1486+
return OperationStatus.InvalidData;
1487+
}
14481488
byte val = toBase64[(byte)*src];
14491489
if (val > 64)
14501490
{

‎test/Base64DecodingTestsUTF16.cs‎

Lines changed: 96 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,12 @@ public void BadPaddingUTF16Base64SSE()
473473
BadPaddingBase64UTF16(Base64.DecodeFromBase64SSE, Base64.SafeBase64ToBinaryWithWhiteSpace);
474474
}
475475

476+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
477+
[Trait("Category", "arm64")]
478+
public void BadPaddingUTF16Base64ARM()
479+
{
480+
BadPaddingBase64UTF16(SimdBase64.Arm.Base64.DecodeFromBase64ARM, Base64.SafeBase64ToBinaryWithWhiteSpace);
481+
}
476482

477483
protected void DoomedBase64RoundtripUTF16(Base64WithWhiteSpaceToBinaryFromUTF16 Base64WithWhiteSpaceToBinaryFromUTF16, DecodeFromBase64DelegateSafeFromUTF16 DecodeFromBase64DelegateSafeFromUTF16)
478484
{
@@ -533,6 +539,12 @@ public void DoomedBase64RoundtripSSEUTF16()
533539
DoomedBase64RoundtripUTF16(Base64.DecodeFromBase64SSE, Base64.SafeBase64ToBinaryWithWhiteSpace);
534540
}
535541

542+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
543+
[Trait("Category", "arm64")]
544+
public void DoomedBase64RoundtripARMUTF16()
545+
{
546+
DoomedBase64RoundtripUTF16(SimdBase64.Arm.Base64.DecodeFromBase64ARM, Base64.SafeBase64ToBinaryWithWhiteSpace);
547+
}
536548

537549

538550
protected void TruncatedDoomedBase64RoundtripUTF16(Base64WithWhiteSpaceToBinaryFromUTF16 Base64WithWhiteSpaceToBinaryFromUTF16, DecodeFromBase64DelegateSafeFromUTF16 DecodeFromBase64DelegateSafeFromUTF16)
@@ -594,6 +606,13 @@ public void TruncatedDoomedBase64RoundtripSSEUTF16()
594606
TruncatedDoomedBase64RoundtripUTF16(Base64.DecodeFromBase64SSE, Base64.SafeBase64ToBinaryWithWhiteSpace);
595607
}
596608

609+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
610+
[Trait("Category", "arm64")]
611+
public void TruncatedDoomedBase64RoundtripARMUTF16()
612+
{
613+
TruncatedDoomedBase64RoundtripUTF16(SimdBase64.Arm.Base64.DecodeFromBase64ARM, Base64.SafeBase64ToBinaryWithWhiteSpace);
614+
}
615+
597616
protected void RoundtripBase64WithSpacesUTF16(Base64WithWhiteSpaceToBinaryFromUTF16 Base64WithWhiteSpaceToBinaryFromUTF16, DecodeFromBase64DelegateSafeFromUTF16 DecodeFromBase64DelegateSafeFromUTF16)
598617
{
599618
if (Base64WithWhiteSpaceToBinaryFromUTF16 == null || DecodeFromBase64DelegateSafeFromUTF16 == null || Base64.MaximalBinaryLengthFromBase64Scalar<char> == null)
@@ -660,6 +679,13 @@ public void RoundtripBase64WithSpacesSSEUTF16()
660679
RoundtripBase64WithSpacesUTF16(Base64.DecodeFromBase64SSE, Base64.SafeBase64ToBinaryWithWhiteSpace);
661680
}
662681

682+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
683+
[Trait("Category", "arm64")]
684+
public void RoundtripBase64WithSpacesARMUTF16()
685+
{
686+
RoundtripBase64WithSpacesUTF16(SimdBase64.Arm.Base64.DecodeFromBase64ARM, Base64.SafeBase64ToBinaryWithWhiteSpace);
687+
}
688+
663689
protected void AbortedSafeRoundtripBase64UTF16(Base64WithWhiteSpaceToBinaryFromUTF16 Base64WithWhiteSpaceToBinaryFromUTF16, DecodeFromBase64DelegateSafeFromUTF16 DecodeFromBase64DelegateSafeFromUTF16)
664690
{
665691
if (Base64WithWhiteSpaceToBinaryFromUTF16 == null || DecodeFromBase64DelegateSafeFromUTF16 == null || Base64.MaximalBinaryLengthFromBase64Scalar<char> == null)
@@ -728,6 +754,14 @@ public void AbortedSafeRoundtripBase64SSEUTF16()
728754
AbortedSafeRoundtripBase64UTF16(Base64.DecodeFromBase64SSE, Base64.SafeBase64ToBinaryWithWhiteSpace);
729755
}
730756

757+
758+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)]
759+
[Trait("Category", "arm64")]
760+
public void AbortedSafeRoundtripBase64ARMUTF16()
761+
{
762+
AbortedSafeRoundtripBase64UTF16(SimdBase64.Arm.Base64.DecodeFromBase64ARM, Base64.SafeBase64ToBinaryWithWhiteSpace);
763+
}
764+
731765
protected void AbortedSafeRoundtripBase64WithSpacesUTF16(Base64WithWhiteSpaceToBinaryFromUTF16 Base64WithWhiteSpaceToBinaryFromUTF16, DecodeFromBase64DelegateSafeFromUTF16 DecodeFromBase64DelegateSafeFromUTF16)
732766
{
733767
if (Base64WithWhiteSpaceToBinaryFromUTF16 == null || DecodeFromBase64DelegateSafeFromUTF16 == null || Base64.MaximalBinaryLengthFromBase64Scalar<char> == null)
@@ -797,6 +831,13 @@ public void AbortedSafeRoundtripBase64WithSpacesSSEUTF16()
797831
AbortedSafeRoundtripBase64WithSpacesUTF16(Base64.DecodeFromBase64SSE, Base64.SafeBase64ToBinaryWithWhiteSpace);
798832
}
799833

834+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
835+
[Trait("Category", "arm64")]
836+
public void AbortedSafeRoundtripBase64WithSpacesARMUTF16()
837+
{
838+
AbortedSafeRoundtripBase64WithSpacesUTF16(SimdBase64.Arm.Base64.DecodeFromBase64ARM, Base64.SafeBase64ToBinaryWithWhiteSpace);
839+
}
840+
800841
protected void StreamingBase64RoundtripUTF16(Base64WithWhiteSpaceToBinaryFromUTF16 Base64WithWhiteSpaceToBinaryFromUTF16, DecodeFromBase64DelegateSafeFromUTF16 DecodeFromBase64DelegateSafeFromUTF16)
801842
{
802843
int len = 2048;
@@ -869,6 +910,14 @@ public void StreamingBase64RoundtripSSEUTF16()
869910
StreamingBase64RoundtripUTF16(Base64.DecodeFromBase64SSE, Base64.SafeBase64ToBinaryWithWhiteSpace);
870911
}
871912

913+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
914+
[Trait("Category", "arm64")]
915+
public void StreamingBase64RoundtripARMUTF16()
916+
{
917+
StreamingBase64RoundtripUTF16(SimdBase64.Arm.Base64.DecodeFromBase64ARM, Base64.SafeBase64ToBinaryWithWhiteSpace);
918+
}
919+
920+
872921
protected static void ReadmeTestUTF16(Base64WithWhiteSpaceToBinaryFromUTF16 Base64WithWhiteSpaceToBinaryFromUTF16, DecodeFromBase64DelegateSafeFromUTF16 DecodeFromBase64DelegateSafeFromUTF16)
873922
{
874923
int len = 2048;
@@ -939,6 +988,13 @@ public void ReadmeTestSSEUTF16()
939988
ReadmeTestUTF16(Base64.DecodeFromBase64SSE, Base64.SafeBase64ToBinaryWithWhiteSpace);
940989
}
941990

991+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
992+
[Trait("Category", "arm64")]
993+
public void ReadmeTestARMUTF16()
994+
{
995+
ReadmeTestUTF16(SimdBase64.Arm.Base64.DecodeFromBase64ARM, Base64.SafeBase64ToBinaryWithWhiteSpace);
996+
}
997+
942998
protected static void ReadmeTestSafeUTF16(Base64WithWhiteSpaceToBinaryFromUTF16 Base64WithWhiteSpaceToBinaryFromUTF16, DecodeFromBase64DelegateSafeFromUTF16 DecodeFromBase64DelegateSafeFromUTF16)
943999
{
9441000
int len = 72;
@@ -987,6 +1043,14 @@ public void ReadmeTestSafeSSEUTF16()
9871043
ReadmeTestSafeUTF16(Base64.DecodeFromBase64SSE, Base64.SafeBase64ToBinaryWithWhiteSpace);
9881044
}
9891045

1046+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
1047+
[Trait("Category", "arm64")]
1048+
public void ReadmeTestSafeARMUTF16()
1049+
{
1050+
ReadmeTestSafeUTF16(SimdBase64.Arm.Base64.DecodeFromBase64ARM, Base64.SafeBase64ToBinaryWithWhiteSpace);
1051+
}
1052+
1053+
9901054
protected void DoomedBase64AtPos0(Base64WithWhiteSpaceToBinaryFromUTF16 Base64WithWhiteSpaceToBinaryFromUTF16, DecodeFromBase64DelegateSafeFromUTF16 DecodeFromBase64DelegateSafeFromUTF16)
9911055
{
9921056
if (Base64WithWhiteSpaceToBinaryFromUTF16 == null || DecodeFromBase64DelegateSafeFromUTF16 == null || Base64.MaximalBinaryLengthFromBase64Scalar<char> == null)
@@ -1057,6 +1121,13 @@ public void DoomedBase64AtPos0SSEUTF16()
10571121
DoomedBase64AtPos0(Base64.DecodeFromBase64SSE, Base64.SafeBase64ToBinaryWithWhiteSpace);
10581122
}
10591123

1124+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
1125+
[Trait("Category", "arm64")]
1126+
public void DoomedBase64AtPos0ARMUTF16()
1127+
{
1128+
DoomedBase64AtPos0(SimdBase64.Arm.Base64.DecodeFromBase64ARM, Base64.SafeBase64ToBinaryWithWhiteSpace);
1129+
}
1130+
10601131
protected static void EnronFilesTestUTF16(Base64WithWhiteSpaceToBinaryFromUTF16 Base64WithWhiteSpaceToBinaryFromUTF16, DecodeFromBase64DelegateSafeFromUTF16 DecodeFromBase64DelegateSafeFromUTF16)
10611132
{
10621133
string[] fileNames = Directory.GetFiles("../../../../benchmark/data/email");
@@ -1103,6 +1174,12 @@ public void EnronFilesTestSSEUTF16()
11031174
EnronFilesTestUTF16(Base64.DecodeFromBase64SSE, Base64.SafeBase64ToBinaryWithWhiteSpace);
11041175
}
11051176

1177+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
1178+
[Trait("Category", "arm64")]
1179+
public void EnronFilesTestARMUTF16()
1180+
{
1181+
EnronFilesTestUTF16(SimdBase64.Arm.Base64.DecodeFromBase64ARM, Base64.SafeBase64ToBinaryWithWhiteSpace);
1182+
}
11061183

11071184
protected static void SwedenZoneBaseFileTestUTF16(Base64WithWhiteSpaceToBinaryFromUTF16 Base64WithWhiteSpaceToBinaryFromUTF16, DecodeFromBase64DelegateSafeFromUTF16 DecodeFromBase64DelegateSafeFromUTF16)
11081185
{
@@ -1146,7 +1223,12 @@ public void SwedenZoneBaseFileTestSSEUTF16()
11461223
SwedenZoneBaseFileTestUTF16(Base64.DecodeFromBase64SSE, Base64.SafeBase64ToBinaryWithWhiteSpace);
11471224
}
11481225

1149-
1226+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
1227+
[Trait("Category", "arm64")]
1228+
public void SwedenZoneBaseFileTestARMUTF16()
1229+
{
1230+
SwedenZoneBaseFileTestUTF16(SimdBase64.Arm.Base64.DecodeFromBase64ARM, Base64.SafeBase64ToBinaryWithWhiteSpace);
1231+
}
11501232

11511233
protected void DoomedPartialBufferUTF16(Base64WithWhiteSpaceToBinaryFromUTF16 Base64WithWhiteSpaceToBinaryFromUTF16, DecodeFromBase64DelegateSafeFromUTF16 DecodeFromBase64DelegateSafeFromUTF16)
11521234
{
@@ -1226,6 +1308,13 @@ public void DoomedPartialBufferSSEUTF16()
12261308
DoomedPartialBufferUTF16(Base64.DecodeFromBase64SSE, Base64.SafeBase64ToBinaryWithWhiteSpace);
12271309
}
12281310

1311+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
1312+
[Trait("Category", "arm64")]
1313+
public void DoomedPartialBufferARMUTF16()
1314+
{
1315+
DoomedPartialBufferUTF16(SimdBase64.Arm.Base64.DecodeFromBase64ARM, Base64.SafeBase64ToBinaryWithWhiteSpace);
1316+
}
1317+
12291318
protected static void Issue511UTF16(Base64WithWhiteSpaceToBinaryFromUTF16 Base64WithWhiteSpaceToBinary)
12301319
{
12311320
ArgumentNullException.ThrowIfNull(Base64WithWhiteSpaceToBinary);
@@ -1344,7 +1433,6 @@ protected void TruncatedCharErrorUTF16(Base64WithWhiteSpaceToBinaryFromUTF16 Bas
13441433
string base64 = Convert.ToBase64String(source);
13451434

13461435
int location = random.Next(0, base64.Length + 1)/4;
1347-
13481436
char[] base64WithGarbage = base64.Insert(location, badNonASCIIString).ToCharArray();
13491437

13501438
// Prepare a buffer for decoding the base64 back to binary
@@ -1457,6 +1545,12 @@ public void TruncatedCharErrorUrlUTF16SSE()
14571545
TruncatedCharErrorUrlUTF16(Base64.DecodeFromBase64SSE,Base64.SafeBase64ToBinaryWithWhiteSpace);
14581546
}
14591547

1548+
[Trait("Category", "arm64")]
1549+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
1550+
public void TruncatedCharErrorUrlUTF16ARM()
1551+
{
1552+
TruncatedCharErrorUrlUTF16(SimdBase64.Arm.Base64.DecodeFromBase64ARM,Base64.SafeBase64ToBinaryWithWhiteSpace);
1553+
}
14601554

14611555
}
14621556

0 commit comments

Comments
 (0)