|
6 | 6 | using System.Runtime.CompilerServices; |
7 | 7 | using System.Runtime.InteropServices; |
8 | 8 | using System.Runtime.Intrinsics; |
| 9 | +using System.Runtime.Intrinsics.Arm; |
9 | 10 | using System.Text; |
10 | 11 |
|
11 | 12 | namespace System.Buffers |
@@ -270,12 +271,31 @@ public static bool Equals<TValueLength>(ref char matchStart, ref readonly Single |
270 | 271 | else |
271 | 272 | { |
272 | 273 | Debug.Assert(state.Value.Length is 2 or 3); |
273 | | - Debug.Assert(matchStart == state.Value[0], "This should only be called after the first character has been checked"); |
274 | 274 |
|
275 | | - // We know that the candidate is 2 or 3 characters long, and that the first character has already been checked. |
276 | | - // We only have to to check whether the last 2 characters also match. |
277 | 275 | ref byte matchByteStart = ref Unsafe.As<char, byte>(ref matchStart); |
278 | | - return Unsafe.ReadUnaligned<uint>(ref Unsafe.Add(ref matchByteStart, state.SecondReadByteOffset)) == state.Value32_1; |
| 276 | + |
| 277 | + if (AdvSimd.IsSupported) |
| 278 | + { |
| 279 | + // See comments on SingleStringSearchValuesPackedThreeChars.CanSkipAnchorMatchVerification. |
| 280 | + // When running on Arm64, this helper is also used to confirm vectorized anchor matches. |
| 281 | + // We do so because we're using UnzipEven when packing inputs, which may produce false positive anchor matches. |
| 282 | + // When called from SingleStringSearchValuesThreeChars (non-packed), we could skip to the else branch instead. |
| 283 | + Debug.Assert(matchStart == state.Value[0] || (matchStart & 0xFF) == state.Value[0]); |
| 284 | + |
| 285 | + uint differentBits = Unsafe.ReadUnaligned<uint>(ref matchByteStart) - state.Value32_0; |
| 286 | + differentBits |= Unsafe.ReadUnaligned<uint>(ref Unsafe.Add(ref matchByteStart, state.SecondReadByteOffset)) - state.Value32_1; |
| 287 | + return differentBits == 0; |
| 288 | + } |
| 289 | + else |
| 290 | + { |
| 291 | + // Otherwise, this path is not used when confirming vectorized anchor matches. |
| 292 | + // It's only used as part of the scalar search loop, which always checks that the first character matches before calling this helper. |
| 293 | + // We know that the candidate is 2 or 3 characters long, and that the first character has already been checked. |
| 294 | + // We only have to to check whether the last 2 characters also match. |
| 295 | + Debug.Assert(matchStart == state.Value[0], "This should only be called after the first character has been checked"); |
| 296 | + |
| 297 | + return Unsafe.ReadUnaligned<uint>(ref Unsafe.Add(ref matchByteStart, state.SecondReadByteOffset)) == state.Value32_1; |
| 298 | + } |
279 | 299 | } |
280 | 300 | } |
281 | 301 | } |
@@ -319,13 +339,32 @@ public static bool Equals<TValueLength>(ref char matchStart, ref readonly Single |
319 | 339 | else |
320 | 340 | { |
321 | 341 | Debug.Assert(state.Value.Length is 2 or 3); |
322 | | - Debug.Assert(TransformInput(matchStart) == state.Value[0], "This should only be called after the first character has been checked"); |
323 | 342 |
|
324 | | - // We know that the candidate is 2 or 3 characters long, and that the first character has already been checked. |
325 | | - // We only have to to check whether the last 2 characters also match. |
326 | 343 | const uint CaseMask = ~0x200020u; |
327 | 344 | ref byte matchByteStart = ref Unsafe.As<char, byte>(ref matchStart); |
328 | | - return (Unsafe.ReadUnaligned<uint>(ref Unsafe.Add(ref matchByteStart, state.SecondReadByteOffset)) & CaseMask) == state.Value32_1; |
| 345 | + |
| 346 | + if (AdvSimd.IsSupported) |
| 347 | + { |
| 348 | + // See comments on SingleStringSearchValuesPackedThreeChars.CanSkipAnchorMatchVerification. |
| 349 | + // When running on Arm64, this helper is also used to confirm vectorized anchor matches. |
| 350 | + // We do so because we're using UnzipEven when packing inputs, which may produce false positive anchor matches. |
| 351 | + // When called from SingleStringSearchValuesThreeChars (non-packed), we could skip to the else branch instead. |
| 352 | + Debug.Assert(TransformInput((char)(matchStart & 0xFF)) == state.Value[0]); |
| 353 | + |
| 354 | + uint differentBits = (Unsafe.ReadUnaligned<uint>(ref matchByteStart) & CaseMask) - state.Value32_0; |
| 355 | + differentBits |= (Unsafe.ReadUnaligned<uint>(ref Unsafe.Add(ref matchByteStart, state.SecondReadByteOffset)) & CaseMask) - state.Value32_1; |
| 356 | + return differentBits == 0; |
| 357 | + } |
| 358 | + else |
| 359 | + { |
| 360 | + // Otherwise, this path is not used when confirming vectorized anchor matches. |
| 361 | + // It's only used as part of the scalar search loop, which always checks that the first character matches before calling this helper. |
| 362 | + // We know that the candidate is 2 or 3 characters long, and that the first character has already been checked. |
| 363 | + // We only have to to check whether the last 2 characters also match. |
| 364 | + Debug.Assert(TransformInput(matchStart) == state.Value[0], "This should only be called after the first character has been checked"); |
| 365 | + |
| 366 | + return (Unsafe.ReadUnaligned<uint>(ref Unsafe.Add(ref matchByteStart, state.SecondReadByteOffset)) & CaseMask) == state.Value32_1; |
| 367 | + } |
329 | 368 | } |
330 | 369 | } |
331 | 370 | } |
@@ -392,7 +431,6 @@ public static bool Equals<TValueLength>(ref char matchStart, ref readonly Single |
392 | 431 | else |
393 | 432 | { |
394 | 433 | Debug.Assert(state.Value.Length is 2 or 3); |
395 | | - Debug.Assert((matchStart & ~0x20) == (state.Value[0] & ~0x20)); |
396 | 434 |
|
397 | 435 | ref byte matchByteStart = ref Unsafe.As<char, byte>(ref matchStart); |
398 | 436 | uint differentBits = (Unsafe.ReadUnaligned<uint>(ref matchByteStart) & state.ToUpperMask32_0) - state.Value32_0; |
|
0 commit comments