File tree Expand file tree Collapse file tree 1 file changed +42
-0
lines changed Expand file tree Collapse file tree 1 file changed +42
-0
lines changed Original file line number Diff line number Diff line change @@ -131,7 +131,49 @@ public static unsafe bool SIMDIsAscii(this ReadOnlySpan<char> s)
131
131
}
132
132
return true ;
133
133
}
134
+
135
+ [ MethodImpl ( MethodImplOptions . AggressiveInlining ) ]
136
+ internal static unsafe nuint GetIndexOfFirstNonAsciiByte ( byte * pBuffer , nuint bufferLength )
137
+ {
138
+ byte * pBufferEnd = pBuffer + bufferLength ;
139
+ byte * pCurrent = pBuffer ;
140
+
141
+ // Process in blocks of 16 bytes when possible
142
+ while ( pCurrent + 16 <= pBufferEnd )
143
+ {
144
+ ulong v1 = * ( ulong * ) pCurrent ;
145
+ ulong v2 = * ( ulong * ) ( pCurrent + 8 ) ;
146
+ ulong v = v1 | v2 ;
147
+
148
+ if ( ( v & 0x8080808080808080 ) != 0 )
149
+ {
150
+ for ( ; pCurrent < pBufferEnd ; pCurrent ++ )
151
+ {
152
+ if ( * pCurrent >= 0b10000000 )
153
+ {
154
+ return ( nuint ) ( pCurrent - pBuffer ) ;
155
+ }
156
+ }
157
+ }
158
+
159
+ pCurrent += 16 ;
160
+ }
161
+
162
+ // Process the tail byte-by-byte
163
+ for ( ; pCurrent < pBufferEnd ; pCurrent ++ )
164
+ {
165
+ if ( * pCurrent >= 0b10000000 )
166
+ {
167
+ return ( nuint ) ( pCurrent - pBuffer ) ;
168
+ }
169
+ }
170
+
171
+ return bufferLength ;
172
+ }
173
+
134
174
}
175
+
176
+
135
177
}
136
178
// Further reading:
137
179
// https://github.com/dotnet/runtime/blob/main/src/libraries/System.Text.Encodings.Web/src/System/Text/Unicode/UnicodeHelpers.cs
You can’t perform that action at this time.
0 commit comments