Skip to content

Commit 75121b2

Browse files
JimBobSquarePantsJimBobSquarePants
authored
Merge pull request #2365 from SixLabors/js/2121
Use compile time constants over method for all MMShuffle operations.
2 parents 84ff63d + 51eb978 commit 75121b2

File tree

15 files changed

+951
-238
lines changed

15 files changed

+951
-238
lines changed

‎src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs

Lines changed: 29 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Numerics;
66
using System.Runtime.CompilerServices;
77
using System.Runtime.InteropServices;
8+
using static SixLabors.ImageSharp.SimdUtils;
89

910
// The JIT can detect and optimize rotation idioms ROTL (Rotate Left)
1011
// and ROTR (Rotate Right) emitting efficient CPU instructions:
@@ -18,9 +19,12 @@ namespace SixLabors.ImageSharp;
1819
internal interface IComponentShuffle
1920
{
2021
/// <summary>
21-
/// Gets the shuffle control.
22+
/// Shuffles then slices 8-bit integers within 128-bit lanes in <paramref name="source"/>
23+
/// using the control and store the results in <paramref name="dest"/>.
2224
/// </summary>
23-
byte Control { get; }
25+
/// <param name="source">The source span of bytes.</param>
26+
/// <param name="dest">The destination span of bytes.</param>
27+
void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest);
2428

2529
/// <summary>
2630
/// Shuffle 8-bit integers within 128-bit lanes in <paramref name="source"/>
@@ -42,37 +46,25 @@ internal interface IShuffle4 : IComponentShuffle
4246

4347
internal readonly struct DefaultShuffle4 : IShuffle4
4448
{
45-
private readonly byte p3;
46-
private readonly byte p2;
47-
private readonly byte p1;
48-
private readonly byte p0;
49-
50-
public DefaultShuffle4(byte p3, byte p2, byte p1, byte p0)
49+
public DefaultShuffle4(byte control)
5150
{
52-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p3, 0, 3, nameof(p3));
53-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p2, 0, 3, nameof(p2));
54-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p1, 0, 3, nameof(p1));
55-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p0, 0, 3, nameof(p0));
56-
57-
this.p3 = p3;
58-
this.p2 = p2;
59-
this.p1 = p1;
60-
this.p0 = p0;
61-
this.Control = SimdUtils.Shuffle.MmShuffle(p3, p2, p1, p0);
51+
DebugGuard.MustBeBetweenOrEqualTo<byte>(control, 0, 3, nameof(control));
52+
this.Control = control;
6253
}
6354

6455
public byte Control { get; }
6556

57+
[MethodImpl(InliningOptions.ShortMethod)]
58+
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
59+
=> HwIntrinsics.Shuffle4Reduce(ref source, ref dest, this.Control);
60+
6661
[MethodImpl(InliningOptions.ShortMethod)]
6762
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
6863
{
6964
ref byte sBase = ref MemoryMarshal.GetReference(source);
7065
ref byte dBase = ref MemoryMarshal.GetReference(dest);
7166

72-
int p3 = this.p3;
73-
int p2 = this.p2;
74-
int p1 = this.p1;
75-
int p0 = this.p0;
67+
Shuffle.InverseMMShuffle(this.Control, out int p3, out int p2, out int p1, out int p0);
7668

7769
for (int i = 0; i < source.Length; i += 4)
7870
{
@@ -86,11 +78,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
8678

8779
internal readonly struct WXYZShuffle4 : IShuffle4
8880
{
89-
public byte Control
90-
{
91-
[MethodImpl(InliningOptions.ShortMethod)]
92-
get => SimdUtils.Shuffle.MmShuffle(2, 1, 0, 3);
93-
}
81+
[MethodImpl(InliningOptions.ShortMethod)]
82+
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
83+
=> HwIntrinsics.Shuffle4Reduce(ref source, ref dest, Shuffle.MMShuffle2103);
9484

9585
[MethodImpl(InliningOptions.ShortMethod)]
9686
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
@@ -112,11 +102,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
112102

113103
internal readonly struct WZYXShuffle4 : IShuffle4
114104
{
115-
public byte Control
116-
{
117-
[MethodImpl(InliningOptions.ShortMethod)]
118-
get => SimdUtils.Shuffle.MmShuffle(0, 1, 2, 3);
119-
}
105+
[MethodImpl(InliningOptions.ShortMethod)]
106+
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
107+
=> HwIntrinsics.Shuffle4Reduce(ref source, ref dest, Shuffle.MMShuffle0123);
120108

121109
[MethodImpl(InliningOptions.ShortMethod)]
122110
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
@@ -138,11 +126,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
138126

139127
internal readonly struct YZWXShuffle4 : IShuffle4
140128
{
141-
public byte Control
142-
{
143-
[MethodImpl(InliningOptions.ShortMethod)]
144-
get => SimdUtils.Shuffle.MmShuffle(0, 3, 2, 1);
145-
}
129+
[MethodImpl(InliningOptions.ShortMethod)]
130+
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
131+
=> HwIntrinsics.Shuffle4Reduce(ref source, ref dest, Shuffle.MMShuffle0321);
146132

147133
[MethodImpl(InliningOptions.ShortMethod)]
148134
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
@@ -164,11 +150,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
164150

165151
internal readonly struct ZYXWShuffle4 : IShuffle4
166152
{
167-
public byte Control
168-
{
169-
[MethodImpl(InliningOptions.ShortMethod)]
170-
get => SimdUtils.Shuffle.MmShuffle(3, 0, 1, 2);
171-
}
153+
[MethodImpl(InliningOptions.ShortMethod)]
154+
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
155+
=> HwIntrinsics.Shuffle4Reduce(ref source, ref dest, Shuffle.MMShuffle3012);
172156

173157
[MethodImpl(InliningOptions.ShortMethod)]
174158
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
@@ -197,11 +181,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
197181

198182
internal readonly struct XWZYShuffle4 : IShuffle4
199183
{
200-
public byte Control
201-
{
202-
[MethodImpl(InliningOptions.ShortMethod)]
203-
get => SimdUtils.Shuffle.MmShuffle(1, 2, 3, 0);
204-
}
184+
[MethodImpl(InliningOptions.ShortMethod)]
185+
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
186+
=> HwIntrinsics.Shuffle4Reduce(ref source, ref dest, Shuffle.MMShuffle1230);
205187

206188
[MethodImpl(InliningOptions.ShortMethod)]
207189
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)

‎src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs

Lines changed: 13 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
using System.Runtime.CompilerServices;
55
using System.Runtime.InteropServices;
6+
using static SixLabors.ImageSharp.SimdUtils;
67

78
namespace SixLabors.ImageSharp;
89

@@ -13,45 +14,33 @@ internal interface IPad3Shuffle4 : IComponentShuffle
1314

1415
internal readonly struct DefaultPad3Shuffle4 : IPad3Shuffle4
1516
{
16-
private readonly byte p3;
17-
private readonly byte p2;
18-
private readonly byte p1;
19-
private readonly byte p0;
20-
21-
public DefaultPad3Shuffle4(byte p3, byte p2, byte p1, byte p0)
17+
public DefaultPad3Shuffle4(byte control)
2218
{
23-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p3, 0, 3, nameof(p3));
24-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p2, 0, 3, nameof(p2));
25-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p1, 0, 3, nameof(p1));
26-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p0, 0, 3, nameof(p0));
27-
28-
this.p3 = p3;
29-
this.p2 = p2;
30-
this.p1 = p1;
31-
this.p0 = p0;
32-
this.Control = SimdUtils.Shuffle.MmShuffle(p3, p2, p1, p0);
19+
DebugGuard.MustBeBetweenOrEqualTo<byte>(control, 0, 3, nameof(control));
20+
this.Control = control;
3321
}
3422

3523
public byte Control { get; }
3624

25+
[MethodImpl(InliningOptions.ShortMethod)]
26+
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
27+
=> HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref dest, this.Control);
28+
3729
[MethodImpl(InliningOptions.ShortMethod)]
3830
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
3931
{
4032
ref byte sBase = ref MemoryMarshal.GetReference(source);
4133
ref byte dBase = ref MemoryMarshal.GetReference(dest);
4234

43-
int p3 = this.p3;
44-
int p2 = this.p2;
45-
int p1 = this.p1;
46-
int p0 = this.p0;
35+
Shuffle.InverseMMShuffle(this.Control, out int p3, out int p2, out int p1, out int p0);
4736

4837
Span<byte> temp = stackalloc byte[4];
4938
ref byte t = ref MemoryMarshal.GetReference(temp);
5039
ref uint tu = ref Unsafe.As<byte, uint>(ref t);
5140

5241
for (int i = 0, j = 0; i < source.Length; i += 3, j += 4)
5342
{
54-
ref var s = ref Unsafe.Add(ref sBase, i);
43+
ref byte s = ref Unsafe.Add(ref sBase, i);
5544
tu = Unsafe.As<byte, uint>(ref s) | 0xFF000000;
5645

5746
Unsafe.Add(ref dBase, j) = Unsafe.Add(ref t, p0);
@@ -64,11 +53,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
6453

6554
internal readonly struct XYZWPad3Shuffle4 : IPad3Shuffle4
6655
{
67-
public byte Control
68-
{
69-
[MethodImpl(InliningOptions.ShortMethod)]
70-
get => SimdUtils.Shuffle.MmShuffle(3, 2, 1, 0);
71-
}
56+
[MethodImpl(InliningOptions.ShortMethod)]
57+
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
58+
=> HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref dest, Shuffle.MMShuffle3210);
7259

7360
[MethodImpl(InliningOptions.ShortMethod)]
7461
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)

‎src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
using System.Runtime.CompilerServices;
55
using System.Runtime.InteropServices;
6+
using static SixLabors.ImageSharp.SimdUtils;
67

78
namespace SixLabors.ImageSharp;
89

@@ -13,33 +14,25 @@ internal interface IShuffle3 : IComponentShuffle
1314

1415
internal readonly struct DefaultShuffle3 : IShuffle3
1516
{
16-
private readonly byte p2;
17-
private readonly byte p1;
18-
private readonly byte p0;
19-
20-
public DefaultShuffle3(byte p2, byte p1, byte p0)
17+
public DefaultShuffle3(byte control)
2118
{
22-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p2, 0, 2, nameof(p2));
23-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p1, 0, 2, nameof(p1));
24-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p0, 0, 2, nameof(p0));
25-
26-
this.p2 = p2;
27-
this.p1 = p1;
28-
this.p0 = p0;
29-
this.Control = SimdUtils.Shuffle.MmShuffle(3, p2, p1, p0);
19+
DebugGuard.MustBeBetweenOrEqualTo<byte>(control, 0, 3, nameof(control));
20+
this.Control = control;
3021
}
3122

3223
public byte Control { get; }
3324

25+
[MethodImpl(InliningOptions.ShortMethod)]
26+
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
27+
=> HwIntrinsics.Shuffle3Reduce(ref source, ref dest, this.Control);
28+
3429
[MethodImpl(InliningOptions.ShortMethod)]
3530
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
3631
{
3732
ref byte sBase = ref MemoryMarshal.GetReference(source);
3833
ref byte dBase = ref MemoryMarshal.GetReference(dest);
3934

40-
int p2 = this.p2;
41-
int p1 = this.p1;
42-
int p0 = this.p0;
35+
Shuffle.InverseMMShuffle(this.Control, out _, out int p2, out int p1, out int p0);
4336

4437
for (int i = 0; i < source.Length; i += 3)
4538
{

‎src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs

Lines changed: 12 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
using System.Runtime.CompilerServices;
55
using System.Runtime.InteropServices;
6+
using static SixLabors.ImageSharp.SimdUtils;
67

78
namespace SixLabors.ImageSharp;
89

@@ -13,34 +14,25 @@ internal interface IShuffle4Slice3 : IComponentShuffle
1314

1415
internal readonly struct DefaultShuffle4Slice3 : IShuffle4Slice3
1516
{
16-
private readonly byte p2;
17-
private readonly byte p1;
18-
private readonly byte p0;
19-
20-
public DefaultShuffle4Slice3(byte p3, byte p2, byte p1, byte p0)
17+
public DefaultShuffle4Slice3(byte control)
2118
{
22-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p3, 0, 3, nameof(p3));
23-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p2, 0, 3, nameof(p2));
24-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p1, 0, 3, nameof(p1));
25-
DebugGuard.MustBeBetweenOrEqualTo<byte>(p0, 0, 3, nameof(p0));
26-
27-
this.p2 = p2;
28-
this.p1 = p1;
29-
this.p0 = p0;
30-
this.Control = SimdUtils.Shuffle.MmShuffle(p3, p2, p1, p0);
19+
DebugGuard.MustBeBetweenOrEqualTo<byte>(control, 0, 3, nameof(control));
20+
this.Control = control;
3121
}
3222

3323
public byte Control { get; }
3424

25+
[MethodImpl(InliningOptions.ShortMethod)]
26+
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
27+
=> HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref dest, this.Control);
28+
3529
[MethodImpl(InliningOptions.ShortMethod)]
3630
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
3731
{
3832
ref byte sBase = ref MemoryMarshal.GetReference(source);
3933
ref byte dBase = ref MemoryMarshal.GetReference(dest);
4034

41-
int p2 = this.p2;
42-
int p1 = this.p1;
43-
int p0 = this.p0;
35+
Shuffle.InverseMMShuffle(this.Control, out _, out int p2, out int p1, out int p0);
4436

4537
for (int i = 0, j = 0; i < dest.Length; i += 3, j += 4)
4638
{
@@ -53,11 +45,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
5345

5446
internal readonly struct XYZWShuffle4Slice3 : IShuffle4Slice3
5547
{
56-
public byte Control
57-
{
58-
[MethodImpl(InliningOptions.ShortMethod)]
59-
get => SimdUtils.Shuffle.MmShuffle(3, 2, 1, 0);
60-
}
48+
[MethodImpl(InliningOptions.ShortMethod)]
49+
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
50+
=> HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref dest, Shuffle.MMShuffle3210);
6151

6252
[MethodImpl(InliningOptions.ShortMethod)]
6353
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)

‎src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,7 @@ private static void Shuffle4(
297297
// shuffle controls to add to the library.
298298
// We can add static ROS instances if need be in the future.
299299
Span<byte> bytes = stackalloc byte[Vector256<byte>.Count];
300-
Shuffle.MmShuffleSpan(ref bytes, control);
300+
Shuffle.MMShuffleSpan(ref bytes, control);
301301
Vector256<byte> vshuffle = Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(bytes));
302302

303303
ref Vector256<byte> sourceBase =
@@ -333,7 +333,7 @@ private static void Shuffle4(
333333
{
334334
// Ssse3
335335
Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
336-
Shuffle.MmShuffleSpan(ref bytes, control);
336+
Shuffle.MMShuffleSpan(ref bytes, control);
337337
Vector128<byte> vshuffle = Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(bytes));
338338

339339
ref Vector128<byte> sourceBase =
@@ -382,7 +382,7 @@ private static void Shuffle3(
382382
Vector128<byte> vmaske = Ssse3.AlignRight(vmasko, vmasko, 12);
383383

384384
Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
385-
Shuffle.MmShuffleSpan(ref bytes, control);
385+
Shuffle.MMShuffleSpan(ref bytes, control);
386386
Vector128<byte> vshuffle = Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(bytes));
387387

388388
ref Vector128<byte> sourceBase =
@@ -445,7 +445,7 @@ private static void Pad3Shuffle4(
445445
Vector128<byte> vfill = Vector128.Create(0xff000000ff000000ul).AsByte();
446446

447447
Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
448-
Shuffle.MmShuffleSpan(ref bytes, control);
448+
Shuffle.MMShuffleSpan(ref bytes, control);
449449
Vector128<byte> vshuffle = Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(bytes));
450450

451451
ref Vector128<byte> sourceBase =
@@ -489,7 +489,7 @@ private static void Shuffle4Slice3(
489489
Vector128<byte> vmaske = Ssse3.AlignRight(vmasko, vmasko, 12);
490490

491491
Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
492-
Shuffle.MmShuffleSpan(ref bytes, control);
492+
Shuffle.MMShuffleSpan(ref bytes, control);
493493
Vector128<byte> vshuffle = Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(bytes));
494494

495495
ref Vector128<byte> sourceBase =

0 commit comments

Comments
 (0)