Skip to content

Commit 8bdea10

Browse files
author
Raghuveer Devulapalli
committed
Use Macro for GCC unroll pragma
1 parent 4cbd181 commit 8bdea10

File tree

3 files changed

+31
-25
lines changed

3 files changed

+31
-25
lines changed

‎src/avx512-64bit-argsort.hpp‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ X86_SIMD_SORT_INLINE void argsort_32_64bit(type_t *arr, int64_t *arg, int32_t N)
108108
zmm_t arrzmm[4];
109109
argzmm_t argzmm[4];
110110

111-
#pragma GCC unroll 2
111+
#pragma X86_SIMD_SORT_UNROLL_LOOP(2)
112112
for (int ii = 0; ii < 2; ++ii) {
113113
argzmm[ii] = argtype::loadu(arg + 8 * ii);
114114
arrzmm[ii] = vtype::template i64gather<sizeof(type_t)>(argzmm[ii], arr);
@@ -117,7 +117,7 @@ X86_SIMD_SORT_INLINE void argsort_32_64bit(type_t *arr, int64_t *arg, int32_t N)
117117

118118
uint64_t combined_mask = (0x1ull << (N - 16)) - 0x1ull;
119119
opmask_t load_mask[2] = {0xFF, 0xFF};
120-
#pragma GCC unroll 2
120+
#pragma X86_SIMD_SORT_UNROLL_LOOP(2)
121121
for (int ii = 0; ii < 2; ++ii) {
122122
load_mask[ii] = (combined_mask >> (ii * 8)) & 0xFF;
123123
argzmm[ii + 2] = argtype::maskz_loadu(load_mask[ii], arg + 16 + 8 * ii);
@@ -151,7 +151,7 @@ X86_SIMD_SORT_INLINE void argsort_64_64bit(type_t *arr, int64_t *arg, int32_t N)
151151
zmm_t arrzmm[8];
152152
argzmm_t argzmm[8];
153153

154-
#pragma GCC unroll 4
154+
#pragma X86_SIMD_SORT_UNROLL_LOOP(4)
155155
for (int ii = 0; ii < 4; ++ii) {
156156
argzmm[ii] = argtype::loadu(arg + 8 * ii);
157157
arrzmm[ii] = vtype::template i64gather<sizeof(type_t)>(argzmm[ii], arr);
@@ -160,7 +160,7 @@ X86_SIMD_SORT_INLINE void argsort_64_64bit(type_t *arr, int64_t *arg, int32_t N)
160160

161161
opmask_t load_mask[4] = {0xFF, 0xFF, 0xFF, 0xFF};
162162
uint64_t combined_mask = (0x1ull << (N - 32)) - 0x1ull;
163-
#pragma GCC unroll 4
163+
#pragma X86_SIMD_SORT_UNROLL_LOOP(4)
164164
for (int ii = 0; ii < 4; ++ii) {
165165
load_mask[ii] = (combined_mask >> (ii * 8)) & 0xFF;
166166
argzmm[ii + 4] = argtype::maskz_loadu(load_mask[ii], arg + 32 + 8 * ii);
@@ -170,7 +170,7 @@ X86_SIMD_SORT_INLINE void argsort_64_64bit(type_t *arr, int64_t *arg, int32_t N)
170170
argzmm[ii + 4]);
171171
}
172172

173-
#pragma GCC unroll 4
173+
#pragma X86_SIMD_SORT_UNROLL_LOOP(4)
174174
for (int ii = 0; ii < 8; ii = ii + 2) {
175175
bitonic_merge_two_zmm_64bit<vtype, argtype>(
176176
arrzmm[ii], arrzmm[ii + 1], argzmm[ii], argzmm[ii + 1]);
@@ -179,11 +179,11 @@ X86_SIMD_SORT_INLINE void argsort_64_64bit(type_t *arr, int64_t *arg, int32_t N)
179179
bitonic_merge_four_zmm_64bit<vtype, argtype>(arrzmm + 4, argzmm + 4);
180180
bitonic_merge_eight_zmm_64bit<vtype, argtype>(arrzmm, argzmm);
181181

182-
#pragma GCC unroll 4
182+
#pragma X86_SIMD_SORT_UNROLL_LOOP(4)
183183
for (int ii = 0; ii < 4; ++ii) {
184184
argtype::storeu(arg + 8 * ii, argzmm[ii]);
185185
}
186-
#pragma GCC unroll 4
186+
#pragma X86_SIMD_SORT_UNROLL_LOOP(4)
187187
for (int ii = 0; ii < 4; ++ii) {
188188
argtype::mask_storeu(arg + 32 + 8 * ii, load_mask[ii], argzmm[ii + 4]);
189189
}
@@ -203,7 +203,7 @@ X86_SIMD_SORT_INLINE void argsort_64_64bit(type_t *arr, int64_t *arg, int32_t N)
203203
// zmm_t arrzmm[16];
204204
// argzmm_t argzmm[16];
205205
//
206-
//#pragma GCC unroll 8
206+
//#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
207207
// for (int ii = 0; ii < 8; ++ii) {
208208
// argzmm[ii] = argtype::loadu(arg + 8*ii);
209209
// arrzmm[ii] = vtype::template i64gather<sizeof(type_t)>(argzmm[ii], arr);
@@ -213,19 +213,19 @@ X86_SIMD_SORT_INLINE void argsort_64_64bit(type_t *arr, int64_t *arg, int32_t N)
213213
// opmask_t load_mask[8] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
214214
// if (N != 128) {
215215
// uint64_t combined_mask = (0x1ull << (N - 64)) - 0x1ull;
216-
//#pragma GCC unroll 8
216+
//#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
217217
// for (int ii = 0; ii < 8; ++ii) {
218218
// load_mask[ii] = (combined_mask >> (ii*8)) & 0xFF;
219219
// }
220220
// }
221-
//#pragma GCC unroll 8
221+
//#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
222222
// for (int ii = 0; ii < 8; ++ii) {
223223
// argzmm[ii+8] = argtype::maskz_loadu(load_mask[ii], arg + 64 + 8*ii);
224224
// arrzmm[ii+8] = vtype::template mask_i64gather<sizeof(type_t)>(vtype::zmm_max(), load_mask[ii], argzmm[ii+8], arr);
225225
// arrzmm[ii+8] = sort_zmm_64bit<vtype, argtype>(arrzmm[ii+8], argzmm[ii+8]);
226226
// }
227227
//
228-
//#pragma GCC unroll 8
228+
//#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
229229
// for (int ii = 0; ii < 16; ii = ii + 2) {
230230
// bitonic_merge_two_zmm_64bit<vtype, argtype>(arrzmm[ii], arrzmm[ii + 1], argzmm[ii], argzmm[ii + 1]);
231231
// }
@@ -237,11 +237,11 @@ X86_SIMD_SORT_INLINE void argsort_64_64bit(type_t *arr, int64_t *arg, int32_t N)
237237
// bitonic_merge_eight_zmm_64bit<vtype, argtype>(arrzmm+8, argzmm+8);
238238
// bitonic_merge_sixteen_zmm_64bit<vtype, argtype>(arrzmm, argzmm);
239239
//
240-
//#pragma GCC unroll 8
240+
//#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
241241
// for (int ii = 0; ii < 8; ++ii) {
242242
// argtype::storeu(arg + 8*ii, argzmm[ii]);
243243
// }
244-
//#pragma GCC unroll 8
244+
//#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
245245
// for (int ii = 0; ii < 8; ++ii) {
246246
// argtype::mask_storeu(arg + 64 + 8*ii, load_mask[ii], argzmm[ii + 8]);
247247
// }

‎src/avx512-common-argsort.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ static inline int64_t partition_avx512_unrolled(type_t *arr,
198198
// first and last vtype::numlanes values are partitioned at the end
199199
zmm_t vec_left[num_unroll], vec_right[num_unroll];
200200
argzmm_t argvec_left[num_unroll], argvec_right[num_unroll];
201-
#pragma GCC unroll 8
201+
#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
202202
for (int ii = 0; ii < num_unroll; ++ii) {
203203
argvec_left[ii] = argtype::loadu(arg + left + vtype::numlanes * ii);
204204
vec_left[ii] = vtype::template i64gather<sizeof(type_t)>(
@@ -224,7 +224,7 @@ static inline int64_t partition_avx512_unrolled(type_t *arr,
224224
*/
225225
if ((r_store + vtype::numlanes) - right < left - l_store) {
226226
right -= num_unroll * vtype::numlanes;
227-
#pragma GCC unroll 8
227+
#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
228228
for (int ii = 0; ii < num_unroll; ++ii) {
229229
arg_vec[ii]
230230
= argtype::loadu(arg + right + ii * vtype::numlanes);
@@ -233,7 +233,7 @@ static inline int64_t partition_avx512_unrolled(type_t *arr,
233233
}
234234
}
235235
else {
236-
#pragma GCC unroll 8
236+
#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
237237
for (int ii = 0; ii < num_unroll; ++ii) {
238238
arg_vec[ii] = argtype::loadu(arg + left + ii * vtype::numlanes);
239239
curr_vec[ii] = vtype::template i64gather<sizeof(type_t)>(
@@ -242,7 +242,7 @@ static inline int64_t partition_avx512_unrolled(type_t *arr,
242242
left += num_unroll * vtype::numlanes;
243243
}
244244
// partition the current vector and save it on both sides of the array
245-
#pragma GCC unroll 8
245+
#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
246246
for (int ii = 0; ii < num_unroll; ++ii) {
247247
int32_t amount_gt_pivot
248248
= partition_vec<vtype>(arg,
@@ -259,7 +259,7 @@ static inline int64_t partition_avx512_unrolled(type_t *arr,
259259
}
260260

261261
/* partition and save vec_left and vec_right */
262-
#pragma GCC unroll 8
262+
#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
263263
for (int ii = 0; ii < num_unroll; ++ii) {
264264
int32_t amount_gt_pivot
265265
= partition_vec<vtype>(arg,
@@ -273,7 +273,7 @@ static inline int64_t partition_avx512_unrolled(type_t *arr,
273273
l_store += (vtype::numlanes - amount_gt_pivot);
274274
r_store -= amount_gt_pivot;
275275
}
276-
#pragma GCC unroll 8
276+
#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
277277
for (int ii = 0; ii < num_unroll; ++ii) {
278278
int32_t amount_gt_pivot
279279
= partition_vec<vtype>(arg,

‎src/avx512-common-qsort.h‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,12 @@
8787

8888
#define LIKELY(x) __builtin_expect((x), 1)
8989
#define UNLIKELY(x) __builtin_expect((x), 0)
90+
#if __GNUC__ >= 8
91+
#define X86_SIMD_SORT_UNROLL_LOOP(num)\
92+
GCC unroll num
93+
#else
94+
#define X86_SIMD_SORT_UNROLL_LOOP(num)
95+
#endif
9096

9197
template <typename type>
9298
struct zmm_vector;
@@ -382,7 +388,7 @@ static inline int64_t partition_avx512_unrolled(type_t *arr,
382388
// We will now have atleast 16 registers worth of data to process:
383389
// left and right vtype::numlanes values are partitioned at the end
384390
zmm_t vec_left[num_unroll], vec_right[num_unroll];
385-
#pragma GCC unroll 8
391+
#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
386392
for (int ii = 0; ii < num_unroll; ++ii) {
387393
vec_left[ii] = vtype::loadu(arr + left + vtype::numlanes * ii);
388394
vec_right[ii] = vtype::loadu(
@@ -403,20 +409,20 @@ static inline int64_t partition_avx512_unrolled(type_t *arr,
403409
*/
404410
if ((r_store + vtype::numlanes) - right < left - l_store) {
405411
right -= num_unroll * vtype::numlanes;
406-
#pragma GCC unroll 8
412+
#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
407413
for (int ii = 0; ii < num_unroll; ++ii) {
408414
curr_vec[ii] = vtype::loadu(arr + right + ii * vtype::numlanes);
409415
}
410416
}
411417
else {
412-
#pragma GCC unroll 8
418+
#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
413419
for (int ii = 0; ii < num_unroll; ++ii) {
414420
curr_vec[ii] = vtype::loadu(arr + left + ii * vtype::numlanes);
415421
}
416422
left += num_unroll * vtype::numlanes;
417423
}
418424
// partition the current vector and save it on both sides of the array
419-
#pragma GCC unroll 8
425+
#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
420426
for (int ii = 0; ii < num_unroll; ++ii) {
421427
int32_t amount_ge_pivot
422428
= partition_vec<vtype>(arr,
@@ -432,7 +438,7 @@ static inline int64_t partition_avx512_unrolled(type_t *arr,
432438
}
433439

434440
/* partition and save vec_left[8] and vec_right[8] */
435-
#pragma GCC unroll 8
441+
#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
436442
for (int ii = 0; ii < num_unroll; ++ii) {
437443
int32_t amount_ge_pivot
438444
= partition_vec<vtype>(arr,
@@ -445,7 +451,7 @@ static inline int64_t partition_avx512_unrolled(type_t *arr,
445451
l_store += (vtype::numlanes - amount_ge_pivot);
446452
r_store -= amount_ge_pivot;
447453
}
448-
#pragma GCC unroll 8
454+
#pragma X86_SIMD_SORT_UNROLL_LOOP(8)
449455
for (int ii = 0; ii < num_unroll; ++ii) {
450456
int32_t amount_ge_pivot
451457
= partition_vec<vtype>(arr,

0 commit comments

Comments
 (0)