Merge branch 'intel:main' into postgres

Author: Raghuveer Devulapalli

.github/workflows/build-numpy.yml

@@ -14,7 +14,7 @@ jobs:
   np-multiarray-tgl:
 
     if: github.repository == 'intel/x86-simd-sort'
-    runs-on: intel-ubuntu-latest
+    runs-on: intel-ubuntu-24.04
 
     steps:
     - name: Checkout x86-simd-sort
@@ -80,7 +80,7 @@ jobs:
   np-multiarray-spr:
 
     if: github.repository == 'intel/x86-simd-sort'
-    runs-on: intel-ubuntu-latest
+    runs-on: intel-ubuntu-24.04
 
     steps:
     - name: Checkout x86-simd-sort

.github/workflows/c-cpp.yml

@@ -11,7 +11,7 @@ permissions: read-all
 jobs:
   SKL-gcc9:
 
-    runs-on: intel-ubuntu-latest
+    runs-on: intel-ubuntu-24.04
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -41,7 +41,7 @@ jobs:
 
   SKX-gcc10:
 
-    runs-on: intel-ubuntu-latest
+    runs-on: intel-ubuntu-24.04
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -71,7 +71,7 @@ jobs:
 
   TGL-gcc11:
 
-    runs-on: intel-ubuntu-latest
+    runs-on: intel-ubuntu-24.04
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -100,7 +100,7 @@ jobs:
 
   SPR-gcc13:
 
-    runs-on: intel-ubuntu-latest
+    runs-on: intel-ubuntu-24.04
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -135,9 +135,41 @@ jobs:
     - name: Run test suite on SPR
       run: sde -spr -- ./builddir/testexe
 
+  SKX-SKL-openmp:
+
+    runs-on: intel-ubuntu-24.04
+
+    steps:
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt -y install g++-10 libgtest-dev meson curl git
+
+    - name: Install Intel SDE
+      run: |
+        curl -o /tmp/sde.tar.xz https://downloadmirror.intel.com/784319/sde-external-9.24.0-2023-07-13-lin.tar.xz
+        mkdir /tmp/sde && tar -xvf /tmp/sde.tar.xz -C /tmp/sde/
+        sudo mv /tmp/sde/* /opt/sde && sudo ln -s /opt/sde/sde64 /usr/bin/sde
+
+    - name: Build
+      env:
+        CXX: g++-10
+      run: |
+        make clean
+        meson setup -Dbuild_tests=true -Duse_openmp=true --warnlevel 2 --werror --buildtype release builddir
+        cd builddir
+        ninja
+
+    - name: Run test suite on SKX and SKL
+      run: |
+        sde -skx -- ./builddir/testexe
+        sde -skl -- ./builddir/testexe
+
   SPR-gcc13-special-cases:
 
-    runs-on: intel-ubuntu-latest
+    runs-on: intel-ubuntu-24.04
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -172,7 +204,7 @@ jobs:
 
   manylinux-32bit:
 
-    runs-on: intel-ubuntu-latest
+    runs-on: intel-ubuntu-24.04
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -184,7 +216,7 @@ jobs:
 
   SPR-icpx:
 
-    runs-on: intel-ubuntu-latest
+    runs-on: intel-ubuntu-24.04
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -194,8 +226,7 @@ jobs:
         echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
         sudo add-apt-repository -y "deb https://apt.repos.intel.com/oneapi all main"
         sudo apt update --allow-insecure-repositories
-        sudo apt --allow-unauthenticated -y install intel-oneapi-compiler-dpcpp-cpp libgtest-dev curl git python3-pip
-        sudo pip3 install meson ninja
+        sudo apt --allow-unauthenticated -y install intel-oneapi-compiler-dpcpp-cpp libgtest-dev curl git python3-pip meson
 
     - name: Install Intel SDE
       run: |

.github/workflows/linting.yml

@@ -11,7 +11,7 @@ permissions: read-all
 jobs:
   clang-format:
 
-    runs-on: intel-ubuntu-latest
+    runs-on: intel-ubuntu-24.04
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

.github/workflows/scorecard.yml

@@ -22,7 +22,7 @@ jobs:
 
     name: Scorecard analysis
     if: github.repository == 'intel/x86-simd-sort'
-    runs-on: ubuntu-latest
+    runs-on: intel-ubuntu-24.04
     permissions:
       # Needed to upload the results to code-scanning dashboard.
       security-events: write

Makefile

@@ -1,10 +1,30 @@
-meson:
-	meson setup -Dbuild_tests=true -Dbuild_benchmarks=true --warnlevel 2 --werror --buildtype release builddir
+test:
+	meson setup -Dbuild_tests=true -Duse_openmp=false --warnlevel 2 --werror --buildtype release builddir
 	cd builddir && ninja
 
-mesondebug:
-	meson setup -Dbuild_tests=true -Dbuild_benchmarks=true --warnlevel 2 --werror --buildtype debug debug
+test_openmp:
+	meson setup -Dbuild_tests=true -Duse_openmp=true --warnlevel 2 --werror --buildtype release builddir
+	cd builddir && ninja
+
+bench:
+	meson setup -Dbuild_benchmarks=true --warnlevel 2 --werror --buildtype release builddir
+	cd builddir && ninja
+
+debug:
+	meson setup -Dbuild_tests=true --warnlevel 2 --werror --buildtype debug debug
 	cd debug && ninja
 
+sharedlib:
+	meson setup --warnlevel 2 --werror --buildtype release builddir
+	cd builddir && ninja
+
+staticlib:
+	meson setup -Dlib_type=static --warnlevel 2 --werror --buildtype release builddir
+	cd builddir && ninja
+
+install:
+	meson setup --warnlevel 2 --werror --buildtype release builddir
+	cd builddir && meson install
+
 clean:
 	$(RM) -rf $(TESTOBJS) $(BENCHOBJS) $(UTILOBJS) testexe benchexe builddir debug

README.md

@@ -45,7 +45,9 @@ int32_t, double, uint64_t, int64_t]`
 
 ## Key-value sort routines on pairs of arrays
 ```cpp
-void x86simdsort::keyvalue_qsort(T1* key, T2* val, size_t size, bool hasnan);
+void x86simdsort::keyvalue_qsort(T1* key, T2* val, size_t size, bool hasnan, bool descending);
+void x86simdsort::keyvalue_select(T1* key, T2* val, size_t k, size_t size, bool hasnan, bool descending);
+void x86simdsort::keyvalue_partial_sort(T1* key, T2* val, size_t k, size_t size, bool hasnan, bool descending);
 ```
 Supported datatypes: `T1`, `T2` $\in$ `[float, uint32_t, int32_t, double,
 uint64_t, int64_t]` Note that keyvalue sort is not yet supported for 16-bit
@@ -78,6 +80,16 @@ benchmark](https://github.com/google/benchmark) frameworks respectively. You
 can configure meson to build them both by using `-Dbuild_tests=true` and
 `-Dbuild_benchmarks=true`.
 
+### Note about building with avx512 by g++ v9 and v10
+
+There is a risk when compile with avx512 by g++ v9 and v10,
+as some `MMX Technology` instructions is used by g++ v9/v10 
+without clearing fpu state. 
+Check [issue 154](https://github.com/intel/x86-simd-sort/issues/154) 
+for more details.
+
+Adding `g++` option `-mno-mmx`, which disables `MMX Technology` instructions, is a possible workaround.
+
 ## Example usage
 
 #### Sort an array of floats
@@ -159,13 +171,11 @@ different metrics:
 
 The performance data (shown in the plot below) can be collected by building the
 benchmarks suite and running `./builddir/benchexe --benchmark_filter==*obj*`.
-The data plot shown below was collected on a processor with AVX-512 because
-`object_qsort` is currently accelerated only on AVX-512 (we plan to add the
-AVX2 version soon). For the simplest of cases where we want to sort an array of
-struct by one of its members, `object_qsort` can be up-to 5x faster for 32-bit
-data type and about 4x for 64-bit data type.  It tends to do even better when
-the metric to sort by gets more complicated. Sorting by Euclidean distance can
-be up-to 10x faster.
+The data plot shown below was collected on a processor with AVX-512. For the
+simplest of cases where we want to sort an array of struct by one of its
+members, `object_qsort` can be up-to 5x faster for 32-bit data type and about
+4x for 64-bit data type.  It tends to do even better when the metric to sort by
+gets more complicated. Sorting by Euclidean distance can be up-to 10x faster.
 
 ![alt text](./misc/object_qsort-perf.jpg?raw=true)
 

benchmarks/bench-keyvalue.hpp

@@ -13,7 +13,8 @@ static void scalarkvsort(benchmark::State &state, Args &&...args)
     std::vector<T> key_bkp = key;
     // benchmark
     for (auto _ : state) {
-        xss::scalar::keyvalue_qsort(key.data(), val.data(), arrsize, false);
+        xss::scalar::keyvalue_qsort(
+                key.data(), val.data(), arrsize, false, false);
         state.PauseTiming();
         key = key_bkp;
         state.ResumeTiming();

lib/meson.build

@@ -1,12 +1,18 @@
 libtargets = []
 
+# Add compile flags for OpenMP if enabled
+openmpflags = []
+if get_option('use_openmp')
+  openmpflags = ['-DXSS_USE_OPENMP=true', '-fopenmp']
+endif
+
 if cpp.has_argument('-march=haswell')
   libtargets += static_library('libavx',
     files(
       'x86simdsort-avx2.cpp',
       ),
     include_directories : [src],
-    cpp_args : ['-march=haswell'],
+    cpp_args : ['-march=haswell', openmpflags],
     gnu_symbol_visibility : 'inlineshidden',
     )
 endif
@@ -17,7 +23,7 @@ if cpp.has_argument('-march=skylake-avx512')
       'x86simdsort-skx.cpp',
       ),
     include_directories : [src],
-    cpp_args : ['-march=skylake-avx512'],
+    cpp_args : ['-march=skylake-avx512', openmpflags],
     gnu_symbol_visibility : 'inlineshidden',
     )
 endif

lib/x86simdsort-avx2.cpp

@@ -34,38 +34,48 @@
         return x86simdsortStatic::argselect(arr, k, arrsize, hasnan); \
     }
 
-#define DEFINE_KEYVALUE_METHODS(type) \
-    template <> \
-    void keyvalue_qsort(type *key, uint64_t *val, size_t arrsize, bool hasnan) \
-    { \
-        x86simdsortStatic::keyvalue_qsort(key, val, arrsize, hasnan); \
-    } \
-    template <> \
-    void keyvalue_qsort(type *key, int64_t *val, size_t arrsize, bool hasnan) \
-    { \
-        x86simdsortStatic::keyvalue_qsort(key, val, arrsize, hasnan); \
-    } \
-    template <> \
-    void keyvalue_qsort(type *key, double *val, size_t arrsize, bool hasnan) \
-    { \
-        x86simdsortStatic::keyvalue_qsort(key, val, arrsize, hasnan); \
-    } \
+#define DEFINE_KEYVALUE_METHODS_BASE(type1, type2) \
     template <> \
-    void keyvalue_qsort(type *key, uint32_t *val, size_t arrsize, bool hasnan) \
+    void keyvalue_qsort(type1 *key, \
+                        type2 *val, \
+                        size_t arrsize, \
+                        bool hasnan, \
+                        bool descending) \
     { \
-        x86simdsortStatic::keyvalue_qsort(key, val, arrsize, hasnan); \
+        x86simdsortStatic::keyvalue_qsort( \
+                key, val, arrsize, hasnan, descending); \
     } \
     template <> \
-    void keyvalue_qsort(type *key, int32_t *val, size_t arrsize, bool hasnan) \
+    void keyvalue_select(type1 *key, \
+                         type2 *val, \
+                         size_t k, \
+                         size_t arrsize, \
+                         bool hasnan, \
+                         bool descending) \
     { \
-        x86simdsortStatic::keyvalue_qsort(key, val, arrsize, hasnan); \
+        x86simdsortStatic::keyvalue_select( \
+                key, val, k, arrsize, hasnan, descending); \
     } \
     template <> \
-    void keyvalue_qsort(type *key, float *val, size_t arrsize, bool hasnan) \
+    void keyvalue_partial_sort(type1 *key, \
+                               type2 *val, \
+                               size_t k, \
+                               size_t arrsize, \
+                               bool hasnan, \
+                               bool descending) \
     { \
-        x86simdsortStatic::keyvalue_qsort(key, val, arrsize, hasnan); \
+        x86simdsortStatic::keyvalue_partial_sort( \
+                key, val, k, arrsize, hasnan, descending); \
     }
 
+#define DEFINE_KEYVALUE_METHODS(type) \
+    DEFINE_KEYVALUE_METHODS_BASE(type, uint64_t) \
+    DEFINE_KEYVALUE_METHODS_BASE(type, int64_t) \
+    DEFINE_KEYVALUE_METHODS_BASE(type, double) \
+    DEFINE_KEYVALUE_METHODS_BASE(type, uint32_t) \
+    DEFINE_KEYVALUE_METHODS_BASE(type, int32_t) \
+    DEFINE_KEYVALUE_METHODS_BASE(type, float)
+
 namespace xss {
 namespace avx2 {
     DEFINE_ALL_METHODS(uint32_t)