Use IPS4o in-place sorting

Author: pmenon

src/codegen/code_context.cpp

@@ -142,6 +142,7 @@ class InstructionCounts : public llvm::ModulePass {
   }
 
   void DumpStats() const {
+#ifndef NDEBUG
     LOG_DEBUG("# functions: %" PRId64 " (%" PRId64
               " external), # blocks: %" PRId64 ", # instructions: %" PRId64,
               func_count_, external_func_count_, basic_block_count_,
@@ -150,6 +151,7 @@ class InstructionCounts : public llvm::ModulePass {
       const char *inst_name = llvm::Instruction::getOpcodeName(iter.first);
       LOG_DEBUG("↳ %s: %" PRId64, inst_name, iter.second);
     }
+#endif
   }
 
  private:

src/codegen/compact_storage.cpp

@@ -12,6 +12,8 @@
 
 #include "codegen/compact_storage.h"
 
+#include "ips4o/ips4o.hpp"
+
 #include "codegen/type/sql_type.h"
 
 namespace peloton {
@@ -138,11 +140,10 @@ llvm::Type *CompactStorage::Setup(CodeGen &codegen,
 
   // Sort the entries by decreasing size. This minimizes storage overhead due to
   // padding (potentially) added by LLVM.
-  // TODO: Does this help?
-  std::sort(storage_format_.begin(), storage_format_.end(),
-            [](const EntryInfo &left, const EntryInfo &right) {
-              return right.num_bytes < left.num_bytes;
-            });
+  ips4o::sort(storage_format_.begin(), storage_format_.end(),
+              [](const EntryInfo &left, const EntryInfo &right) {
+                return right.num_bytes < left.num_bytes;
+              });
 
   // Now we construct the LLVM type of this storage space. First comes bytes
   // to manage the null bitmap. Then all the data elements.

src/codegen/updateable_storage.cpp

@@ -12,7 +12,7 @@
 
 #include "codegen/updateable_storage.h"
 
-#include "pdqsort/pdqsort.h"
+#include "ips4o/ips4o.hpp"
 
 #include "codegen/lang/if.h"
 #include "codegen/type/sql_type.h"
@@ -66,11 +66,11 @@ llvm::Type *UpdateableStorage::Finalize(CodeGen &codegen) {
   }
 
   // Sort the entries by decreasing size
-  pdqsort(storage_format_.begin(), storage_format_.end(),
-          [](const CompactStorage::EntryInfo &left,
-             const CompactStorage::EntryInfo &right) {
-            return right.num_bytes < left.num_bytes;
-          });
+  ips4o::sort(storage_format_.begin(), storage_format_.end(),
+              [](const CompactStorage::EntryInfo &left,
+                 const CompactStorage::EntryInfo &right) {
+                return right.num_bytes < left.num_bytes;
+              });
 
   // Now we construct the LLVM type of this storage space
   std::vector<llvm::Type *> llvm_types;

src/codegen/util/sorter.cpp

@@ -14,7 +14,7 @@
 
 #include <queue>
 
-#include "pdqsort/pdqsort.h"
+#include "ips4o/ips4o.hpp"
 
 #include "common/synchronization/count_down_latch.h"
 #include "common/timer.h"
@@ -88,7 +88,7 @@ void Sorter::Sort() {
 
   // Sort the sucker
   auto cmp = [this](char *l, char *r) { return cmp_func_(l, r) < 0; };
-  pdqsort(tuples_.begin(), tuples_.end(), cmp);
+  ips4o::sort(tuples_.begin(), tuples_.end(), cmp);
 
   // Setup pointers
   tuples_start_ = tuples_.data();
@@ -170,7 +170,7 @@ void Sorter::SortParallel(
     // Sort the local separators and choose the median
     char *separator = nullptr;
     if (idx < separators.size() - 1) {
-      pdqsort(separators[idx].begin(), separators[idx].end(), cmp);
+      ips4o::sort(separators[idx].begin(), separators[idx].end(), cmp);
       separator = separators[idx][sorters.size() / 2];
     }
 

src/threadpool/worker_pool.cpp

@@ -24,7 +24,7 @@ void WorkerFunc(std::string thread_name, std::atomic_bool *is_running,
   constexpr auto kMinPauseTime = std::chrono::microseconds(1);
   constexpr auto kMaxPauseTime = std::chrono::microseconds(1000);
 
-  LOG_DEBUG("Thread %s starting ...", thread_name.c_str());
+  LOG_INFO("Thread %s starting ...", thread_name.c_str());
 
   auto pause_time = kMinPauseTime;
   while (is_running->load() || !task_queue->IsEmpty()) {
@@ -39,7 +39,7 @@ void WorkerFunc(std::string thread_name, std::atomic_bool *is_running,
     }
   }
 
-  LOG_DEBUG("Thread %s exiting ...", thread_name.c_str());
+  LOG_INFO("Thread %s exiting ...", thread_name.c_str());
 }
 
 }  // namespace

test/codegen/sorter_test.cpp

@@ -139,7 +139,7 @@ TEST_F(SorterTest, ParallelSortTest) {
   sorter_.SortParallel(thread_states, 0);
 
   timer.Stop();
-  LOG_DEBUG("Parallel sort took: %.2lf ms", timer.GetDuration());
+  LOG_INFO("Parallel sort took: %.2lf ms", timer.GetDuration());
 
   // Check main sorter is sorted
   CheckSorted(sorter_, true);

third_party/ips4o/LICENSE

@@ -0,0 +1,27 @@
+BSD 2-Clause License
+
+Copyright © 2017, Michael Axtmann <[email protected]>
+Copyright © 2017, Daniel Ferizovic <[email protected]>
+Copyright © 2017, Sascha Witt <[email protected]>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

third_party/ips4o/README.md

@@ -0,0 +1,35 @@
+# In-place Parallel Super Scalar Samplesort (IPS⁴o)
+
+This is the implementation of the algorithm presented in the [eponymous paper](https://arxiv.org/abs/1705.02257),
+which contains an in-depth description of its inner workings, as well as an extensive experimental performance evaluation.
+Here's the abstract:
+
+> We present a sorting algorithm that works in-place, executes in parallel, is
+> cache-efficient, avoids branch-mispredictions, and performs work O(n log n) for
+> arbitrary inputs with high probability. The main algorithmic contributions are
+> new ways to make distribution-based algorithms in-place: On the practical side,
+> by using coarse-grained block-based permutations, and on the theoretical side,
+> we show how to eliminate the recursion stack. Extensive experiments show that
+> our algorithm IPS⁴o scales well on a variety of multi-core machines. We
+> outperform our closest in-place competitor by a factor of up to 3. Even as
+> a sequential algorithm, we are up to 1.5 times faster than the closest
+> sequential competitor, BlockQuicksort.
+
+## Usage
+
+```C++
+#include "ips4o.hpp"
+
+// sort sequentially
+ips4o::sort(begin, end[, comparator])
+
+// sort in parallel (uses OpenMP if available, std::thread otherwise)
+ips4o::parallel::sort(begin, end[, comparator])
+```
+
+Make sure to compile with C++14 support. Currently, the code does not compile on Windows.
+
+For the parallel algorithm, you need to enable either OpenMP (`-fopenmp`) or C++ threads (e.g., `-pthread`).
+You also need a CPU that supports 16-byte compare-and-exchange instructions.
+If you get undefined references to `__atomic_fetch_add_16`, either set your CPU correctly (e.g., `-march=native`),
+  enable the instructions explicitly (`-mcx16`), or try linking against GCC's libatomic (`-latomic`).

third_party/ips4o/ips4o.hpp

@@ -0,0 +1,38 @@
+/******************************************************************************
+ * ips4o.hpp
+ *
+ * In-place Parallel Super Scalar Samplesort (IPS⁴o)
+ *
+ ******************************************************************************
+ * BSD 2-Clause License
+ *
+ * Copyright © 2017, Michael Axtmann <[email protected]>
+ * Copyright © 2017, Daniel Ferizovic <[email protected]>
+ * Copyright © 2017, Sascha Witt <[email protected]>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *****************************************************************************/
+
+#pragma once
+
+#include "ips4o/ips4o.hpp"

third_party/ips4o/ips4o/base_case.hpp

@@ -0,0 +1,82 @@
+/******************************************************************************
+ * ips4o/base_case.hpp
+ *
+ * In-place Parallel Super Scalar Samplesort (IPS⁴o)
+ *
+ ******************************************************************************
+ * BSD 2-Clause License
+ *
+ * Copyright © 2017, Michael Axtmann <[email protected]>
+ * Copyright © 2017, Daniel Ferizovic <[email protected]>
+ * Copyright © 2017, Sascha Witt <[email protected]>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *****************************************************************************/
+
+#pragma once
+
+#include <algorithm>
+#include <cstddef>
+#include <utility>
+
+#include "ips4o_fwd.hpp"
+#include "utils.hpp"
+
+namespace ips4o {
+namespace detail {
+
+/**
+ * Insertion sort.
+ */
+template <class It, class Comp>
+void insertionSort(const It begin, const It end, Comp comp) {
+    IPS4O_ASSUME_NOT(begin >= end);
+
+    for (It it = begin + 1; it < end; ++it) {
+        auto val = std::move(*it);
+        if (comp(val, *begin)) {
+            std::move_backward(begin, it, it + 1);
+            *begin = std::move(val);
+        } else {
+            auto cur = it;
+            for (auto next = it - 1; comp(val, *next); --next) {
+                *cur = std::move(*next);
+                cur = next;
+            }
+            *cur = std::move(val);
+        }
+    }
+}
+
+/**
+ * Wrapper for base case sorter, for easier swapping.
+ */
+template <class It, class Comp>
+inline void baseCaseSort(It begin, It end, Comp&& comp) {
+    if (begin == end) return;
+    detail::insertionSort(std::move(begin), std::move(end), std::forward<Comp>(comp));
+}
+
+
+}  // namespace detail
+}  // namespace ips4o