WIP

Author: Lukas955

src/tools/fdsdump/src/3rd_party/xxhash/xxhash.h

@@ -3,12 +3,14 @@ include_directories("${CMAKE_CURRENT_SOURCE_DIR}")
 # Subdirectories with components
 add_subdirectory(common)
 add_subdirectory(lister)
+add_subdirectory(aggregator)
 
 add_executable(fdsdump
     main.cpp
     options.cpp
     $<TARGET_OBJECTS:common_obj>
 	$<TARGET_OBJECTS:lister_obj>
+    $<TARGET_OBJECTS:aggregator_obj>
 )
 
 target_include_directories(fdsdump

src/tools/fdsdump/src/CMakeLists.txt

@@ -0,0 +1,8 @@
+# Create an aggregator "object" library
+set(AGGREGATOR_SRC
+    aggregator.cpp
+    arenaAllocator.cpp
+    hashTable.cpp
+)
+
+add_library(aggregator_obj OBJECT ${AGGREGATOR_SRC})

src/tools/fdsdump/src/aggregator/CMakeLists.txt

@@ -0,0 +1,9 @@
+
+#include "aggregator.hpp"
+
+void
+mode_aggregate(const shared_iemgr &iemgr, const Options &opts)
+{
+
+}
+

src/tools/fdsdump/src/aggregator/aggregator.cpp

@@ -0,0 +1,8 @@
+
+#pragma once
+
+#include <options.hpp>
+#include <common/flowProvider.hpp>
+
+void
+mode_aggregate(const shared_iemgr &iemgr, const Options &opts);

src/tools/fdsdump/src/aggregator/aggregator.hpp

@@ -0,0 +1,23 @@
+/**
+ * @file
+ * @author Michal Sedlak <[email protected]>
+ * @brief Simple arena allocator
+ */
+#include "arenaAllocator.hpp"
+
+#include <cassert>
+
+uint8_t *
+ArenaAllocator::allocate(size_t size)
+{
+    assert(size <= BLOCK_SIZE);
+
+    if (BLOCK_SIZE - m_offset < size) {
+        m_offset = 0;
+        m_blocks.push_back(std::unique_ptr<uint8_t []>(new uint8_t[BLOCK_SIZE]));
+    }
+
+    uint8_t *mem = m_blocks.back().get() + m_offset;
+    m_offset += size;
+    return mem;
+}

src/tools/fdsdump/src/aggregator/arenaAllocator.cpp

@@ -0,0 +1,39 @@
+/**
+ * @file
+ * @author Michal Sedlak <[email protected]>
+ * @brief Simple arena allocator
+ */
+#pragma once
+
+#include <memory>
+#include <vector>
+
+static constexpr size_t BLOCK_SIZE = 4 * 1024 * 1024;
+
+/**
+ * @brief A simple arena allocator.
+ *
+ * Provides easy allocation in continuous memory areas and the ability to deallocate all allocated
+ * memory upon destruction.
+ */
+class ArenaAllocator {
+public:
+    ArenaAllocator() {}
+
+    /** @brief Disallow copy constructor. */
+    ArenaAllocator(const ArenaAllocator &) = delete;
+    /** @brief Disallow move constructor. */
+    ArenaAllocator(ArenaAllocator &&) = delete;
+
+    /**
+     * @brief Allocate bytes.
+     * @param[in] size  The number of bytes
+     * @warning size cannot be more than BLOCK_SIZE!
+     * @return Pointer to the bytes
+     */
+    uint8_t *allocate(size_t size);
+
+private:
+    std::vector<std::unique_ptr<uint8_t []>> m_blocks;
+    size_t m_offset = BLOCK_SIZE;
+};

src/tools/fdsdump/src/aggregator/arenaAllocator.hpp

@@ -0,0 +1,124 @@
+/**
+ * \file
+ * \author Michal Sedlak <[email protected]>
+ * \brief Generic binary heap
+ */
+#pragma once
+
+#include <vector>
+#include <algorithm>
+
+/**
+ * @brief A generic binary heap implementation. It can be used for example
+ *   as a priority queue.
+ * @tparam ItemType  The item type
+ * @tparam CompareFn The comparator function type
+ */
+template <typename ItemType, typename CompareFn>
+class BinaryHeap {
+public:
+    /**
+     * @brief Constructs a new instance.
+     * @param[in] compare The comparator
+     */
+    BinaryHeap(CompareFn compare) : m_compare(compare) {}
+
+    /**
+     * @brief Push an item onto the heap.
+     * @param[in] item The item
+     */
+    void
+    push(ItemType item)
+    {
+        m_items.push_back(item);
+        sift_up(m_items.size() - 1);
+    }
+
+    /**
+     * @brief Push an item onto the heap and also pop one off.
+     * @param[in] item  The item to push
+     * @return The popped of item
+     */
+    ItemType
+    push_pop(ItemType item)
+    {
+        if (m_compare(item, m_items[0])) {
+            ItemType result = m_items[0];
+            m_items[0] = item;
+            sift_down(0);
+            return result;
+
+        } else {
+            return item;
+        }
+    }
+
+    /**
+     * @brief Pop an item off the top of the heap.
+     * @return The item
+     */
+    ItemType
+    pop()
+    {
+        ItemType result = m_items[0];
+        m_items[0] = m_items[m_items.size() - 1];
+        m_items.pop_back();
+        sift_down(0);
+        return result;
+    }
+
+    /**
+     * @brief Get the item at the top of the heap.
+     * @return The item
+     */
+    ItemType top() const { return m_items[0]; }
+
+    /**
+     * @brief Get the size of the heap
+     * @return The number of items the heap holds
+     */
+    size_t size() const { return m_items.size(); }
+
+private:
+    std::vector<ItemType> m_items;
+    CompareFn m_compare;
+
+    void
+    sift_up(size_t idx)
+    {
+        while (idx > 0) {
+            size_t parent = (idx - 1) / 2;
+            if (m_compare(m_items[parent], m_items[idx])) {
+                std::swap(m_items[parent], m_items[idx]);
+                idx = parent;
+            } else {
+                break;
+            }
+        }
+    }
+
+    void
+    sift_down(size_t idx)
+    {
+        for (;;) {
+            size_t left = 2 * idx + 1;
+            size_t right = 2 * idx + 2;
+            size_t smallest = idx;
+
+            if (left < m_items.size() && m_compare(m_items[smallest], m_items[left])) {
+                smallest = left;
+            }
+            if (right < m_items.size() && m_compare(m_items[smallest], m_items[right])) {
+                smallest = right;
+            }
+
+            if (smallest == idx) {
+                break;
+            }
+
+            std::swap(m_items[smallest], m_items[idx]);
+
+            idx = smallest;
+        }
+    }
+};

src/tools/fdsdump/src/aggregator/binaryHeap.hpp

@@ -0,0 +1,153 @@
+/**
+ * @file
+ * @author Michal Sedlak <[email protected]>
+ * @brief Efficient hash table implementation
+ */
+
+#define XXH_INLINE_ALL
+
+#include <xmmintrin.h>
+
+#include "hashTable.hpp"
+#include "3rd_party/xxhash/xxhash.h"
+
+static constexpr double EXPAND_WHEN_THIS_FULL = 0.95;
+static constexpr unsigned int EXPAND_WITH_FACTOR_OF = 2;
+static constexpr uint8_t EMPTY_BIT = 0x80;
+
+HashTable::HashTable(std::size_t key_size, std::size_t value_size) :
+    m_key_size(key_size), m_value_size(value_size)
+{
+    init_blocks();
+}
+
+void
+HashTable::init_blocks()
+{
+    HashTableBlock zeroed_block;
+
+    memset(&zeroed_block, 0, sizeof(zeroed_block));
+    for (int i = 0; i < 16; i++) {
+        zeroed_block.tags[i] |= EMPTY_BIT; // Indicate that the spot is empty
+    }
+
+    m_blocks.resize(m_block_count);
+    for (auto &block : m_blocks) {
+        block = zeroed_block;
+    }
+}
+
+bool
+HashTable::lookup(uint8_t *key, uint8_t *&item, bool create_if_not_found)
+{
+    uint64_t hash = XXH3_64bits(key, m_key_size); // The hash of the key
+    uint64_t index = (hash >> 7) & (m_block_count - 1); // The starting block index
+
+    for (;;) {
+        HashTableBlock &block = m_blocks[index];
+
+        uint8_t item_tag = (hash & 0xFF) & ~EMPTY_BIT; // Get item tag from part of the hash with the empty bit cleared
+        auto block_tags = _mm_load_si128(reinterpret_cast<__m128i *>(block.tags)); // Load the current block metadata (16B)
+        auto hash_mask = _mm_set1_epi8(item_tag); // Repeat the item tag 16 times
+        auto empty_mask = _mm_set1_epi8(EMPTY_BIT); // Repeat the empty tag 16 times
+
+        auto hash_match = _mm_movemask_epi8(_mm_cmpeq_epi8(block_tags, hash_mask)); // Check if any of the metadata matched our item tag
+        auto empty_match = _mm_movemask_epi8(_mm_cmpeq_epi8(block_tags, empty_mask)); // Check if any of the metadata matched an empty spot
+
+        int item_index = 0;
+
+        while (hash_match) { // While there are any set bits indicating that the tag of the item we're looking for matched
+            auto one_index = __builtin_ctz(hash_match); // Get index of the set bit, i.e. the index of the item in the block
+            item_index += one_index;
+
+            uint8_t *record = block.items[item_index]; // The record whose item tag matched
+            if (memcmp(record, key, m_key_size) == 0) { // Does the key match as well or was it just a hash collision?
+                item = record;
+                return true; // We found the item
+            }
+
+            // Move on to the next set bit
+            hash_match >>= one_index + 1;
+            item_index += 1;
+        }
+
+        // If we got here we didn't match, but we found an empty spot in the block which
+        // indicates that we're done with the search. The item cannot be in the next block
+        // if the current block contains an empty spot
+        if (empty_match) {
+
+            if (!create_if_not_found) {
+                // If we're just looking for the item and we haven't found it, we're done
+                return false;
+            }
+
+            // Create a new record
+            auto empty_index = __builtin_ctz(empty_match);
+            block.tags[empty_index] = item_tag;
+
+            uint8_t *record = m_allocator.allocate(m_key_size + m_value_size);
+            block.items[empty_index] = record;
+            m_items.push_back(record);
+            m_record_count++;
+
+            memcpy(record, key, m_key_size); // Copy the key, leave the value part uninitialized
+            item = record;
+
+            // If the hash table has reached a specified percentage of fullness, expand the hash table
+            if (double(m_record_count) / (16 * double(m_block_count)) >= EXPAND_WHEN_THIS_FULL) {
+                expand();
+            }
+
+            return false;
+        }
+
+        index = (index + 1) & (m_block_count - 1); // Move on to the next block
+    }
+}
+
+void
+HashTable::expand()
+{
+    // Grow the amount of blocks by a specified factor
+    m_block_count *= EXPAND_WITH_FACTOR_OF;
+
+    // Reinitialize the blocks
+    init_blocks();
+
+    // Reassign all the items to the newly initialized blocks
+    for (uint8_t *item : m_items) {
+        uint64_t hash = XXH3_64bits(item, m_key_size);
+        uint64_t index = (hash >> 7) & (m_block_count - 1);
+        uint8_t item_tag = (hash & 0xFF) & ~EMPTY_BIT;
+
+        // Find a spot for the item and insert it
+        for (;;) {
+            HashTableBlock &block = m_blocks[index];
+
+            auto block_tags = _mm_load_si128(reinterpret_cast<__m128i *>(block.tags));
+            auto empty_mask = _mm_set1_epi8(EMPTY_BIT);
+            auto empty_match = _mm_movemask_epi8(_mm_cmpeq_epi8(block_tags, empty_mask));
+            if (empty_match) { // Does this black have an empty spot for our item?
+                auto empty_index = __builtin_ctz(empty_match);
+                block.tags[empty_index] = item_tag;
+                block.items[empty_index] = item;
+                break;
+            }
+
+            index = (index + 1) & (m_block_count - 1);
+        }
+
+    }
+}
+
+bool
+HashTable::find(uint8_t *key, uint8_t *&item)
+{
+    return lookup(key, item, false);
+}
+
+bool
+HashTable::find_or_create(uint8_t *key, uint8_t *&item)
+{
+    return lookup(key, item, true);
+}