Huffman dictionary to stream

Wosser1sProductions · Wosser1sProductions · commit a6b3359db073 · 2018-10-19T18:19:25.000+02:00
+ Huffman dictionary can now correctly be written to a stream, with minimal amount of bits
+ Huffman dictionary can be correctly read from stream
TODO: Proper decoding function, dict to tree conversion, coe clean-up and integration with Encoder
diff --git a/Debug/Debug.bg b/Debug/Debug.bg
diff --git a/Huffman.cpp b/Huffman.cpp
@@ -5,17 +5,74 @@
 #include "utils.hpp"
 #include "Logger.hpp"
 
+////////////////////////////////////////////////////////////////////////////////
+///  Private functions
+////////////////////////////////////////////////////////////////////////////////
 
-template<class T>
-algo::Huffman<T>::Huffman(void) {
+/**
+ *  @brief  A comparator to sort Codeword pairs by bit length.
+ */
+struct CodewordComparator {
+    bool operator()(const std::pair<uint8_t, algo::Codeword>& first,
+                    const std::pair<uint8_t, algo::Codeword>& second)
+    {
+        return first.second.len > second.second.len;
+    }
+};
 
+/**
+ *  @brief  Add the given settings to the output stream according to the amount of bits
+ *          specified in the Huffman class.
+ *  @param  length
+ *      The length of the sequence that will follow this header.
+ *      If the length is 0, only one '0' bit will be written.
+ *  @param  bit_length
+ *      The amount of bits needed for every data element in the sequence following this header.
+ *      Keys always use KEY_BITS as length, and values use bit_length, which is different for each group.
+ *      This is done to minimize the amoutn of bits needed to save the Huffman dictionary.
+ *  @param  writer
+ *      The outputstream to write to.
+ */
+template<class T>
+void algo::Huffman<T>::add_huffman_dict_header(uint32_t length, uint32_t bit_length, util::BitStreamWriter& writer) {
+    if (length > 0) {
+        writer.put(algo::Huffman<T>::DICT_HDR_HAS_ITEMS_BITS + algo::Huffman<T>::DICT_HDR_SEQ_LENGTH_BITS,
+                   0x80 | (length & 0x7F)); // MSB is HAS_ITEMS setting + 7 bits length
+        writer.put(algo::Huffman<T>::DICT_HDR_ITEM_BITS,
+                   bit_length & 0xF);       // 4 bits for bit length of every dict item
+    } else {
+        writer.put_bit(0);
+    }
 }
 
+/**
+ *  @brief  Read a dictionary header from the inputstream and set the given variables.
+ *
+ *  @param  reader
+ *      The inputstream to read from.
+ *  @param  length
+ *      The length of the sequence that will follow this header (will be set).
+ *  @param  bit_length
+ *      The amount of bits for every value element in the following sequence (will be set).
+ *
+ *  @return Returns true if there is data after this header. (first bit was set)
+ */
 template<class T>
-algo::Huffman<T>::~Huffman(void) {
-    this->deleteTree(this->tree_root);
+bool algo::Huffman<T>::read_huffman_dict_header(util::BitStreamReader& reader, uint32_t& length, uint32_t& bit_length) {
+    if (reader.get_bit()) {
+        length     = reader.get(algo::Huffman<T>::DICT_HDR_SEQ_LENGTH_BITS);
+        bit_length = reader.get(algo::Huffman<T>::DICT_HDR_ITEM_BITS);
+        return true;
+    }
+
+    return false;
 }
 
+/**
+ *  @brief  Deallocate every node in the given tree.
+ *  @param  root
+ *      The node to start with and delete its children.
+ */
 template<class T>
 void algo::Huffman<T>::deleteTree(algo::Node<> *root) {
     if (root == nullptr) return;
@@ -34,30 +91,26 @@ void algo::Huffman<T>::deleteTree(algo::Node<> *root) {
  *      The current stream of bits for a path in the tree.
  */
 template<class T>
-size_t algo::Huffman<T>::buildDict(const algo::Node<> * const node, std::vector<bool> stream) {
+void algo::Huffman<T>::buildDict(const algo::Node<> * const node, std::vector<bool> stream) {
     if (node == nullptr) {
-        return 0u;
+        return;
     }
 
     // Check if leaf
     if (node->left == nullptr && node->right == nullptr) {
-        const uint32_t size = uint32_t(stream.size());
-
         this->dict[node->data] = Codeword {
             std::accumulate(stream.begin(), stream.end(), uint32_t(0u),
                             [=](uint32_t x, uint32_t y) { return (x << 1u) | y; }),
-            size
+            uint32_t(stream.size())
         };
-
-        return size;
     }
 
     std::vector<bool> lstream(stream);
     lstream.push_back(false);
     stream.push_back(true);
 
-    return std::max(this->buildDict(node->left , lstream),
-                    this->buildDict(node->right, stream));
+    this->buildDict(node->left , lstream);
+    this->buildDict(node->right, stream);
 }
 
 /**
@@ -89,6 +142,18 @@ void algo::Huffman<T>::decode(const algo::Node<> * const node, util::BitStreamRe
     }
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+template<class T>
+algo::Huffman<T>::Huffman(void) {
+
+}
+
+template<class T>
+algo::Huffman<T>::~Huffman(void) {
+    this->deleteTree(this->tree_root);
+}
+
 /**
  *  @brief  Encode bits of length sizeof(T) with Huffman encoding and
  *          write the Huffman dict and the encoded data to an outputstream.
@@ -98,7 +163,7 @@ void algo::Huffman<T>::decode(const algo::Node<> * const node, util::BitStreamRe
  *  @return Returns a new bitstream with the encoded data.
  */
 template<class T>
-util::BitStreamWriter* algo::Huffman<T>::encode(util::BitStreamReader& reader) {
+util::BitStreamWriter* algo::Huffman<T>::encode(util::BitStreamReader& reader) {    
     const size_t length = reader.get_size() * 8u;
 
     // Calculate frequencies
@@ -115,6 +180,8 @@ util::BitStreamWriter* algo::Huffman<T>::encode(util::BitStreamReader& reader) {
 
     for (const auto& pair: freqs) {
         pq.push(util::allocVar<algo::Node<>>(pair.first, pair.second));
+
+        util::Logger::WriteLn(std::string_format("%02X: %d", pair.first, pair.second), false);
     }
 
     while (pq.size() > 1) {
@@ -128,67 +195,113 @@ util::BitStreamWriter* algo::Huffman<T>::encode(util::BitStreamReader& reader) {
 
     this->tree_root = pq.top();
 
-    const size_t h_table_bits        = this->buildDict(this->tree_root, std::vector<bool>());
-    const size_t h_dict_total_length = (algo::Huffman<>::KEY_BITS + h_table_bits)
-                                            * this->dict.size()     // Every {key: val} pair
-                                     + algo::Huffman<>::KEY_BITS    // Length of table itself
-                                     + algo::Huffman<>::SIZE_BITS;  // Bits per value
+    this->buildDict(this->tree_root, std::vector<bool>());
 
-    util::Logger::WriteLn(std::string_format("[Huffman] {key:%d, val:%d} for %d entries + %d hdr bits (%.1f total bytes).",
-                                             algo::Huffman<>::KEY_BITS, h_table_bits, this->dict.size(),
-                                             (algo::Huffman<>::KEY_BITS + algo::Huffman<>::SIZE_BITS),
-                                             float(h_dict_total_length) / 8.0f));
+    // Create new list with dict elements sorted by bit length for saving to stream
+    // Sort the dictionary by value bit length
+    std::vector<std::pair<uint8_t, algo::Codeword>> sorted_dict(this->dict.begin(), this->dict.end());
+    std::sort(sorted_dict.begin(), sorted_dict.end(), CodewordComparator());
 
-    util::BitStreamWriter *writer = util::allocVar<util::BitStreamWriter>((h_dict_total_length + length) / 8 + 1);
+    // Determine frequencies of each bit length with {bit_length: freq}
+    std::unordered_map<uint32_t, uint32_t> bit_freqs;
+    for (const auto& w : sorted_dict) {
+        bit_freqs[w.second.len]++;
+    }
 
-    writer->put(algo::Huffman<>::KEY_BITS , uint32_t(this->dict.size()));   ///< Put table size
-    writer->put(algo::Huffman<>::SIZE_BITS, uint32_t(h_table_bits));        ///< Put bit length of a table value
+    // Calculate total needed length for dict
+    size_t h_dict_total_length = (algo::Huffman<>::KEY_BITS * this->dict.size())  // Amount of bits needed for keys
+                               + ((algo::Huffman<>::DICT_HDR_HAS_ITEMS_BITS + algo::Huffman<>::DICT_HDR_ITEM_BITS + algo::Huffman<>::DICT_HDR_SEQ_LENGTH_BITS)
+                                  * bit_freqs.size())                             // Amount of bits for each header
+                               + 1;                                               // Stop bit
+    for (const auto& f : bit_freqs) {
+        h_dict_total_length += f.first * f.second;  // Amount of bits for each header group
+    }
 
-    for (const auto& pair : this->dict) {
-        writer->put(algo::Huffman<>::KEY_BITS, pair.first);  // Put Key
-        writer->put(h_table_bits, pair.second.word);         // Put Val
+    util::Logger::WriteLn(std::string_format("[Huffman] Dict{key:%d, val:*} for %d entries + hdr bits: %.1f total bytes.",
+                                             algo::Huffman<>::KEY_BITS, this->dict.size(),
+                                             float(h_dict_total_length) / 8.0f));
+
+    //*** Save the Huffman dictionary to a stream ***//
+    util::BitStreamWriter *writer = util::allocVar<util::BitStreamWriter>((h_dict_total_length + length) / 8 + 1);
+    uint32_t seq_len = 0u, bit_len = 0u;
+
+    // Add headers for each group of same length key:val pairs
+    // and write them to the stream
+    for (const auto& w : sorted_dict) {
+        if (seq_len == 0) {
+            // New group
+            bit_len = w.second.len;
+            seq_len = bit_freqs[bit_len];
+            add_huffman_dict_header(seq_len, bit_len, *writer);
+        }
+
+        writer->put(algo::Huffman<>::KEY_BITS, w.first);  // Put Key
+        writer->put(bit_len, w.second.word);              // Put Val
+        seq_len--;
     }
 
+    add_huffman_dict_header(0, 0, *writer);
+
 
     /*******************************************************************************/
 
     /*ori*/
     reader.set_position(0);
-    while(reader.get_position() != length) {
-        const T word = T(reader.get(algo::Huffman<>::KEY_BITS));
-        util::Logger::Write(std::string_format("%X", word), false);
-    } util::Logger::WriteLn(std::string_format(" (%d bytes)", length/8), false);
+//    while(reader.get_position() != length) {
+//        const T word = T(reader.get(algo::Huffman<>::KEY_BITS));
+//        util::Logger::Write(std::string_format("%X", word), false);
+//    }
+    util::Logger::WriteLn(std::string_format(" (%d bytes)", length/8), false);
 
     /*encoded*/
+    // Encode
     reader.set_position(0);
     while(reader.get_position() != length) {
         const T word = T(reader.get(algo::Huffman<>::KEY_BITS));
-        util::Logger::Write(std::string_format("%X", this->dict[word]), false);
-
         writer->put(this->dict[word].len, this->dict[word].word); //TODO
-    } util::Logger::WriteLn("", false);
+    }
+
+    /*encoded stream*/
+    size_t len = writer->get_position() / 8;
+//    for (size_t i = 0; i < len; i++) {
+//        util::Logger::Write(std::string_format("%X", writer->get_buffer()[i]), false);
+//    }
+    util::Logger::WriteLn(std::string_format(" (%d bytes)", len), false);
 
     /*decoded*/
     util::BitStreamReader enc(writer->get_buffer(), (writer->get_position() / 8) + 1);
-    size_t table_size = enc.get(algo::Huffman<>::KEY_BITS);
-    size_t entry_bits = enc.get(algo::Huffman<>::SIZE_BITS);
-    enc.set_position(enc.get_position() + (algo::Huffman<>::KEY_BITS + entry_bits) * table_size);
 
-    util::BitStreamWriter out(length/8);
+    // readDictFromStream(enc);
+    uint32_t dseq_len = 0u, dbit_len = 0u;
+    this->dict.clear();
+//    this->deleteTree(this->tree_root);
 
-    while (enc.get_position() <=  enc.get_size() * 8u) {
-        this->decode(this->tree_root, enc, out);
-    } util::Logger::WriteLn("", false);
+    while(this->read_huffman_dict_header(enc, dseq_len, dbit_len)) { // While header is followed by sequence
+        while (dseq_len--) { // For each element, read {key, val}
+            this->dict[T(enc.get(algo::Huffman<>::KEY_BITS))] = Codeword { enc.get(dbit_len), dbit_len };
+            // TODO Add element to tree
+        }
+    }
 
-    out.set_position(0);
-    for (size_t i = 0; i < out.get_size(); i++) {
-        util::Logger::Write(std::string_format("%X", out.get_buffer()[i]), false);
-    } util::Logger::WriteLn("", false);
+//    util::BitStreamWriter out(length/8);
+
+//    while (enc.get_position() <=  enc.get_size() * 8u) {
+//        this->decode(this->tree_root, enc, out);
+//    } util::Logger::WriteLn("", false);
+
+//    out.set_position(0);
+//    for (size_t i = 0; i < out.get_size(); i++) {
+//        util::Logger::Write(std::string_format("%X", out.get_buffer()[i]), false);
+//    } util::Logger::WriteLn("", false);
+
+//    util::Logger::WriteLn("", false);
+//    this->printTree();
+//    util::Logger::WriteLn("", false);
 
-    util::Logger::WriteLn("", false);
-    this->printTree();
     util::Logger::WriteLn("", false);
 
+    this->printDict();
+
     return writer;
 }
 
@@ -203,12 +316,16 @@ util::BitStreamWriter* algo::Huffman<T>::encode(util::BitStreamReader& reader) {
 template<class T>
 util::BitStreamWriter* algo::Huffman<T>::decode(util::BitStreamReader& reader) {
     const size_t table_size = reader.get(algo::Huffman<>::KEY_BITS);    ///< Get table size
-    const size_t entry_bits = reader.get(algo::Huffman<>::SIZE_BITS);   ///< Get entry bit length
     const size_t data_bits  = reader.get_size() * 8u;                   ///< Amount of data bits
 
-    for (size_t i = 0; i < table_size; i++) {
-        this->dict[T(reader.get(algo::Huffman<>::KEY_BITS))] = Codeword { reader.get(entry_bits), 0u };
-    }
+    
+    // TODO if first bit is zero => no Huffman table => do nothing, just pass the stream back
+    // use internal flag to enable Huffman, if disabled, write 1 zero to stream before data,
+    // and later just call huffman.decode() (see TODO this TODO)
+    
+//    for (size_t i = 0; i < table_size; i++) {
+//        this->dict[T(reader.get(algo::Huffman<>::KEY_BITS))] = Codeword { reader.get(entry_bits), 0u };
+//    }
 
 
     // TODO Create tree from dict
diff --git a/Huffman.hpp b/Huffman.hpp
@@ -58,10 +58,10 @@ namespace algo {
     /**
      *  Data struct for Huffman dictionary entries.
      */
-    typedef struct {
+    struct Codeword {
         uint32_t word;
         uint32_t len;
-    } Codeword;
+    };
 
     /**
      *  @brief Huffman class
@@ -73,7 +73,10 @@ namespace algo {
 
             std::unordered_map<T, Codeword> dict;
 
-            size_t buildDict(const algo::Node<> * const, std::vector<bool>);
+            void add_huffman_dict_header(uint32_t, uint32_t, util::BitStreamWriter&);
+            bool read_huffman_dict_header(util::BitStreamReader&, uint32_t&, uint32_t&);
+
+            void buildDict(const algo::Node<> * const, std::vector<bool>);
             void decode(const algo::Node<> * const, util::BitStreamReader&, util::BitStreamWriter&);
 
             void deleteTree(algo::Node<>*);
@@ -89,8 +92,11 @@ namespace algo {
             void printDict(void);
             void printTree(void);
 
-            static constexpr size_t KEY_BITS  = util::size_of<T>();
-            static constexpr size_t SIZE_BITS = util::size_of<T>();
+            static constexpr size_t KEY_BITS  = util::size_of<T>(); ///< Bit length for keys in Huffman dict
+
+            static constexpr size_t DICT_HDR_HAS_ITEMS_BITS  = 1u;  ///< Whether there are dictionary items following (bit length)
+            static constexpr size_t DICT_HDR_SEQ_LENGTH_BITS = 7u;  ///< Amunt of bits to represent the length of following items
+            static constexpr size_t DICT_HDR_ITEM_BITS       = 4u;  ///< Amunt of bits to represent the length of following items
     };
 
     extern template class Node<uint8_t>;
diff --git a/main.hpp b/main.hpp
@@ -23,6 +23,6 @@
 #endif // 1
 
 //#define LOG_OFF       ///< Force logging off
-#define LOG_LOCAL     ///< Enable Block-level logging (a lot of overhead, use sparingly)
+//#define LOG_LOCAL     ///< Enable Block-level logging (a lot of overhead, use sparingly)
 
 #endif // MAIN_HPP
