Add test case for writing and reading huge tar file

This test case is disabled by default because it needs too much storage,
but serves as documentation for the future.

Author: TheMarex

unit_tests/storage/tar.cpp

@@ -3,6 +3,8 @@
 #include "../common/range_tools.hpp"
 #include "../common/temporary_file.hpp"
 
+#include <boost/function_output_iterator.hpp>
+#include <boost/iterator/function_input_iterator.hpp>
 #include <boost/test/unit_test.hpp>
 
 BOOST_AUTO_TEST_SUITE(tar)
@@ -187,4 +189,35 @@ BOOST_AUTO_TEST_CASE(continue_write_tar_file)
     CHECK_EQUAL_COLLECTIONS(result_64bit_vector, vector_64bit);
 }
 
+// This test case is disabled by default because it needs 70 GiB of storage
+BOOST_AUTO_TEST_CASE(write_huge_tar_file, *boost::unit_test::disabled())
+{
+    TemporaryFile tmp{TEST_DATA_DIR "/tar_huge_write_test.tar"};
+
+    std::uint64_t reference_checksum = 0;
+    {
+        storage::tar::FileWriter writer(tmp.path, storage::tar::FileWriter::GenerateFingerprint);
+        std::uint64_t value = 0;
+        const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
+            reference_checksum += value;
+            return value++;
+        };
+        std::uint64_t num_elements = (70ULL * 1024ULL * 1024ULL * 1024ULL) / sizeof(std::uint64_t);
+        writer.WriteStreaming<std::uint64_t>(
+            "huge_data",
+            boost::make_function_input_iterator(encode_function, boost::infinite()),
+            num_elements);
+    }
+
+    std::uint64_t checksum = 0;
+    {
+        storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
+        reader.ReadStreaming<std::uint64_t>(
+            "huge_data",
+            boost::make_function_output_iterator([&](const auto &value) { checksum += value; }));
+    }
+
+    BOOST_CHECK_EQUAL(checksum, reference_checksum);
+}
+
 BOOST_AUTO_TEST_SUITE_END()