Updated scene serialisation to use binary serialiser

Author: jonjondev

engine/core/scene/SceneFile.cpp

@@ -11,6 +11,7 @@
 
 #include <resources/PackFile.h>
 #include <resources/ResourceSystem.h>
+#include <resources/PackFileData.h>
 #include <resources/SceneData.h>
 #include <utils/Logging.h>
 
@@ -125,15 +126,23 @@ bool SceneFile::Deserialise(std::vector<Entity*>& entities)
     {
         PackFile* packFile = ResourceSystem::GetInstance().GetPackFile();
 
-        std::shared_ptr<SceneData> sceneData = packFile->FindData<SceneData>(ScenePath);
-        if (!sceneData)
+        std::shared_ptr<PackFileData> sceneDataBuffer = packFile->FindData<PackFileData>(ScenePath);
+        if (!sceneDataBuffer)
         {
             CC_LOG_WARNING("Failed to find scene \"{}\" in pack file", ScenePath)
             return false;
         }
 
-        String sceneString(sceneData->data);
-        for (const String& entityData : sceneString.Split('|'))
+        BinarySerialisation::Buffer buffer;
+        uint8_t* data = reinterpret_cast<uint8_t*>(sceneDataBuffer->data);
+        size_t dataSize = sceneDataBuffer->dataSize;
+        buffer.data.assign(data, dataSize);
+
+        // TODO - Fold into resource system
+        SceneData sceneData;
+        sceneData.serialise(buffer, BinarySerialisation::DESERIALISE);
+
+        for (const String& entityData : sceneData.entities)
         {
             deserialiseEntityString(entityData.Str(), "");
         }

engine/render/renderer/platform/vulkan/Font.cpp

@@ -10,7 +10,7 @@
 #include "Font.h"
 
 #include <freetype/freetype.h>
-#include <resources/GenericFileData.h>
+#include <resources/PackFileData.h>
 #include <resources/PackFile.h>
 #include <resources/ResourceSystem.h>
 #include <utils/Defer.h>
@@ -27,7 +27,7 @@ namespace Siege::Vulkan
 Font::Font(const char* filePath)
 {
     PackFile* packFile = ResourceSystem::GetInstance().GetPackFile();
-    std::shared_ptr<GenericFileData> fileData = packFile->FindData<GenericFileData>(filePath);
+    std::shared_ptr<PackFileData> fileData = packFile->FindData<PackFileData>(filePath);
 
     FT_Open_Args args;
     args.flags = FT_OPEN_MEMORY;

engine/render/renderer/platform/vulkan/Shader.cpp

@@ -9,7 +9,7 @@
 
 #include "Shader.h"
 
-#include <resources/GenericFileData.h>
+#include <resources/PackFileData.h>
 #include <resources/PackFile.h>
 #include <resources/ResourceSystem.h>
 #include <utils/Logging.h>
@@ -174,7 +174,7 @@ void Shader::Destroy()
 MHArray<char> Shader::ReadFileAsBinary(const String& filePath)
 {
     PackFile* packFile = ResourceSystem::GetInstance().GetPackFile();
-    std::shared_ptr<GenericFileData> fileData = packFile->FindData<GenericFileData>(filePath);
+    std::shared_ptr<PackFileData> fileData = packFile->FindData<PackFileData>(filePath);
     MHArray<char> buffer(fileData->data, fileData->dataSize);
     return buffer;
 }

engine/resources/GenericFileData.h renamed to engine/resources/PackFileData.h

@@ -7,16 +7,16 @@
 //     https://opensource.org/licenses/Zlib
 //
 
-#ifndef SIEGE_ENGINE_GENERICFILEDATA_H
-#define SIEGE_ENGINE_GENERICFILEDATA_H
+#ifndef SIEGE_ENGINE_PACKFILEDATA_H
+#define SIEGE_ENGINE_PACKFILEDATA_H
 
 #include "PackFile.h"
 
 namespace Siege
 {
 
 #pragma pack(push, 1)
-struct GenericFileData
+struct PackFileData
 {
     uint32_t dataSize = 0;
     char data[];
@@ -26,12 +26,12 @@ struct GenericFileData
         return &data[0];
     }
 
-    static GenericFileData* Create(char* data, uint32_t dataSize)
+    static PackFileData* Create(char* data, uint32_t dataSize)
     {
-        uint32_t totalDataSize = sizeof(GenericFileData) + dataSize;
+        uint32_t totalDataSize = sizeof(PackFileData) + dataSize;
 
         void* mem = malloc(totalDataSize);
-        GenericFileData* fileData = new (mem) GenericFileData();
+        PackFileData* fileData = new (mem) PackFileData();
 
         fileData->dataSize = dataSize;
         memcpy(&fileData->data[0], data, dataSize);
@@ -42,12 +42,12 @@ struct GenericFileData
     static uint32_t GetDataSize(void* objectData)
     {
         if (!objectData) return 0;
-        GenericFileData* fileData = reinterpret_cast<GenericFileData*>(objectData);
-        return sizeof(GenericFileData) + fileData->dataSize;
+        PackFileData* fileData = reinterpret_cast<PackFileData*>(objectData);
+        return sizeof(PackFileData) + fileData->dataSize;
     }
 };
 #pragma pack(pop)
 
 } // namespace Siege
 
-#endif // SIEGE_ENGINE_GENERICFILEDATA_H
+#endif // SIEGE_ENGINE_PACKFILEDATA_H

engine/resources/SceneData.h

@@ -10,43 +10,20 @@
 #ifndef SIEGE_ENGINE_SCENEDATA_H
 #define SIEGE_ENGINE_SCENEDATA_H
 
-#include "PackFile.h"
+#include <utils/BinarySerialisation.h>
 
 namespace Siege
 {
 
-#pragma pack(push, 1)
-struct SceneData
+struct SceneData : BinarySerialisation::BinarySerialisable
 {
-    uint32_t dataSize = 0;
-    char data[];
+    std::vector<String> entities;
 
-    const char* GetData() const
+    void serialise(BinarySerialisation::Buffer& buffer, BinarySerialisation::SerialisationMode mode) override
     {
-        return &data[0];
-    }
-
-    static SceneData* Create(char* data, uint32_t dataSize)
-    {
-        uint32_t totalDataSize = sizeof(SceneData) + dataSize;
-
-        void* mem = malloc(totalDataSize);
-        SceneData* sceneData = new (mem) SceneData();
-
-        sceneData->dataSize = dataSize;
-        memcpy(&sceneData->data[0], data, dataSize);
-
-        return sceneData;
-    }
-
-    static uint32_t GetDataSize(void* objectData)
-    {
-        if (!objectData) return 0;
-        SceneData* sceneData = reinterpret_cast<SceneData*>(objectData);
-        return sizeof(SceneData) + sceneData->dataSize;
+        BinarySerialisation::serialise(buffer, entities, mode);
     }
 };
-#pragma pack(pop)
 
 } // namespace Siege
 

engine/utils/BinarySerialisation.h

@@ -26,11 +26,11 @@ enum SerialisationMode : u_int8_t
 };
 
 #define SERIALISE_NATIVE(type)                                                \
-    void serialise(Buffer& buffer, type& value, SerialisationMode mode)       \
+    inline void serialise(Buffer& buffer, type& value, SerialisationMode mode)       \
     {                                                                         \
         serialiseNative(buffer, value, mode);                                 \
     }                                                                         \
-    void serialise(Buffer& buffer, const type& value, SerialisationMode mode) \
+    inline void serialise(Buffer& buffer, const type& value, SerialisationMode mode) \
     {                                                                         \
         serialiseNative(buffer, value, mode);                                 \
     }
@@ -83,6 +83,7 @@ void serialiseNative(Buffer& buffer, const T& value, SerialisationMode mode)
 }
 
 SERIALISE_NATIVE(bool)
+SERIALISE_NATIVE(char)
 SERIALISE_NATIVE(uint32_t)
 SERIALISE_NATIVE(int32_t)
 SERIALISE_NATIVE(float)
@@ -106,6 +107,32 @@ void serialiseContainer(Buffer& buffer, T& value)
     }
 }
 
+inline void serialise(Buffer& buffer, String& value, SerialisationMode mode)
+{
+
+    switch (mode)
+    {
+        case SERIALISE:
+        {
+            std::string temp(value.Str());
+            serialiseContainer(buffer, temp);
+            break;
+        }
+        case DESERIALISE:
+        {
+            value.Clear();
+
+            uint32_t size;
+            serialise(buffer, size, mode);
+            value.Reserve(size);
+
+            value.Append(reinterpret_cast<char*>(buffer.data.data() + buffer.cursor), size);
+            buffer.cursor += size;
+            break;
+        }
+    }
+}
+
 template<typename T>
 void serialise(Buffer& buffer, std::vector<T>& value, SerialisationMode mode)
 {

engine/utils/String.cpp

@@ -549,6 +549,19 @@ void String::Append(char character)
     Assign(cstr);
 }
 
+void String::Append(const char *string, size_t size)
+{
+    const char* data = Data();
+    size_t lhsLength = Size();
+    size_t fullLength = lhsLength + size;
+
+    char* cstr = static_cast<char*>(alloca(fullLength + 1));
+    strcpy(cstr, data);
+    strncpy(cstr + lhsLength, string, size);
+
+    Assign(cstr);
+}
+
 void String::Prepend(const String& string)
 {
     // Mirrors c-string implementation

engine/utils/String.h

@@ -183,6 +183,11 @@ class String
      */
     String& operator+=(const char* rhs);
 
+    /**
+     * Wide-char c-string addition compound assignment operator overload for appending wide char c-strings
+     * @param rhs - the wide char c-string to append
+     * @return a reference to the original String object
+     */
     String& operator+=(const wchar_t* rhs);
 
     /**
@@ -439,6 +444,13 @@ class String
      */
     void Append(char character);
 
+    /**
+     * Appends the value of a c-string to the String for a number of characters
+     * @param string - the c-string to append
+     * @param length - the length of the string to append
+     */
+    void Append(const char* string, size_t length);
+
     /**
      * Prepends the value of another String
      * @param string - the String to prepend

packer/src/main.cpp

@@ -7,12 +7,11 @@
 //     https://opensource.org/licenses/Zlib
 //
 
-#include <resources/GenericFileData.h>
+#include <resources/PackFileData.h>
 #include <resources/PackFile.h>
 #include <resources/ResourceSystem.h>
 #include <resources/SceneData.h>
 #include <resources/StaticMeshData.h>
-#include <resources/Texture2DData.h>
 #include <utils/Logging.h>
 
 #include <fstream>
@@ -55,28 +54,24 @@ int main(int argc, char* argv[])
         CC_LOG_INFO("Reading asset at path {}", file.c_str())
 
         void* data = nullptr;
-        uint32_t bodyDataSize = 0;
+        uint32_t dataSize = 0;
         Siege::String fullPath = assetsDir + "/" + Siege::String(file.c_str());
         std::filesystem::path extension = file.extension();
         if (extension == ".sm")
         {
-            data = PackStaticMeshFile(fullPath, assetsDir);
-            bodyDataSize = Siege::StaticMeshData::GetDataSize(data);
+            data = PackStaticMeshFile(fullPath, assetsDir, dataSize);
         }
         else if (extension == ".jpg" || extension == ".jpeg" || extension == ".png")
         {
-            data = PackTexture2DFile(fullPath);
-            bodyDataSize = Siege::Texture2DData::GetDataSize(data);
+            data = PackTexture2DFile(fullPath, dataSize);
         }
         else if (extension == ".spv" || extension == ".ttf")
         {
-            data = PackGenericFile(fullPath);
-            bodyDataSize = Siege::GenericFileData::GetDataSize(data);
+            data = PackGenericFile(fullPath, dataSize);
         }
         else if (extension == ".scene")
         {
-            data = PackSceneFile(fullPath);
-            bodyDataSize = Siege::SceneData::GetDataSize(data);
+            data = PackSceneFile(fullPath, dataSize);
         }
 
         if (!data)
@@ -86,12 +81,11 @@ int main(int argc, char* argv[])
             continue;
         }
 
-        PackFile::TocEntry* tocEntry =
-            PackFile::TocEntry::Create(file.c_str(), entriesDataSize, bodyDataSize);
+        PackFile::TocEntry* tocEntry = PackFile::TocEntry::Create(file.c_str(), entriesDataSize, dataSize);
 
         entries.emplace_back(tocEntry, data);
 
-        entriesDataSize += bodyDataSize;
+        entriesDataSize += dataSize;
         entriesTocSize += tocEntry->GetDataSize();
 
         dynamicAllocations.push_back(data);
@@ -109,7 +103,6 @@ int main(int argc, char* argv[])
         header.bodySize,
         header.tocOffset)
     uint64_t writeTotal = 0;
-    uint64_t dataOffset = 0;
 
     std::ofstream outputFileStream;
     outputFileStream.open(outputFile, std::ios::out | std::ios::binary);
@@ -121,12 +114,15 @@ int main(int argc, char* argv[])
                 writeTotal)
 
     entriesDataSize = 0;
+    Bytef* bodyDataBufferCompressed  = nullptr;
     for (const std::pair<PackFile::TocEntry*, void*>& entry : entries)
     {
         uLongf bodyDataSizeUncompressed = entry.first->dataSize;
         uLongf bodyDataSizeCompressed = compressBound(bodyDataSizeUncompressed);
 
-        Bytef bodyDataBufferCompressed[bodyDataSizeCompressed];
+        Bytef* tempBuffer = static_cast<Bytef*>(realloc(bodyDataBufferCompressed, bodyDataSizeCompressed));
+        bodyDataBufferCompressed = tempBuffer;
+
         int result = compress2(bodyDataBufferCompressed, &bodyDataSizeCompressed,
             static_cast<Bytef*>(entry.second), bodyDataSizeUncompressed, Z_BEST_COMPRESSION);
         CC_ASSERT(result == Z_OK, "Compression failed for entry: " + Siege::String(entry.first->name));
@@ -136,13 +132,14 @@ int main(int argc, char* argv[])
         entry.first->dataSizeCompressed = bodyDataSizeCompressed;
         writeTotal += bodyDataSizeCompressed;
         entriesDataSize += bodyDataSizeCompressed;
-        CC_LOG_INFO("Adding DATA \"{}\" to pack file with size: {}, compressed to {}% (write total: {})",
+        CC_LOG_INFO("Adding DATA \"{}\" to pack file with size: {} from {}, compressed to ~{}% (write total: {})",
                     entry.first->name,
                     bodyDataSizeCompressed,
-                    static_cast<uint8_t>(static_cast<float>(bodyDataSizeCompressed) / static_cast<float>(bodyDataSizeUncompressed) * 100.f),
+                    bodyDataSizeUncompressed,
+                    static_cast<uint8_t>(ceilf(static_cast<float>(bodyDataSizeCompressed) / static_cast<float>(bodyDataSizeUncompressed) * 100.f)),
                     writeTotal)
-        dataOffset += bodyDataSizeCompressed;
     }
+    free(bodyDataBufferCompressed);
 
     outputFileStream.write(PACKER_MAGIC_NUMBER_TOC, PACKER_MAGIC_NUMBER_SIZE);
     writeTotal += PACKER_MAGIC_NUMBER_SIZE;