Skip to content

Commit c92dbea

Browse files
jonjondevjonjondev
committed
Implemented zlib compression for packed assets
1 parent 3c96562 commit c92dbea

File tree

12 files changed

+76
-54
lines changed

12 files changed

+76
-54
lines changed

‎engine/core/scene/SceneFile.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ bool SceneFile::Deserialise(std::vector<Entity*>& entities)
125125
{
126126
PackFile* packFile = ResourceSystem::GetInstance().GetPackFile();
127127

128-
SceneData* sceneData = packFile->FindData<SceneData>(ScenePath);
128+
std::shared_ptr<SceneData> sceneData = packFile->FindData<SceneData>(ScenePath);
129129
if (!sceneData)
130130
{
131131
CC_LOG_WARNING("Failed to find scene \"{}\" in pack file", ScenePath)

‎engine/render/renderer/platform/vulkan/Font.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ namespace Siege::Vulkan
2727
Font::Font(const char* filePath)
2828
{
2929
PackFile* packFile = ResourceSystem::GetInstance().GetPackFile();
30-
GenericFileData* fileData = packFile->FindData<GenericFileData>(filePath);
30+
std::shared_ptr<GenericFileData> fileData = packFile->FindData<GenericFileData>(filePath);
3131

3232
FT_Open_Args args;
3333
args.flags = FT_OPEN_MEMORY;

‎engine/render/renderer/platform/vulkan/Shader.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ void Shader::Destroy()
174174
MHArray<char> Shader::ReadFileAsBinary(const String& filePath)
175175
{
176176
PackFile* packFile = ResourceSystem::GetInstance().GetPackFile();
177-
GenericFileData* fileData = packFile->FindData<GenericFileData>(filePath);
177+
std::shared_ptr<GenericFileData> fileData = packFile->FindData<GenericFileData>(filePath);
178178
MHArray<char> buffer(fileData->data, fileData->dataSize);
179179
return buffer;
180180
}

‎engine/render/renderer/platform/vulkan/StaticMesh.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ StaticMesh::StaticMesh(const char* filePath, Material* material)
4343
{
4444
// TODO(Aryeh): How to extract material data from object files?
4545
PackFile* packFile = ResourceSystem::GetInstance().GetPackFile();
46-
StaticMeshData* vertexData = packFile->FindData<StaticMeshData>(filePath);
46+
std::shared_ptr<StaticMeshData> vertexData = packFile->FindData<StaticMeshData>(filePath);
4747

4848
CC_ASSERT(vertexData->verticesCount > 0, "Cannot load in a file with no vertices!")
4949
CC_ASSERT(vertexData->indicesCount > 0, "Cannot load in a file with no indices!")

‎engine/render/renderer/platform/vulkan/Texture2D.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ Texture2D& Texture2D::operator=(Texture2D&& other)
9393
void Texture2D::LoadFromFile(const char* filePath)
9494
{
9595
PackFile* packFile = ResourceSystem::GetInstance().GetPackFile();
96-
Texture2DData* texture2dData = packFile->FindData<Texture2DData>(filePath);
96+
std::shared_ptr<Texture2DData> texture2dData = packFile->FindData<Texture2DData>(filePath);
9797
uint64_t imageSize = texture2dData->GetImageSize();
9898
const uint8_t* pixelPtr = texture2dData->GetPixels();
9999

‎engine/resources/Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ resourcesObjects := $(call findobjs,$(resourcesSrcDir),$(resourcesBinDir),$(reso
1616
resourcesDepends := $(patsubst %.o, %.d, $(call rwildcard,$(resourcesBinDir)/,*.o))
1717

1818
linkFlags += -l utils
19+
compileFlags += -I $(vendorDir)/zlib/build/include
1920

2021
# Build the static library
2122
$(resourcesLib): $(resourcesObjects)

‎engine/resources/PackFile.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ bool PackFile::LoadFromPath(const String& filepath)
2828
{
2929
std::ifstream inputFileStream;
3030
inputFileStream.open(filepath, std::ios::in | std::ios::binary);
31-
inputFileStream.read(reinterpret_cast<char*>(&header), sizeof(PackFile::Header));
31+
inputFileStream.read(reinterpret_cast<char*>(&header), sizeof(Header));
3232

3333
body = reinterpret_cast<char*>(malloc(header.bodySize));
3434
inputFileStream.read(body, (uint32_t) header.bodySize);
@@ -41,7 +41,7 @@ bool PackFile::LoadFromPath(const String& filepath)
4141
char* tocEnd = body + header.bodySize;
4242
while (tocCurr < tocEnd)
4343
{
44-
PackFile::TocEntry* toc = reinterpret_cast<PackFile::TocEntry*>(tocCurr);
44+
TocEntry* toc = reinterpret_cast<TocEntry*>(tocCurr);
4545
if (!toc)
4646
{
4747
break;
@@ -69,7 +69,7 @@ PackFile::TocEntry* PackFile::TocEntry::Create(const String& name,
6969
uint32_t nameDataSize = name.Size() + 1;
7070
void* mem = malloc(sizeof(TocEntry) + nameDataSize);
7171

72-
PackFile::TocEntry* tocEntry = new (mem) TocEntry();
72+
TocEntry* tocEntry = new (mem) TocEntry();
7373
tocEntry->dataOffset = dataOffset;
7474
tocEntry->dataSize = dataSize;
7575
strcpy(&tocEntry->name[0], name.Str());

‎engine/resources/PackFile.h‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,13 @@
1111
#define SIEGE_ENGINE_PACKFILE_H
1212

1313
#include <utils/String.h>
14+
#include <utils/Logging.h>
1415

1516
#include <filesystem>
1617
#include <map>
1718

19+
#include <zlib.h>
20+
1821
#define PACKER_MAGIC_NUMBER_FILE "pck"
1922
#define PACKER_MAGIC_NUMBER_TOC "toc!"
2023
#define PACKER_MAGIC_NUMBER_SIZE sizeof(uint32_t)
@@ -46,6 +49,7 @@ class PackFile
4649
{
4750
uint32_t dataOffset;
4851
uint32_t dataSize;
52+
uint32_t dataSizeCompressed;
4953
char name[];
5054

5155
uint32_t GetDataSize() const
@@ -66,14 +70,22 @@ class PackFile
6670
bool LoadFromPath(const String& filepath);
6771

6872
template<typename T>
69-
T* FindData(const String& filepath)
73+
std::shared_ptr<T> FindData(const String& filepath)
7074
{
71-
PackFile::TocEntry* toc = entries[filepath];
75+
const TocEntry* toc = entries[filepath];
7276
if (!toc)
7377
{
7478
return nullptr;
7579
}
76-
return reinterpret_cast<T*>(body + toc->dataOffset);
80+
81+
uLongf bodyDataSizeUncompressed = toc->dataSize;
82+
uLongf bodyDataSizeCompressed = toc->dataSizeCompressed;
83+
void* mem = malloc(bodyDataSizeUncompressed);
84+
85+
int result = uncompress(static_cast<Bytef*>(mem), &bodyDataSizeUncompressed, reinterpret_cast<Bytef*>(body + toc->dataOffset), bodyDataSizeCompressed);
86+
CC_ASSERT(result == Z_OK, "Decompression failed for filepath: " + filepath);
87+
88+
return std::shared_ptr<T>(static_cast<T*>(mem), free);
7789
}
7890

7991
const std::map<String, TocEntry*>& GetEntries();

‎packer/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ packerBuildDir := $(packerBinDir)/build
1818

1919
# Set build vars
2020
linkFlags += -l resources -l utils -L $(vendorDir)/zlib/build/lib -L $(vendorDir)/assimp/build/lib -l assimp -l z -l stdc++
21-
compileFlags += -I $(vendorDir)/stb_image -I $(vendorDir)/assimp/include -I $(vendorDir)/assimp/build/include
21+
compileFlags += -I $(vendorDir)/stb_image -I $(vendorDir)/assimp/include -I $(vendorDir)/assimp/build/include -I $(vendorDir)/zlib/build/include
2222

2323
.PHONY: all
2424

‎packer/src/main.cpp‎

Lines changed: 42 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include <utils/Logging.h>
1717

1818
#include <fstream>
19+
#include <zlib.h>
1920

2021
#include "types/GenericFileDataPacker.h"
2122
#include "types/SceneDataPacker.h"
@@ -43,18 +44,11 @@ int main(int argc, char* argv[])
4344
inputFiles.emplace_back(argv[currentArg]);
4445
}
4546

46-
uint32_t dataSize = 0;
47-
uint32_t entriesSize = 0;
48-
49-
struct BodyDataEntry
50-
{
51-
void* ptr;
52-
uint32_t size;
53-
};
47+
uint32_t entriesDataSize = 0;
48+
uint32_t entriesTocSize = 0;
5449

5550
std::vector<void*> dynamicAllocations;
56-
std::vector<PackFile::TocEntry*> tocEntries;
57-
std::vector<BodyDataEntry> bodyDataEntries;
51+
std::vector<std::pair<PackFile::TocEntry*, void*>> entries;
5852
for (auto& file : inputFiles)
5953
{
6054
if (file.empty()) continue;
@@ -93,22 +87,21 @@ int main(int argc, char* argv[])
9387
}
9488

9589
PackFile::TocEntry* tocEntry =
96-
PackFile::TocEntry::Create(file.c_str(), dataSize, bodyDataSize);
90+
PackFile::TocEntry::Create(file.c_str(), entriesDataSize, bodyDataSize);
9791

98-
bodyDataEntries.push_back({data, bodyDataSize});
99-
tocEntries.push_back(tocEntry);
92+
entries.emplace_back(tocEntry, data);
10093

101-
dataSize += bodyDataSize;
102-
entriesSize += tocEntry->GetDataSize();
94+
entriesDataSize += bodyDataSize;
95+
entriesTocSize += tocEntry->GetDataSize();
10396

10497
dynamicAllocations.push_back(data);
10598
dynamicAllocations.push_back(tocEntry);
10699
}
107100

108101
PackFile::Header header {{PACKER_MAGIC_NUMBER_FILE},
109102
PACKER_FILE_VERSION,
110-
dataSize + PACKER_MAGIC_NUMBER_SIZE + entriesSize,
111-
dataSize};
103+
entriesDataSize + PACKER_MAGIC_NUMBER_SIZE + entriesTocSize,
104+
entriesDataSize};
112105

113106
CC_LOG_INFO(
114107
"Beginning pack file version {} write process for body size {} and ToC offset of {}...",
@@ -127,16 +120,28 @@ int main(int argc, char* argv[])
127120
sizeof(PackFile::Header),
128121
writeTotal)
129122

130-
for (auto entry : bodyDataEntries)
123+
entriesDataSize = 0;
124+
for (const std::pair<PackFile::TocEntry*, void*>& entry : entries)
131125
{
132-
uint32_t bodyDataSize = entry.size;
133-
outputFileStream.write(reinterpret_cast<char*>(entry.ptr), bodyDataSize);
134-
writeTotal += bodyDataSize;
135-
CC_LOG_INFO("Adding DATA (offset: {}) to pack file with size: {} (write total: {})",
136-
dataOffset,
137-
bodyDataSize,
126+
uLongf bodyDataSizeUncompressed = entry.first->dataSize;
127+
uLongf bodyDataSizeCompressed = compressBound(bodyDataSizeUncompressed);
128+
129+
Bytef bodyDataBufferCompressed[bodyDataSizeCompressed];
130+
int result = compress2(bodyDataBufferCompressed, &bodyDataSizeCompressed,
131+
static_cast<Bytef*>(entry.second), bodyDataSizeUncompressed, Z_BEST_COMPRESSION);
132+
CC_ASSERT(result == Z_OK, "Compression failed for entry: " + Siege::String(entry.first->name));
133+
134+
outputFileStream.write(reinterpret_cast<char*>(bodyDataBufferCompressed), static_cast<long>(bodyDataSizeCompressed));
135+
entry.first->dataOffset = entriesDataSize;
136+
entry.first->dataSizeCompressed = bodyDataSizeCompressed;
137+
writeTotal += bodyDataSizeCompressed;
138+
entriesDataSize += bodyDataSizeCompressed;
139+
CC_LOG_INFO("Adding DATA \"{}\" to pack file with size: {}, compressed to {}% (write total: {})",
140+
entry.first->name,
141+
bodyDataSizeCompressed,
142+
static_cast<uint8_t>(static_cast<float>(bodyDataSizeCompressed) / static_cast<float>(bodyDataSizeUncompressed) * 100.f),
138143
writeTotal)
139-
dataOffset += bodyDataSize;
144+
dataOffset += bodyDataSizeCompressed;
140145
}
141146

142147
outputFileStream.write(PACKER_MAGIC_NUMBER_TOC, PACKER_MAGIC_NUMBER_SIZE);
@@ -145,18 +150,23 @@ int main(int argc, char* argv[])
145150
PACKER_MAGIC_NUMBER_SIZE,
146151
writeTotal)
147152

148-
for (Siege::PackFile::TocEntry* toc : tocEntries)
153+
for (const std::pair<PackFile::TocEntry*, void*>& entry : entries)
149154
{
150-
outputFileStream.write(reinterpret_cast<char*>(toc), toc->GetDataSize());
151-
writeTotal += toc->GetDataSize();
155+
outputFileStream.write(reinterpret_cast<char*>(entry.first), entry.first->GetDataSize());
156+
writeTotal += entry.first->GetDataSize();
152157
CC_LOG_INFO(
153-
"Adding TOC ENTRY \"{}\" (offset: {}) to pack file with size: {} (write total: {})",
154-
toc->name,
155-
toc->dataOffset,
156-
toc->GetDataSize(),
158+
"Adding TOC \"{}\" (offset: {}) to pack file with size: {} (write total: {})",
159+
entry.first->name,
160+
entry.first->dataOffset,
161+
entry.first->GetDataSize(),
157162
writeTotal)
158163
}
159164

165+
header.bodySize = entriesDataSize + PACKER_MAGIC_NUMBER_SIZE + entriesTocSize;
166+
header.tocOffset = entriesDataSize;
167+
outputFileStream.seekp(0);
168+
outputFileStream.write(reinterpret_cast<char*>(&header), sizeof(PackFile::Header));
169+
160170
outputFileStream.close();
161171
CC_LOG_INFO("Ended pack file write process (write total: {})", writeTotal)
162172

@@ -176,7 +186,6 @@ int main(int argc, char* argv[])
176186
{
177187
CC_LOG_WARNING("Missing ToC entry for input file \"{}\"", file.c_str())
178188
errors = true;
179-
continue;
180189
}
181190
}
182191

0 commit comments

Comments
 (0)