Skip to content

Commit 5777a88

Browse files
committed
Support 64-bit sizes in Compression
1 parent e45cc68 commit 5777a88

File tree

13 files changed

+58
-48
lines changed

13 files changed

+58
-48
lines changed

core/io/compression.cpp

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,13 @@ static ZSTD_DCtx *current_zstd_d_ctx = nullptr;
4747
static bool current_zstd_long_distance_matching;
4848
static int current_zstd_window_log_size;
4949

50-
int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode) {
50+
int64_t Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int64_t p_src_size, Mode p_mode) {
5151
switch (p_mode) {
5252
case MODE_BROTLI: {
5353
ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
5454
} break;
5555
case MODE_FASTLZ: {
56+
ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a FastLZ/LZ77 file 2 GiB or larger. LZ77 supports larger files, but FastLZ's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
5657
if (p_src_size < 16) {
5758
uint8_t src[16];
5859
memset(&src[p_src_size], 0, 16 - p_src_size);
@@ -65,6 +66,7 @@ int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size,
6566
} break;
6667
case MODE_DEFLATE:
6768
case MODE_GZIP: {
69+
ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a Deflate or GZip file 2 GiB or larger. Deflate and GZip are both limited to 4 GiB, and ZLib's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
6870
int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
6971

7072
z_stream strm;
@@ -95,22 +97,23 @@ int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size,
9597
ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1);
9698
ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, zstd_window_log_size);
9799
}
98-
int max_dst_size = get_max_compressed_buffer_size(p_src_size, MODE_ZSTD);
99-
int ret = ZSTD_compressCCtx(cctx, p_dst, max_dst_size, p_src, p_src_size, zstd_level);
100+
const int64_t max_dst_size = get_max_compressed_buffer_size(p_src_size, MODE_ZSTD);
101+
const size_t ret = ZSTD_compressCCtx(cctx, p_dst, max_dst_size, p_src, p_src_size, zstd_level);
100102
ZSTD_freeCCtx(cctx);
101-
return ret;
103+
return (int64_t)ret;
102104
} break;
103105
}
104106

105107
ERR_FAIL_V(-1);
106108
}
107109

108-
int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) {
110+
int64_t Compression::get_max_compressed_buffer_size(int64_t p_src_size, Mode p_mode) {
109111
switch (p_mode) {
110112
case MODE_BROTLI: {
111113
ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
112114
} break;
113115
case MODE_FASTLZ: {
116+
ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a FastLZ/LZ77 file 2 GiB or larger. LZ77 supports larger files, but FastLZ's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
114117
int ss = p_src_size + p_src_size * 6 / 100;
115118
if (ss < 66) {
116119
ss = 66;
@@ -120,6 +123,7 @@ int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) {
120123
} break;
121124
case MODE_DEFLATE:
122125
case MODE_GZIP: {
126+
ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a Deflate or GZip file 2 GiB or larger. Deflate and GZip are both limited to 4 GiB, and ZLib's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
123127
int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
124128

125129
z_stream strm;
@@ -142,7 +146,7 @@ int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) {
142146
ERR_FAIL_V(-1);
143147
}
144148

145-
int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
149+
int64_t Compression::decompress(uint8_t *p_dst, int64_t p_dst_max_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode) {
146150
switch (p_mode) {
147151
case MODE_BROTLI: {
148152
#ifdef BROTLI_ENABLED
@@ -155,6 +159,7 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
155159
#endif
156160
} break;
157161
case MODE_FASTLZ: {
162+
ERR_FAIL_COND_V_MSG(p_dst_max_size > INT32_MAX, -1, "Cannot decompress a FastLZ/LZ77 file 2 GiB or larger. LZ77 supports larger files, but FastLZ's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
158163
int ret_size = 0;
159164

160165
if (p_dst_max_size < 16) {
@@ -169,6 +174,7 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
169174
} break;
170175
case MODE_DEFLATE:
171176
case MODE_GZIP: {
177+
ERR_FAIL_COND_V_MSG(p_dst_max_size > INT32_MAX, -1, "Cannot decompress a Deflate or GZip file 2 GiB or larger. Deflate and GZip are both limited to 4 GiB, and ZLib's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
172178
int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
173179

174180
z_stream strm;
@@ -207,8 +213,8 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
207213
current_zstd_window_log_size = zstd_window_log_size;
208214
}
209215

210-
int ret = ZSTD_decompressDCtx(current_zstd_d_ctx, p_dst, p_dst_max_size, p_src, p_src_size);
211-
return ret;
216+
size_t ret = ZSTD_decompressDCtx(current_zstd_d_ctx, p_dst, p_dst_max_size, p_src, p_src_size);
217+
return (int64_t)ret;
212218
} break;
213219
}
214220

@@ -220,7 +226,7 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
220226
This is required for compressed data whose final uncompressed size is unknown, as is the case for HTTP response bodies.
221227
This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer.
222228
*/
223-
int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
229+
int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int64_t p_max_dst_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode) {
224230
uint8_t *dst = nullptr;
225231
int out_mark = 0;
226232

core/io/compression.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,8 @@ class Compression {
5252
MODE_BROTLI
5353
};
5454

55-
static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
56-
static int get_max_compressed_buffer_size(int p_src_size, Mode p_mode = MODE_ZSTD);
57-
static int decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
58-
static int decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode);
55+
static int64_t compress(uint8_t *p_dst, const uint8_t *p_src, int64_t p_src_size, Mode p_mode = MODE_ZSTD);
56+
static int64_t get_max_compressed_buffer_size(int64_t p_src_size, Mode p_mode = MODE_ZSTD);
57+
static int64_t decompress(uint8_t *p_dst, int64_t p_dst_max_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode = MODE_ZSTD);
58+
static int decompress_dynamic(Vector<uint8_t> *p_dst_vect, int64_t p_max_dst_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode);
5959
};

core/io/file_access_compressed.cpp

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ Error FileAccessCompressed::open_after_magic(Ref<FileAccess> p_base) {
6868
read_block_count = bc;
6969
read_block_size = read_blocks.size() == 1 ? read_total : block_size;
7070

71-
int ret = Compression::decompress(buffer.ptrw(), read_block_size, comp_buffer.ptr(), read_blocks[0].csize, cmode);
71+
const int64_t ret = Compression::decompress(buffer.ptrw(), read_block_size, comp_buffer.ptr(), read_blocks[0].csize, cmode);
7272
read_block = 0;
7373
read_pos = 0;
7474

@@ -137,10 +137,11 @@ void FileAccessCompressed::_close() {
137137

138138
Vector<uint8_t> cblock;
139139
cblock.resize(Compression::get_max_compressed_buffer_size(bl, cmode));
140-
int s = Compression::compress(cblock.ptrw(), bp, bl, cmode);
140+
const int64_t compressed_size = Compression::compress(cblock.ptrw(), bp, bl, cmode);
141+
ERR_FAIL_COND_MSG(compressed_size < 0, "FileAccessCompressed: Error compressing data.");
141142

142-
f->store_buffer(cblock.ptr(), s);
143-
block_sizes.push_back(s);
143+
f->store_buffer(cblock.ptr(), (uint64_t)compressed_size);
144+
block_sizes.push_back(compressed_size);
144145
}
145146

146147
f->seek(16); //ok write block sizes
@@ -200,7 +201,7 @@ void FileAccessCompressed::seek(uint64_t p_position) {
200201
read_block = block_idx;
201202
f->seek(read_blocks[read_block].offset);
202203
f->get_buffer(comp_buffer.ptrw(), read_blocks[read_block].csize);
203-
int ret = Compression::decompress(buffer.ptrw(), read_blocks.size() == 1 ? read_total : block_size, comp_buffer.ptr(), read_blocks[read_block].csize, cmode);
204+
const int64_t ret = Compression::decompress(buffer.ptrw(), read_blocks.size() == 1 ? read_total : block_size, comp_buffer.ptr(), read_blocks[read_block].csize, cmode);
204205
ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt.");
205206
read_block_size = read_block == read_block_count - 1 ? read_total % block_size : block_size;
206207
}
@@ -288,7 +289,7 @@ uint64_t FileAccessCompressed::get_buffer(uint8_t *p_dst, uint64_t p_length) con
288289

289290
// Read the next block of compressed data.
290291
f->get_buffer(comp_buffer.ptrw(), read_blocks[read_block].csize);
291-
int ret = Compression::decompress(buffer.ptrw(), read_blocks.size() == 1 ? read_total : block_size, comp_buffer.ptr(), read_blocks[read_block].csize, cmode);
292+
const int64_t ret = Compression::decompress(buffer.ptrw(), read_blocks.size() == 1 ? read_total : block_size, comp_buffer.ptr(), read_blocks[read_block].csize, cmode);
292293
ERR_FAIL_COND_V_MSG(ret == -1, -1, "Compressed file is corrupt.");
293294
read_block_size = read_block == read_block_count - 1 ? read_total % block_size : block_size;
294295
read_pos = 0;

core/io/remote_filesystem_client.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,7 @@ Error RemoteFilesystemClient::_synchronize_with_server(const String &p_host, int
215215
Vector<uint8_t> file_cache_buffer;
216216
if (file_cache.size()) {
217217
StringBuilder sbuild;
218-
for (int i = 0; i < file_cache.size(); i++) {
218+
for (int64_t i = 0; i < file_cache.size(); i++) {
219219
sbuild.append(file_cache[i].path);
220220
sbuild.append("::");
221221
sbuild.append(itos(file_cache[i].server_modified_time));
@@ -224,7 +224,7 @@ Error RemoteFilesystemClient::_synchronize_with_server(const String &p_host, int
224224
String s = sbuild.as_string();
225225
CharString cs = s.utf8();
226226
file_cache_buffer.resize(Compression::get_max_compressed_buffer_size(cs.length(), Compression::MODE_ZSTD));
227-
int res_len = Compression::compress(file_cache_buffer.ptrw(), (const uint8_t *)cs.ptr(), cs.length(), Compression::MODE_ZSTD);
227+
const int64_t res_len = Compression::compress(file_cache_buffer.ptrw(), (const uint8_t *)cs.ptr(), cs.length(), Compression::MODE_ZSTD);
228228
file_cache_buffer.resize(res_len);
229229

230230
tcp_client->put_32(cs.length()); // Size of buffer uncompressed

core/variant/variant_call.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -799,7 +799,7 @@ struct _VariantCall {
799799
if (p_instance->size() > 0) {
800800
Compression::Mode mode = (Compression::Mode)(p_mode);
801801
compressed.resize(Compression::get_max_compressed_buffer_size(p_instance->size(), mode));
802-
int result = Compression::compress(compressed.ptrw(), p_instance->ptr(), p_instance->size(), mode);
802+
int64_t result = Compression::compress(compressed.ptrw(), p_instance->ptr(), p_instance->size(), mode);
803803

804804
result = result >= 0 ? result : 0;
805805
compressed.resize(result);
@@ -822,7 +822,7 @@ struct _VariantCall {
822822
}
823823

824824
decompressed.resize(buffer_size);
825-
int result = Compression::decompress(decompressed.ptrw(), buffer_size, p_instance->ptr(), p_instance->size(), mode);
825+
int64_t result = Compression::decompress(decompressed.ptrw(), buffer_size, p_instance->ptr(), p_instance->size(), mode);
826826

827827
result = result >= 0 ? result : 0;
828828
decompressed.resize(result);

editor/debugger/editor_file_server.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,8 @@ void EditorFileServer::poll() {
173173

174174
ERR_FAIL_COND(err != OK);
175175
// Decompress the text with all the files
176-
Compression::decompress(file_buffer_decompressed.ptr(), file_buffer_decompressed.size(), file_buffer.ptr(), file_buffer.size(), Compression::MODE_ZSTD);
176+
const int64_t decompressed_size = Compression::decompress(file_buffer_decompressed.ptr(), file_buffer_decompressed.size(), file_buffer.ptr(), file_buffer.size(), Compression::MODE_ZSTD);
177+
ERR_FAIL_COND_MSG(decompressed_size != file_buffer_decompressed.size(), "Error decompressing file buffer. Decompressed size did not match the expected size.");
177178
String files_text = String::utf8((const char *)file_buffer_decompressed.ptr(), file_buffer_decompressed.size());
178179
Vector<String> files = files_text.split("\n");
179180

editor/doc_tools.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1798,10 +1798,10 @@ Error DocTools::save_classes(const String &p_default_path, const HashMap<String,
17981798
return OK;
17991799
}
18001800

1801-
Error DocTools::load_compressed(const uint8_t *p_data, int p_compressed_size, int p_uncompressed_size) {
1801+
Error DocTools::load_compressed(const uint8_t *p_data, int64_t p_compressed_size, int64_t p_uncompressed_size) {
18021802
Vector<uint8_t> data;
18031803
data.resize(p_uncompressed_size);
1804-
int ret = Compression::decompress(data.ptrw(), p_uncompressed_size, p_data, p_compressed_size, Compression::MODE_DEFLATE);
1804+
const int64_t ret = Compression::decompress(data.ptrw(), p_uncompressed_size, p_data, p_compressed_size, Compression::MODE_DEFLATE);
18051805
ERR_FAIL_COND_V_MSG(ret == -1, ERR_FILE_CORRUPT, "Compressed file is corrupt.");
18061806
class_list.clear();
18071807

@@ -1816,7 +1816,7 @@ Error DocTools::load_compressed(const uint8_t *p_data, int p_compressed_size, in
18161816
return OK;
18171817
}
18181818

1819-
Error DocTools::load_xml(const uint8_t *p_data, int p_size) {
1819+
Error DocTools::load_xml(const uint8_t *p_data, int64_t p_size) {
18201820
Ref<XMLParser> parser = memnew(XMLParser);
18211821
Error err = parser->_open_buffer(p_data, p_size);
18221822
if (err) {

editor/doc_tools.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,6 @@ class DocTools {
5555
Error save_classes(const String &p_default_path, const HashMap<String, String> &p_class_path, bool p_use_relative_schema = true);
5656

5757
Error _load(Ref<XMLParser> parser);
58-
Error load_compressed(const uint8_t *p_data, int p_compressed_size, int p_uncompressed_size);
59-
Error load_xml(const uint8_t *p_data, int p_size);
58+
Error load_compressed(const uint8_t *p_data, int64_t p_compressed_size, int64_t p_uncompressed_size);
59+
Error load_xml(const uint8_t *p_data, int64_t p_size);
6060
};

editor/editor_translation.cpp

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ void load_editor_translations(const String &p_locale) {
6161
if (etl->lang == p_locale) {
6262
Vector<uint8_t> data;
6363
data.resize(etl->uncomp_size);
64-
int ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
64+
const int64_t ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
6565
ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt.");
6666

6767
Ref<FileAccessMemory> fa;
@@ -89,7 +89,7 @@ void load_property_translations(const String &p_locale) {
8989
if (etl->lang == p_locale) {
9090
Vector<uint8_t> data;
9191
data.resize(etl->uncomp_size);
92-
int ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
92+
const int64_t ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
9393
ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt.");
9494

9595
Ref<FileAccessMemory> fa;
@@ -117,7 +117,7 @@ void load_doc_translations(const String &p_locale) {
117117
if (dtl->lang == p_locale) {
118118
Vector<uint8_t> data;
119119
data.resize(dtl->uncomp_size);
120-
int ret = Compression::decompress(data.ptrw(), dtl->uncomp_size, dtl->data, dtl->comp_size, Compression::MODE_DEFLATE);
120+
const int64_t ret = Compression::decompress(data.ptrw(), dtl->uncomp_size, dtl->data, dtl->comp_size, Compression::MODE_DEFLATE);
121121
ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt.");
122122

123123
Ref<FileAccessMemory> fa;
@@ -145,7 +145,7 @@ void load_extractable_translations(const String &p_locale) {
145145
if (etl->lang == p_locale) {
146146
Vector<uint8_t> data;
147147
data.resize(etl->uncomp_size);
148-
int ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
148+
const int64_t ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
149149
ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt.");
150150

151151
Ref<FileAccessMemory> fa;
@@ -177,7 +177,7 @@ Vector<Vector<String>> get_extractable_message_list() {
177177

178178
Vector<uint8_t> data;
179179
data.resize(etl->uncomp_size);
180-
int ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
180+
const int64_t ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
181181
ERR_FAIL_COND_V_MSG(ret == -1, list, "Compressed file is corrupt.");
182182

183183
Ref<FileAccessMemory> fa;

modules/enet/enet_connection.cpp

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -422,11 +422,11 @@ size_t ENetConnection::Compressor::enet_compress(void *context, const ENetBuffer
422422
compressor->src_mem.resize(inLimit);
423423
}
424424

425-
int total = inLimit;
426-
int ofs = 0;
425+
size_t total = inLimit;
426+
size_t ofs = 0;
427427
while (total) {
428428
for (size_t i = 0; i < inBufferCount; i++) {
429-
int to_copy = MIN(total, int(inBuffers[i].dataLength));
429+
const size_t to_copy = MIN(total, inBuffers[i].dataLength);
430430
memcpy(&compressor->src_mem.write[ofs], inBuffers[i].data, to_copy);
431431
ofs += to_copy;
432432
total -= to_copy;
@@ -450,28 +450,29 @@ size_t ENetConnection::Compressor::enet_compress(void *context, const ENetBuffer
450450
}
451451
}
452452

453-
int req_size = Compression::get_max_compressed_buffer_size(ofs, mode);
453+
const int64_t req_size = Compression::get_max_compressed_buffer_size(ofs, mode);
454454
if (compressor->dst_mem.size() < req_size) {
455455
compressor->dst_mem.resize(req_size);
456456
}
457-
int ret = Compression::compress(compressor->dst_mem.ptrw(), compressor->src_mem.ptr(), ofs, mode);
457+
const int64_t ret = Compression::compress(compressor->dst_mem.ptrw(), compressor->src_mem.ptr(), ofs, mode);
458458

459459
if (ret < 0) {
460460
return 0;
461461
}
462462

463-
if (ret > int(outLimit)) {
463+
const size_t ret_size = size_t(ret);
464+
if (ret_size > outLimit) {
464465
return 0; // Do not bother
465466
}
466467

467-
memcpy(outData, compressor->dst_mem.ptr(), ret);
468+
memcpy(outData, compressor->dst_mem.ptr(), ret_size);
468469

469470
return ret;
470471
}
471472

472473
size_t ENetConnection::Compressor::enet_decompress(void *context, const enet_uint8 *inData, size_t inLimit, enet_uint8 *outData, size_t outLimit) {
473474
Compressor *compressor = (Compressor *)(context);
474-
int ret = -1;
475+
int64_t ret = -1;
475476
switch (compressor->mode) {
476477
case COMPRESS_FASTLZ: {
477478
ret = Compression::decompress(outData, outLimit, inData, inLimit, Compression::MODE_FASTLZ);

0 commit comments

Comments
 (0)