Skip to content

Commit a35d78c

Browse files
gitstergitster
committed
Merge branch 'jc/zlib-wrap' into maint
* jc/zlib-wrap: zlib: allow feeding more than 4GB in one go zlib: zlib can only process 4GB at a time zlib: wrap deflateBound() too zlib: wrap deflate side of the API zlib: wrap inflateInit2 used to accept only for gzip format zlib: wrap remaining calls to direct inflate/inflateEnd zlib wrapper: refactor error message formatter
2 parents 184cb4d + e01503b commit a35d78c

15 files changed

+321
-124
lines changed

‎archive-zip.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -90,14 +90,14 @@ static void copy_le32(unsigned char *dest, unsigned int n)
9090
static void *zlib_deflate(void *data, unsigned long size,
9191
int compression_level, unsigned long *compressed_size)
9292
{
93-
z_stream stream;
93+
git_zstream stream;
9494
unsigned long maxsize;
9595
void *buffer;
9696
int result;
9797

9898
memset(&stream, 0, sizeof(stream));
99-
deflateInit(&stream, compression_level);
100-
maxsize = deflateBound(&stream, size);
99+
git_deflate_init(&stream, compression_level);
100+
maxsize = git_deflate_bound(&stream, size);
101101
buffer = xmalloc(maxsize);
102102

103103
stream.next_in = data;
@@ -106,15 +106,15 @@ static void *zlib_deflate(void *data, unsigned long size,
106106
stream.avail_out = maxsize;
107107

108108
do {
109-
result = deflate(&stream, Z_FINISH);
109+
result = git_deflate(&stream, Z_FINISH);
110110
} while (result == Z_OK);
111111

112112
if (result != Z_STREAM_END) {
113113
free(buffer);
114114
return NULL;
115115
}
116116

117-
deflateEnd(&stream);
117+
git_deflate_end(&stream);
118118
*compressed_size = stream.total_out;
119119

120120
return buffer;

‎builtin/apply.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1634,7 +1634,7 @@ static inline int metadata_changes(struct patch *patch)
16341634
static char *inflate_it(const void *data, unsigned long size,
16351635
unsigned long inflated_size)
16361636
{
1637-
z_stream stream;
1637+
git_zstream stream;
16381638
void *out;
16391639
int st;
16401640

‎builtin/index-pack.c

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ static void unlink_base_data(struct base_data *c)
265265
static void *unpack_entry_data(unsigned long offset, unsigned long size)
266266
{
267267
int status;
268-
z_stream stream;
268+
git_zstream stream;
269269
void *buf = xmalloc(size);
270270

271271
memset(&stream, 0, sizeof(stream));
@@ -355,7 +355,7 @@ static void *get_data_from_pack(struct object_entry *obj)
355355
off_t from = obj[0].idx.offset + obj[0].hdr_size;
356356
unsigned long len = obj[1].idx.offset - from;
357357
unsigned char *data, *inbuf;
358-
z_stream stream;
358+
git_zstream stream;
359359
int status;
360360

361361
data = xmalloc(obj->size);
@@ -666,26 +666,26 @@ static void parse_pack_objects(unsigned char *sha1)
666666

667667
static int write_compressed(struct sha1file *f, void *in, unsigned int size)
668668
{
669-
z_stream stream;
669+
git_zstream stream;
670670
int status;
671671
unsigned char outbuf[4096];
672672

673673
memset(&stream, 0, sizeof(stream));
674-
deflateInit(&stream, zlib_compression_level);
674+
git_deflate_init(&stream, zlib_compression_level);
675675
stream.next_in = in;
676676
stream.avail_in = size;
677677

678678
do {
679679
stream.next_out = outbuf;
680680
stream.avail_out = sizeof(outbuf);
681-
status = deflate(&stream, Z_FINISH);
681+
status = git_deflate(&stream, Z_FINISH);
682682
sha1write(f, outbuf, sizeof(outbuf) - stream.avail_out);
683683
} while (status == Z_OK);
684684

685685
if (status != Z_STREAM_END)
686686
die("unable to deflate appended object (%d)", status);
687687
size = stream.total_out;
688-
deflateEnd(&stream);
688+
git_deflate_end(&stream);
689689
return size;
690690
}
691691

‎builtin/pack-objects.c

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -126,13 +126,13 @@ static void *get_delta(struct object_entry *entry)
126126

127127
static unsigned long do_compress(void **pptr, unsigned long size)
128128
{
129-
z_stream stream;
129+
git_zstream stream;
130130
void *in, *out;
131131
unsigned long maxsize;
132132

133133
memset(&stream, 0, sizeof(stream));
134-
deflateInit(&stream, pack_compression_level);
135-
maxsize = deflateBound(&stream, size);
134+
git_deflate_init(&stream, pack_compression_level);
135+
maxsize = git_deflate_bound(&stream, size);
136136

137137
in = *pptr;
138138
out = xmalloc(maxsize);
@@ -142,9 +142,9 @@ static unsigned long do_compress(void **pptr, unsigned long size)
142142
stream.avail_in = size;
143143
stream.next_out = out;
144144
stream.avail_out = maxsize;
145-
while (deflate(&stream, Z_FINISH) == Z_OK)
145+
while (git_deflate(&stream, Z_FINISH) == Z_OK)
146146
; /* nothing */
147-
deflateEnd(&stream);
147+
git_deflate_end(&stream);
148148

149149
free(in);
150150
return stream.total_out;
@@ -160,7 +160,7 @@ static int check_pack_inflate(struct packed_git *p,
160160
off_t len,
161161
unsigned long expect)
162162
{
163-
z_stream stream;
163+
git_zstream stream;
164164
unsigned char fakebuf[4096], *in;
165165
int st;
166166

@@ -187,12 +187,12 @@ static void copy_pack_data(struct sha1file *f,
187187
off_t len)
188188
{
189189
unsigned char *in;
190-
unsigned int avail;
190+
unsigned long avail;
191191

192192
while (len) {
193193
in = use_pack(p, w_curs, offset, &avail);
194194
if (avail > len)
195-
avail = (unsigned int)len;
195+
avail = (unsigned long)len;
196196
sha1write(f, in, avail);
197197
offset += avail;
198198
len -= avail;
@@ -994,7 +994,7 @@ static void check_object(struct object_entry *entry)
994994
const unsigned char *base_ref = NULL;
995995
struct object_entry *base_entry;
996996
unsigned long used, used_0;
997-
unsigned int avail;
997+
unsigned long avail;
998998
off_t ofs;
999999
unsigned char *buf, c;
10001000

‎builtin/unpack-objects.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ static void use(int bytes)
9090

9191
static void *get_data(unsigned long size)
9292
{
93-
z_stream stream;
93+
git_zstream stream;
9494
void *buf = xmalloc(size);
9595

9696
memset(&stream, 0, sizeof(stream));

‎cache.h

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,27 @@
1616
#endif
1717

1818
#include <zlib.h>
19-
#if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200
20-
#define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11)
21-
#endif
22-
23-
void git_inflate_init(z_streamp strm);
24-
void git_inflate_end(z_streamp strm);
25-
int git_inflate(z_streamp strm, int flush);
19+
typedef struct git_zstream {
20+
z_stream z;
21+
unsigned long avail_in;
22+
unsigned long avail_out;
23+
unsigned long total_in;
24+
unsigned long total_out;
25+
unsigned char *next_in;
26+
unsigned char *next_out;
27+
} git_zstream;
28+
29+
void git_inflate_init(git_zstream *);
30+
void git_inflate_init_gzip_only(git_zstream *);
31+
void git_inflate_end(git_zstream *);
32+
int git_inflate(git_zstream *, int flush);
33+
34+
void git_deflate_init(git_zstream *, int level);
35+
void git_deflate_init_gzip(git_zstream *, int level);
36+
void git_deflate_end(git_zstream *);
37+
int git_deflate_end_gently(git_zstream *);
38+
int git_deflate(git_zstream *, int flush);
39+
unsigned long git_deflate_bound(git_zstream *, unsigned long);
2640

2741
#if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT)
2842
#define DTYPE(de) ((de)->d_type)
@@ -1009,7 +1023,7 @@ extern struct packed_git *find_sha1_pack(const unsigned char *sha1,
10091023
extern void pack_report(void);
10101024
extern int open_pack_index(struct packed_git *);
10111025
extern void close_pack_index(struct packed_git *);
1012-
extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *);
1026+
extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *);
10131027
extern void close_pack_windows(struct packed_git *);
10141028
extern void unuse_pack(struct pack_window **);
10151029
extern void free_pack_by_name(const char *);

‎diff.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1839,20 +1839,20 @@ static unsigned char *deflate_it(char *data,
18391839
{
18401840
int bound;
18411841
unsigned char *deflated;
1842-
z_stream stream;
1842+
git_zstream stream;
18431843

18441844
memset(&stream, 0, sizeof(stream));
1845-
deflateInit(&stream, zlib_compression_level);
1846-
bound = deflateBound(&stream, size);
1845+
git_deflate_init(&stream, zlib_compression_level);
1846+
bound = git_deflate_bound(&stream, size);
18471847
deflated = xmalloc(bound);
18481848
stream.next_out = deflated;
18491849
stream.avail_out = bound;
18501850

18511851
stream.next_in = (unsigned char *)data;
18521852
stream.avail_in = size;
1853-
while (deflate(&stream, Z_FINISH) == Z_OK)
1853+
while (git_deflate(&stream, Z_FINISH) == Z_OK)
18541854
; /* nothing */
1855-
deflateEnd(&stream);
1855+
git_deflate_end(&stream);
18561856
*result_size = stream.total_out;
18571857
return deflated;
18581858
}

‎fast-import.c

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1017,7 +1017,7 @@ static int store_object(
10171017
unsigned char sha1[20];
10181018
unsigned long hdrlen, deltalen;
10191019
git_SHA_CTX c;
1020-
z_stream s;
1020+
git_zstream s;
10211021

10221022
hdrlen = sprintf((char *)hdr,"%s %lu", typename(type),
10231023
(unsigned long)dat->len) + 1;
@@ -1050,19 +1050,19 @@ static int store_object(
10501050
delta = NULL;
10511051

10521052
memset(&s, 0, sizeof(s));
1053-
deflateInit(&s, pack_compression_level);
1053+
git_deflate_init(&s, pack_compression_level);
10541054
if (delta) {
10551055
s.next_in = delta;
10561056
s.avail_in = deltalen;
10571057
} else {
10581058
s.next_in = (void *)dat->buf;
10591059
s.avail_in = dat->len;
10601060
}
1061-
s.avail_out = deflateBound(&s, s.avail_in);
1061+
s.avail_out = git_deflate_bound(&s, s.avail_in);
10621062
s.next_out = out = xmalloc(s.avail_out);
1063-
while (deflate(&s, Z_FINISH) == Z_OK)
1064-
/* nothing */;
1065-
deflateEnd(&s);
1063+
while (git_deflate(&s, Z_FINISH) == Z_OK)
1064+
; /* nothing */
1065+
git_deflate_end(&s);
10661066

10671067
/* Determine if we should auto-checkpoint. */
10681068
if ((max_packsize && (pack_size + 60 + s.total_out) > max_packsize)
@@ -1078,14 +1078,14 @@ static int store_object(
10781078
delta = NULL;
10791079

10801080
memset(&s, 0, sizeof(s));
1081-
deflateInit(&s, pack_compression_level);
1081+
git_deflate_init(&s, pack_compression_level);
10821082
s.next_in = (void *)dat->buf;
10831083
s.avail_in = dat->len;
1084-
s.avail_out = deflateBound(&s, s.avail_in);
1084+
s.avail_out = git_deflate_bound(&s, s.avail_in);
10851085
s.next_out = out = xrealloc(out, s.avail_out);
1086-
while (deflate(&s, Z_FINISH) == Z_OK)
1087-
/* nothing */;
1088-
deflateEnd(&s);
1086+
while (git_deflate(&s, Z_FINISH) == Z_OK)
1087+
; /* nothing */
1088+
git_deflate_end(&s);
10891089
}
10901090
}
10911091

@@ -1163,7 +1163,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark)
11631163
off_t offset;
11641164
git_SHA_CTX c;
11651165
git_SHA_CTX pack_file_ctx;
1166-
z_stream s;
1166+
git_zstream s;
11671167
int status = Z_OK;
11681168

11691169
/* Determine if we should auto-checkpoint. */
@@ -1187,7 +1187,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark)
11871187
crc32_begin(pack_file);
11881188

11891189
memset(&s, 0, sizeof(s));
1190-
deflateInit(&s, pack_compression_level);
1190+
git_deflate_init(&s, pack_compression_level);
11911191

11921192
hdrlen = encode_in_pack_object_header(OBJ_BLOB, len, out_buf);
11931193
if (out_sz <= hdrlen)
@@ -1209,7 +1209,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark)
12091209
len -= n;
12101210
}
12111211

1212-
status = deflate(&s, len ? 0 : Z_FINISH);
1212+
status = git_deflate(&s, len ? 0 : Z_FINISH);
12131213

12141214
if (!s.avail_out || status == Z_STREAM_END) {
12151215
size_t n = s.next_out - out_buf;
@@ -1228,7 +1228,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark)
12281228
die("unexpected deflate failure: %d", status);
12291229
}
12301230
}
1231-
deflateEnd(&s);
1231+
git_deflate_end(&s);
12321232
git_SHA1_Final(sha1, &c);
12331233

12341234
if (sha1out)

‎http-backend.c

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -271,16 +271,13 @@ static struct rpc_service *select_service(const char *name)
271271

272272
static void inflate_request(const char *prog_name, int out)
273273
{
274-
z_stream stream;
274+
git_zstream stream;
275275
unsigned char in_buf[8192];
276276
unsigned char out_buf[8192];
277277
unsigned long cnt = 0;
278-
int ret;
279278

280279
memset(&stream, 0, sizeof(stream));
281-
ret = inflateInit2(&stream, (15 + 16));
282-
if (ret != Z_OK)
283-
die("cannot start zlib inflater, zlib err %d", ret);
280+
git_inflate_init_gzip_only(&stream);
284281

285282
while (1) {
286283
ssize_t n = xread(0, in_buf, sizeof(in_buf));
@@ -296,7 +293,7 @@ static void inflate_request(const char *prog_name, int out)
296293
stream.next_out = out_buf;
297294
stream.avail_out = sizeof(out_buf);
298295

299-
ret = inflate(&stream, Z_NO_FLUSH);
296+
ret = git_inflate(&stream, Z_NO_FLUSH);
300297
if (ret != Z_OK && ret != Z_STREAM_END)
301298
die("zlib error inflating request, result %d", ret);
302299

@@ -311,7 +308,7 @@ static void inflate_request(const char *prog_name, int out)
311308
}
312309

313310
done:
314-
inflateEnd(&stream);
311+
git_inflate_end(&stream);
315312
close(out);
316313
}
317314

0 commit comments

Comments
 (0)