Skip to content

Commit 583dab0

Browse files
gitstergitster
committed
Merge branch 'tb/weak-sha1-for-tail-sum' into jch
The checksum at the tail of files are now computed without collision detection protection. * tb/weak-sha1-for-tail-sum: csum-file.c: use fast SHA-1 implementation when available Makefile: allow specifying a SHA-1 for non-cryptographic uses hash.h: scaffolding for _fast hashing variants sha1: do not redefine `platform_SHA_CTX` and friends i5500-git-daemon.sh: use compile-able version of Git without OpenSSL pack-objects: use finalize_object_file() to rename pack/idx/etc finalize_object_file(): implement collision check finalize_object_file(): refactor unlink_or_warn() placement finalize_object_file(): check for name collision before renaming
2 parents 5316f76 + abe19da commit 583dab0

File tree

9 files changed

+229
-17
lines changed

9 files changed

+229
-17
lines changed

‎Makefile‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -521,6 +521,10 @@ include shared.mak
521521
# Define APPLE_COMMON_CRYPTO_SHA1 to use Apple's CommonCrypto for
522522
# SHA-1.
523523
#
524+
# Define the same Makefile knobs as above, but suffixed with _FAST to
525+
# use the corresponding implementations for "fast" SHA-1 hashing for
526+
# non-cryptographic purposes.
527+
#
524528
# If don't enable any of the *_SHA1 settings in this section, Git will
525529
# default to its built-in sha1collisiondetection library, which is a
526530
# collision-detecting sha1 This is slower, but may detect attempted
@@ -2000,6 +2004,27 @@ endif
20002004
endif
20012005
endif
20022006

2007+
ifdef OPENSSL_SHA1_FAST
2008+
ifndef OPENSSL_SHA1
2009+
EXTLIBS += $(LIB_4_CRYPTO)
2010+
BASIC_CFLAGS += -DSHA1_OPENSSL_FAST
2011+
endif
2012+
else
2013+
ifdef BLK_SHA1_FAST
2014+
ifndef BLK_SHA1
2015+
LIB_OBJS += block-sha1/sha1.o
2016+
BASIC_CFLAGS += -DSHA1_BLK_FAST
2017+
endif
2018+
else
2019+
ifdef APPLE_COMMON_CRYPTO_SHA1_FAST
2020+
ifndef APPLE_COMMON_CRYPTO_SHA1
2021+
COMPAT_CFLAGS += -DCOMMON_DIGEST_FOR_OPENSSL
2022+
BASIC_CFLAGS += -DSHA1_APPLE_FAST
2023+
endif
2024+
endif
2025+
endif
2026+
endif
2027+
20032028
ifdef OPENSSL_SHA256
20042029
EXTLIBS += $(LIB_4_CRYPTO)
20052030
BASIC_CFLAGS += -DSHA256_OPENSSL

‎block-sha1/sha1.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@ void blk_SHA1_Init(blk_SHA_CTX *ctx);
1616
void blk_SHA1_Update(blk_SHA_CTX *ctx, const void *dataIn, size_t len);
1717
void blk_SHA1_Final(unsigned char hashout[20], blk_SHA_CTX *ctx);
1818

19+
#ifndef platform_SHA_CTX
1920
#define platform_SHA_CTX blk_SHA_CTX
2021
#define platform_SHA1_Init blk_SHA1_Init
2122
#define platform_SHA1_Update blk_SHA1_Update
2223
#define platform_SHA1_Final blk_SHA1_Final
24+
#endif

‎csum-file.c‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ void hashflush(struct hashfile *f)
5050

5151
if (offset) {
5252
if (!f->skip_hash)
53-
the_hash_algo->update_fn(&f->ctx, f->buffer, offset);
53+
the_hash_algo->fast_update_fn(&f->ctx, f->buffer, offset);
5454
flush(f, f->buffer, offset);
5555
f->offset = 0;
5656
}
@@ -73,7 +73,7 @@ int finalize_hashfile(struct hashfile *f, unsigned char *result,
7373
if (f->skip_hash)
7474
hashclr(f->buffer, the_repository->hash_algo);
7575
else
76-
the_hash_algo->final_fn(f->buffer, &f->ctx);
76+
the_hash_algo->fast_final_fn(f->buffer, &f->ctx);
7777

7878
if (result)
7979
hashcpy(result, f->buffer, the_repository->hash_algo);
@@ -128,7 +128,7 @@ void hashwrite(struct hashfile *f, const void *buf, unsigned int count)
128128
* f->offset is necessarily zero.
129129
*/
130130
if (!f->skip_hash)
131-
the_hash_algo->update_fn(&f->ctx, buf, nr);
131+
the_hash_algo->fast_update_fn(&f->ctx, buf, nr);
132132
flush(f, buf, nr);
133133
} else {
134134
/*
@@ -174,7 +174,7 @@ static struct hashfile *hashfd_internal(int fd, const char *name,
174174
f->name = name;
175175
f->do_crc = 0;
176176
f->skip_hash = 0;
177-
the_hash_algo->init_fn(&f->ctx);
177+
the_hash_algo->fast_init_fn(&f->ctx);
178178

179179
f->buffer_len = buffer_len;
180180
f->buffer = xmalloc(buffer_len);
@@ -208,7 +208,7 @@ void hashfile_checkpoint(struct hashfile *f, struct hashfile_checkpoint *checkpo
208208
{
209209
hashflush(f);
210210
checkpoint->offset = f->total;
211-
the_hash_algo->clone_fn(&checkpoint->ctx, &f->ctx);
211+
the_hash_algo->fast_clone_fn(&checkpoint->ctx, &f->ctx);
212212
}
213213

214214
int hashfile_truncate(struct hashfile *f, struct hashfile_checkpoint *checkpoint)
@@ -219,7 +219,7 @@ int hashfile_truncate(struct hashfile *f, struct hashfile_checkpoint *checkpoint
219219
lseek(f->fd, offset, SEEK_SET) != offset)
220220
return -1;
221221
f->total = offset;
222-
the_hash_algo->clone_fn(&f->ctx, &checkpoint->ctx);
222+
the_hash_algo->fast_clone_fn(&f->ctx, &checkpoint->ctx);
223223
f->offset = 0; /* hashflush() was called in checkpoint */
224224
return 0;
225225
}
@@ -245,9 +245,9 @@ int hashfile_checksum_valid(const unsigned char *data, size_t total_len)
245245
if (total_len < the_hash_algo->rawsz)
246246
return 0; /* say "too short"? */
247247

248-
the_hash_algo->init_fn(&ctx);
249-
the_hash_algo->update_fn(&ctx, data, data_len);
250-
the_hash_algo->final_fn(got, &ctx);
248+
the_hash_algo->fast_init_fn(&ctx);
249+
the_hash_algo->fast_update_fn(&ctx, data, data_len);
250+
the_hash_algo->fast_final_fn(got, &ctx);
251251

252252
return hasheq(got, data + data_len, the_repository->hash_algo);
253253
}

‎hash.h‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,36 @@
1515
#include "block-sha1/sha1.h"
1616
#endif
1717

18+
#if defined(SHA1_APPLE_FAST)
19+
# include <CommonCrypto/CommonDigest.h>
20+
# define platform_SHA_CTX_fast CC_SHA1_CTX
21+
# define platform_SHA1_Init_fast CC_SHA1_Init
22+
# define platform_SHA1_Update_fast CC_SHA1_Update
23+
# define platform_SHA1_Final_fast CC_SHA1_Final
24+
#elif defined(SHA1_OPENSSL_FAST)
25+
# include <openssl/sha.h>
26+
# if defined(OPENSSL_API_LEVEL) && OPENSSL_API_LEVEL >= 3
27+
# define SHA1_NEEDS_CLONE_HELPER_FAST
28+
# include "sha1/openssl.h"
29+
# define platform_SHA_CTX_fast openssl_SHA1_CTX
30+
# define platform_SHA1_Init_fast openssl_SHA1_Init
31+
# define platform_SHA1_Clone_fast openssl_SHA1_Clone
32+
# define platform_SHA1_Update_fast openssl_SHA1_Update
33+
# define platform_SHA1_Final_fast openssl_SHA1_Final
34+
# else
35+
# define platform_SHA_CTX_fast SHA_CTX
36+
# define platform_SHA1_Init_fast SHA1_Init
37+
# define platform_SHA1_Update_fast SHA1_Update
38+
# define platform_SHA1_Final_fast SHA1_Final
39+
# endif
40+
#elif defined(SHA1_BLK_FAST)
41+
# include "block-sha1/sha1.h"
42+
# define platform_SHA_CTX_fast blk_SHA_CTX
43+
# define platform_SHA1_Init_fast blk_SHA1_Init
44+
# define platform_SHA1_Update_fast blk_SHA1_Update
45+
# define platform_SHA1_Final_fast blk_SHA1_Final
46+
#endif
47+
1848
#if defined(SHA256_NETTLE)
1949
#include "sha256/nettle.h"
2050
#elif defined(SHA256_GCRYPT)
@@ -44,14 +74,32 @@
4474
#define platform_SHA1_Final SHA1_Final
4575
#endif
4676

77+
#ifndef platform_SHA_CTX_fast
78+
# define platform_SHA_CTX_fast platform_SHA_CTX
79+
# define platform_SHA1_Init_fast platform_SHA1_Init
80+
# define platform_SHA1_Update_fast platform_SHA1_Update
81+
# define platform_SHA1_Final_fast platform_SHA1_Final
82+
# ifdef platform_SHA1_Clone
83+
# define platform_SHA1_Clone_fast platform_SHA1_Clone
84+
# endif
85+
#endif
86+
4787
#define git_SHA_CTX platform_SHA_CTX
4888
#define git_SHA1_Init platform_SHA1_Init
4989
#define git_SHA1_Update platform_SHA1_Update
5090
#define git_SHA1_Final platform_SHA1_Final
5191

92+
#define git_SHA_CTX_fast platform_SHA_CTX_fast
93+
#define git_SHA1_Init_fast platform_SHA1_Init_fast
94+
#define git_SHA1_Update_fast platform_SHA1_Update_fast
95+
#define git_SHA1_Final_fast platform_SHA1_Final_fast
96+
5297
#ifdef platform_SHA1_Clone
5398
#define git_SHA1_Clone platform_SHA1_Clone
5499
#endif
100+
#ifdef platform_SHA1_Clone_fast
101+
# define git_SHA1_Clone_fast platform_SHA1_Clone_fast
102+
#endif
55103

56104
#ifndef platform_SHA256_CTX
57105
#define platform_SHA256_CTX SHA256_CTX
@@ -81,6 +129,13 @@ static inline void git_SHA1_Clone(git_SHA_CTX *dst, const git_SHA_CTX *src)
81129
memcpy(dst, src, sizeof(*dst));
82130
}
83131
#endif
132+
#ifndef SHA1_NEEDS_CLONE_HELPER_FAST
133+
static inline void git_SHA1_Clone_fast(git_SHA_CTX_fast *dst,
134+
const git_SHA_CTX_fast *src)
135+
{
136+
memcpy(dst, src, sizeof(*dst));
137+
}
138+
#endif
84139

85140
#ifndef SHA256_NEEDS_CLONE_HELPER
86141
static inline void git_SHA256_Clone(git_SHA256_CTX *dst, const git_SHA256_CTX *src)
@@ -178,6 +233,8 @@ enum get_oid_result {
178233
/* A suitably aligned type for stack allocations of hash contexts. */
179234
union git_hash_ctx {
180235
git_SHA_CTX sha1;
236+
git_SHA_CTX_fast sha1_fast;
237+
181238
git_SHA256_CTX sha256;
182239
};
183240
typedef union git_hash_ctx git_hash_ctx;
@@ -222,6 +279,21 @@ struct git_hash_algo {
222279
/* The hash finalization function for object IDs. */
223280
git_hash_final_oid_fn final_oid_fn;
224281

282+
/* The fast / non-cryptographic hash initialization function. */
283+
git_hash_init_fn fast_init_fn;
284+
285+
/* The fast / non-cryptographic hash context cloning function. */
286+
git_hash_clone_fn fast_clone_fn;
287+
288+
/* The fast / non-cryptographic hash update function. */
289+
git_hash_update_fn fast_update_fn;
290+
291+
/* The fast / non-cryptographic hash finalization function. */
292+
git_hash_final_fn fast_final_fn;
293+
294+
/* The fast / non-cryptographic hash finalization function. */
295+
git_hash_final_oid_fn fast_final_oid_fn;
296+
225297
/* The OID of the empty tree. */
226298
const struct object_id *empty_tree;
227299

‎object-file.c‎

Lines changed: 106 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,33 @@ static void git_hash_sha1_final_oid(struct object_id *oid, git_hash_ctx *ctx)
115115
oid->algo = GIT_HASH_SHA1;
116116
}
117117

118+
static void git_hash_sha1_init_fast(git_hash_ctx *ctx)
119+
{
120+
git_SHA1_Init_fast(&ctx->sha1_fast);
121+
}
122+
123+
static void git_hash_sha1_clone_fast(git_hash_ctx *dst, const git_hash_ctx *src)
124+
{
125+
git_SHA1_Clone_fast(&dst->sha1_fast, &src->sha1_fast);
126+
}
127+
128+
static void git_hash_sha1_update_fast(git_hash_ctx *ctx, const void *data,
129+
size_t len)
130+
{
131+
git_SHA1_Update_fast(&ctx->sha1_fast, data, len);
132+
}
133+
134+
static void git_hash_sha1_final_fast(unsigned char *hash, git_hash_ctx *ctx)
135+
{
136+
git_SHA1_Final_fast(hash, &ctx->sha1_fast);
137+
}
138+
139+
static void git_hash_sha1_final_oid_fast(struct object_id *oid, git_hash_ctx *ctx)
140+
{
141+
git_SHA1_Final_fast(oid->hash, &ctx->sha1_fast);
142+
memset(oid->hash + GIT_SHA1_RAWSZ, 0, GIT_MAX_RAWSZ - GIT_SHA1_RAWSZ);
143+
oid->algo = GIT_HASH_SHA1;
144+
}
118145

119146
static void git_hash_sha256_init(git_hash_ctx *ctx)
120147
{
@@ -189,6 +216,11 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = {
189216
.update_fn = git_hash_unknown_update,
190217
.final_fn = git_hash_unknown_final,
191218
.final_oid_fn = git_hash_unknown_final_oid,
219+
.fast_init_fn = git_hash_unknown_init,
220+
.fast_clone_fn = git_hash_unknown_clone,
221+
.fast_update_fn = git_hash_unknown_update,
222+
.fast_final_fn = git_hash_unknown_final,
223+
.fast_final_oid_fn = git_hash_unknown_final_oid,
192224
.empty_tree = NULL,
193225
.empty_blob = NULL,
194226
.null_oid = NULL,
@@ -204,6 +236,11 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = {
204236
.update_fn = git_hash_sha1_update,
205237
.final_fn = git_hash_sha1_final,
206238
.final_oid_fn = git_hash_sha1_final_oid,
239+
.fast_init_fn = git_hash_sha1_init_fast,
240+
.fast_clone_fn = git_hash_sha1_clone_fast,
241+
.fast_update_fn = git_hash_sha1_update_fast,
242+
.fast_final_fn = git_hash_sha1_final_fast,
243+
.fast_final_oid_fn = git_hash_sha1_final_oid_fast,
207244
.empty_tree = &empty_tree_oid,
208245
.empty_blob = &empty_blob_oid,
209246
.null_oid = &null_oid_sha1,
@@ -219,6 +256,11 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = {
219256
.update_fn = git_hash_sha256_update,
220257
.final_fn = git_hash_sha256_final,
221258
.final_oid_fn = git_hash_sha256_final_oid,
259+
.fast_init_fn = git_hash_sha256_init,
260+
.fast_clone_fn = git_hash_sha256_clone,
261+
.fast_update_fn = git_hash_sha256_update,
262+
.fast_final_fn = git_hash_sha256_final,
263+
.fast_final_oid_fn = git_hash_sha256_final_oid,
222264
.empty_tree = &empty_tree_oid_sha256,
223265
.empty_blob = &empty_blob_oid_sha256,
224266
.null_oid = &null_oid_sha256,
@@ -1944,17 +1986,71 @@ static void write_object_file_prepare_literally(const struct git_hash_algo *algo
19441986
hash_object_body(algo, &c, buf, len, oid, hdr, hdrlen);
19451987
}
19461988

1989+
static int check_collision(const char *filename_a, const char *filename_b)
1990+
{
1991+
char buf_a[4096], buf_b[4096];
1992+
int fd_a = -1, fd_b = -1;
1993+
int ret = 0;
1994+
1995+
fd_a = open(filename_a, O_RDONLY);
1996+
if (fd_a < 0) {
1997+
ret = error_errno(_("unable to open %s"), filename_a);
1998+
goto out;
1999+
}
2000+
2001+
fd_b = open(filename_b, O_RDONLY);
2002+
if (fd_b < 0) {
2003+
ret = error_errno(_("unable to open %s"), filename_b);
2004+
goto out;
2005+
}
2006+
2007+
while (1) {
2008+
ssize_t sz_a, sz_b;
2009+
2010+
sz_a = read_in_full(fd_a, buf_a, sizeof(buf_a));
2011+
if (sz_a < 0) {
2012+
ret = error_errno(_("unable to read %s"), filename_a);
2013+
goto out;
2014+
}
2015+
2016+
sz_b = read_in_full(fd_b, buf_b, sizeof(buf_b));
2017+
if (sz_b < 0) {
2018+
ret = error_errno(_("unable to read %s"), filename_b);
2019+
goto out;
2020+
}
2021+
2022+
if (sz_a != sz_b || memcmp(buf_a, buf_b, sz_a)) {
2023+
ret = error(_("files '%s' and '%s' differ in contents"),
2024+
filename_a, filename_b);
2025+
goto out;
2026+
}
2027+
2028+
if (sz_a < sizeof(buf_a))
2029+
break;
2030+
}
2031+
2032+
out:
2033+
if (fd_a > -1)
2034+
close(fd_a);
2035+
if (fd_b > -1)
2036+
close(fd_b);
2037+
return ret;
2038+
}
2039+
19472040
/*
19482041
* Move the just written object into its final resting place.
19492042
*/
19502043
int finalize_object_file(const char *tmpfile, const char *filename)
19512044
{
2045+
struct stat st;
19522046
int ret = 0;
19532047

19542048
if (object_creation_mode == OBJECT_CREATION_USES_RENAMES)
19552049
goto try_rename;
19562050
else if (link(tmpfile, filename))
19572051
ret = errno;
2052+
else
2053+
unlink_or_warn(tmpfile);
19582054

19592055
/*
19602056
* Coda hack - coda doesn't like cross-directory links,
@@ -1969,16 +2065,22 @@ int finalize_object_file(const char *tmpfile, const char *filename)
19692065
*/
19702066
if (ret && ret != EEXIST) {
19712067
try_rename:
1972-
if (!rename(tmpfile, filename))
2068+
if (!stat(filename, &st))
2069+
ret = EEXIST;
2070+
else if (!rename(tmpfile, filename))
19732071
goto out;
1974-
ret = errno;
2072+
else
2073+
ret = errno;
19752074
}
1976-
unlink_or_warn(tmpfile);
19772075
if (ret) {
19782076
if (ret != EEXIST) {
2077+
int saved_errno = errno;
2078+
unlink_or_warn(tmpfile);
2079+
errno = saved_errno;
19792080
return error_errno(_("unable to write file %s"), filename);
19802081
}
1981-
/* FIXME!!! Collision check here ? */
2082+
if (check_collision(tmpfile, filename))
2083+
return -1;
19822084
}
19832085

19842086
out:

0 commit comments

Comments
 (0)