Skip to content

Commit 1b6a18b

Browse files
AnemptyshipAnemptyship
committed
Fix MD5 hash mismatch on big-endian systems (s390x)
- Make md5_trans() endian-safe by explicitly decoding bytes to u32 words - Use compile-time check to preserve zero-copy performance on little-endian - Fix md5_finish() to write length and digest in little-endian byte order - Add md5_update_16() for safe 16-bit pixel data processing
1 parent 192317a commit 1b6a18b

File tree

1 file changed

+133
-118
lines changed

1 file changed

+133
-118
lines changed

‎src/oapv_util.c‎

Lines changed: 133 additions & 118 deletions
Original file line numberDiff line numberDiff line change
@@ -40,97 +40,110 @@
4040
#define HH(x, y, z) (x ^ y ^ z)
4141
#define II(x, y, z) (y ^ (x | ~z))
4242

43-
static void md5_trans(u32 *buf, u32 *msg)
43+
static void md5_trans(u32 *buf, const u8 *msg)
4444
{
4545
register u32 a, b, c, d;
4646

47+
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
48+
const u32 *blk = (const u32 *)msg;
49+
#else
50+
u32 x[16];
51+
int i;
52+
53+
for (i = 0; i < 16; i++) {
54+
x[i] = ((u32)msg[i*4+0]) | (((u32)msg[i*4+1]) << 8) |
55+
(((u32)msg[i*4+2]) << 16) | (((u32)msg[i*4+3]) << 24);
56+
}
57+
const u32 *blk = x;
58+
#endif
59+
4760
a = buf[0];
4861
b = buf[1];
4962
c = buf[2];
5063
d = buf[3];
5164

52-
MD5FUNC(FF, a, b, c, d, msg[0], 7, 0xd76aa478); /* 1 */
53-
MD5FUNC(FF, d, a, b, c, msg[1], 12, 0xe8c7b756); /* 2 */
54-
MD5FUNC(FF, c, d, a, b, msg[2], 17, 0x242070db); /* 3 */
55-
MD5FUNC(FF, b, c, d, a, msg[3], 22, 0xc1bdceee); /* 4 */
65+
MD5FUNC(FF, a, b, c, d, blk[0], 7, 0xd76aa478); /* 1 */
66+
MD5FUNC(FF, d, a, b, c, blk[1], 12, 0xe8c7b756); /* 2 */
67+
MD5FUNC(FF, c, d, a, b, blk[2], 17, 0x242070db); /* 3 */
68+
MD5FUNC(FF, b, c, d, a, blk[3], 22, 0xc1bdceee); /* 4 */
5669

57-
MD5FUNC(FF, a, b, c, d, msg[4], 7, 0xf57c0faf); /* 5 */
58-
MD5FUNC(FF, d, a, b, c, msg[5], 12, 0x4787c62a); /* 6 */
59-
MD5FUNC(FF, c, d, a, b, msg[6], 17, 0xa8304613); /* 7 */
60-
MD5FUNC(FF, b, c, d, a, msg[7], 22, 0xfd469501); /* 8 */
70+
MD5FUNC(FF, a, b, c, d, blk[4], 7, 0xf57c0faf); /* 5 */
71+
MD5FUNC(FF, d, a, b, c, blk[5], 12, 0x4787c62a); /* 6 */
72+
MD5FUNC(FF, c, d, a, b, blk[6], 17, 0xa8304613); /* 7 */
73+
MD5FUNC(FF, b, c, d, a, blk[7], 22, 0xfd469501); /* 8 */
6174

62-
MD5FUNC(FF, a, b, c, d, msg[8], 7, 0x698098d8); /* 9 */
63-
MD5FUNC(FF, d, a, b, c, msg[9], 12, 0x8b44f7af); /* 10 */
64-
MD5FUNC(FF, c, d, a, b, msg[10], 17, 0xffff5bb1); /* 11 */
65-
MD5FUNC(FF, b, c, d, a, msg[11], 22, 0x895cd7be); /* 12 */
75+
MD5FUNC(FF, a, b, c, d, blk[8], 7, 0x698098d8); /* 9 */
76+
MD5FUNC(FF, d, a, b, c, blk[9], 12, 0x8b44f7af); /* 10 */
77+
MD5FUNC(FF, c, d, a, b, blk[10], 17, 0xffff5bb1); /* 11 */
78+
MD5FUNC(FF, b, c, d, a, blk[11], 22, 0x895cd7be); /* 12 */
6679

67-
MD5FUNC(FF, a, b, c, d, msg[12], 7, 0x6b901122); /* 13 */
68-
MD5FUNC(FF, d, a, b, c, msg[13], 12, 0xfd987193); /* 14 */
69-
MD5FUNC(FF, c, d, a, b, msg[14], 17, 0xa679438e); /* 15 */
70-
MD5FUNC(FF, b, c, d, a, msg[15], 22, 0x49b40821); /* 16 */
80+
MD5FUNC(FF, a, b, c, d, blk[12], 7, 0x6b901122); /* 13 */
81+
MD5FUNC(FF, d, a, b, c, blk[13], 12, 0xfd987193); /* 14 */
82+
MD5FUNC(FF, c, d, a, b, blk[14], 17, 0xa679438e); /* 15 */
83+
MD5FUNC(FF, b, c, d, a, blk[15], 22, 0x49b40821); /* 16 */
7184

7285
/* Round 2 */
73-
MD5FUNC(GG, a, b, c, d, msg[1], 5, 0xf61e2562); /* 17 */
74-
MD5FUNC(GG, d, a, b, c, msg[6], 9, 0xc040b340); /* 18 */
75-
MD5FUNC(GG, c, d, a, b, msg[11], 14, 0x265e5a51); /* 19 */
76-
MD5FUNC(GG, b, c, d, a, msg[0], 20, 0xe9b6c7aa); /* 20 */
77-
78-
MD5FUNC(GG, a, b, c, d, msg[5], 5, 0xd62f105d); /* 21 */
79-
MD5FUNC(GG, d, a, b, c, msg[10], 9, 0x2441453); /* 22 */
80-
MD5FUNC(GG, c, d, a, b, msg[15], 14, 0xd8a1e681); /* 23 */
81-
MD5FUNC(GG, b, c, d, a, msg[4], 20, 0xe7d3fbc8); /* 24 */
82-
83-
MD5FUNC(GG, a, b, c, d, msg[9], 5, 0x21e1cde6); /* 25 */
84-
MD5FUNC(GG, d, a, b, c, msg[14], 9, 0xc33707d6); /* 26 */
85-
MD5FUNC(GG, c, d, a, b, msg[3], 14, 0xf4d50d87); /* 27 */
86-
MD5FUNC(GG, b, c, d, a, msg[8], 20, 0x455a14ed); /* 28 */
87-
88-
MD5FUNC(GG, a, b, c, d, msg[13], 5, 0xa9e3e905); /* 29 */
89-
MD5FUNC(GG, d, a, b, c, msg[2], 9, 0xfcefa3f8); /* 30 */
90-
MD5FUNC(GG, c, d, a, b, msg[7], 14, 0x676f02d9); /* 31 */
91-
MD5FUNC(GG, b, c, d, a, msg[12], 20, 0x8d2a4c8a); /* 32 */
86+
MD5FUNC(GG, a, b, c, d, blk[1], 5, 0xf61e2562); /* 17 */
87+
MD5FUNC(GG, d, a, b, c, blk[6], 9, 0xc040b340); /* 18 */
88+
MD5FUNC(GG, c, d, a, b, blk[11], 14, 0x265e5a51); /* 19 */
89+
MD5FUNC(GG, b, c, d, a, blk[0], 20, 0xe9b6c7aa); /* 20 */
90+
91+
MD5FUNC(GG, a, b, c, d, blk[5], 5, 0xd62f105d); /* 21 */
92+
MD5FUNC(GG, d, a, b, c, blk[10], 9, 0x2441453); /* 22 */
93+
MD5FUNC(GG, c, d, a, b, blk[15], 14, 0xd8a1e681); /* 23 */
94+
MD5FUNC(GG, b, c, d, a, blk[4], 20, 0xe7d3fbc8); /* 24 */
95+
96+
MD5FUNC(GG, a, b, c, d, blk[9], 5, 0x21e1cde6); /* 25 */
97+
MD5FUNC(GG, d, a, b, c, blk[14], 9, 0xc33707d6); /* 26 */
98+
MD5FUNC(GG, c, d, a, b, blk[3], 14, 0xf4d50d87); /* 27 */
99+
MD5FUNC(GG, b, c, d, a, blk[8], 20, 0x455a14ed); /* 28 */
100+
101+
MD5FUNC(GG, a, b, c, d, blk[13], 5, 0xa9e3e905); /* 29 */
102+
MD5FUNC(GG, d, a, b, c, blk[2], 9, 0xfcefa3f8); /* 30 */
103+
MD5FUNC(GG, c, d, a, b, blk[7], 14, 0x676f02d9); /* 31 */
104+
MD5FUNC(GG, b, c, d, a, blk[12], 20, 0x8d2a4c8a); /* 32 */
92105

93106
/* Round 3 */
94-
MD5FUNC(HH, a, b, c, d, msg[5], 4, 0xfffa3942); /* 33 */
95-
MD5FUNC(HH, d, a, b, c, msg[8], 11, 0x8771f681); /* 34 */
96-
MD5FUNC(HH, c, d, a, b, msg[11], 16, 0x6d9d6122); /* 35 */
97-
MD5FUNC(HH, b, c, d, a, msg[14], 23, 0xfde5380c); /* 36 */
98-
99-
MD5FUNC(HH, a, b, c, d, msg[1], 4, 0xa4beea44); /* 37 */
100-
MD5FUNC(HH, d, a, b, c, msg[4], 11, 0x4bdecfa9); /* 38 */
101-
MD5FUNC(HH, c, d, a, b, msg[7], 16, 0xf6bb4b60); /* 39 */
102-
MD5FUNC(HH, b, c, d, a, msg[10], 23, 0xbebfbc70); /* 40 */
103-
104-
MD5FUNC(HH, a, b, c, d, msg[13], 4, 0x289b7ec6); /* 41 */
105-
MD5FUNC(HH, d, a, b, c, msg[0], 11, 0xeaa127fa); /* 42 */
106-
MD5FUNC(HH, c, d, a, b, msg[3], 16, 0xd4ef3085); /* 43 */
107-
MD5FUNC(HH, b, c, d, a, msg[6], 23, 0x4881d05); /* 44 */
108-
109-
MD5FUNC(HH, a, b, c, d, msg[9], 4, 0xd9d4d039); /* 45 */
110-
MD5FUNC(HH, d, a, b, c, msg[12], 11, 0xe6db99e5); /* 46 */
111-
MD5FUNC(HH, c, d, a, b, msg[15], 16, 0x1fa27cf8); /* 47 */
112-
MD5FUNC(HH, b, c, d, a, msg[2], 23, 0xc4ac5665); /* 48 */
107+
MD5FUNC(HH, a, b, c, d, blk[5], 4, 0xfffa3942); /* 33 */
108+
MD5FUNC(HH, d, a, b, c, blk[8], 11, 0x8771f681); /* 34 */
109+
MD5FUNC(HH, c, d, a, b, blk[11], 16, 0x6d9d6122); /* 35 */
110+
MD5FUNC(HH, b, c, d, a, blk[14], 23, 0xfde5380c); /* 36 */
111+
112+
MD5FUNC(HH, a, b, c, d, blk[1], 4, 0xa4beea44); /* 37 */
113+
MD5FUNC(HH, d, a, b, c, blk[4], 11, 0x4bdecfa9); /* 38 */
114+
MD5FUNC(HH, c, d, a, b, blk[7], 16, 0xf6bb4b60); /* 39 */
115+
MD5FUNC(HH, b, c, d, a, blk[10], 23, 0xbebfbc70); /* 40 */
116+
117+
MD5FUNC(HH, a, b, c, d, blk[13], 4, 0x289b7ec6); /* 41 */
118+
MD5FUNC(HH, d, a, b, c, blk[0], 11, 0xeaa127fa); /* 42 */
119+
MD5FUNC(HH, c, d, a, b, blk[3], 16, 0xd4ef3085); /* 43 */
120+
MD5FUNC(HH, b, c, d, a, blk[6], 23, 0x4881d05); /* 44 */
121+
122+
MD5FUNC(HH, a, b, c, d, blk[9], 4, 0xd9d4d039); /* 45 */
123+
MD5FUNC(HH, d, a, b, c, blk[12], 11, 0xe6db99e5); /* 46 */
124+
MD5FUNC(HH, c, d, a, b, blk[15], 16, 0x1fa27cf8); /* 47 */
125+
MD5FUNC(HH, b, c, d, a, blk[2], 23, 0xc4ac5665); /* 48 */
113126

114127
/* Round 4 */
115-
MD5FUNC(II, a, b, c, d, msg[0], 6, 0xf4292244); /* 49 */
116-
MD5FUNC(II, d, a, b, c, msg[7], 10, 0x432aff97); /* 50 */
117-
MD5FUNC(II, c, d, a, b, msg[14], 15, 0xab9423a7); /* 51 */
118-
MD5FUNC(II, b, c, d, a, msg[5], 21, 0xfc93a039); /* 52 */
119-
120-
MD5FUNC(II, a, b, c, d, msg[12], 6, 0x655b59c3); /* 53 */
121-
MD5FUNC(II, d, a, b, c, msg[3], 10, 0x8f0ccc92); /* 54 */
122-
MD5FUNC(II, c, d, a, b, msg[10], 15, 0xffeff47d); /* 55 */
123-
MD5FUNC(II, b, c, d, a, msg[1], 21, 0x85845dd1); /* 56 */
124-
125-
MD5FUNC(II, a, b, c, d, msg[8], 6, 0x6fa87e4f); /* 57 */
126-
MD5FUNC(II, d, a, b, c, msg[15], 10, 0xfe2ce6e0); /* 58 */
127-
MD5FUNC(II, c, d, a, b, msg[6], 15, 0xa3014314); /* 59 */
128-
MD5FUNC(II, b, c, d, a, msg[13], 21, 0x4e0811a1); /* 60 */
129-
130-
MD5FUNC(II, a, b, c, d, msg[4], 6, 0xf7537e82); /* 61 */
131-
MD5FUNC(II, d, a, b, c, msg[11], 10, 0xbd3af235); /* 62 */
132-
MD5FUNC(II, c, d, a, b, msg[2], 15, 0x2ad7d2bb); /* 63 */
133-
MD5FUNC(II, b, c, d, a, msg[9], 21, 0xeb86d391); /* 64 */
128+
MD5FUNC(II, a, b, c, d, blk[0], 6, 0xf4292244); /* 49 */
129+
MD5FUNC(II, d, a, b, c, blk[7], 10, 0x432aff97); /* 50 */
130+
MD5FUNC(II, c, d, a, b, blk[14], 15, 0xab9423a7); /* 51 */
131+
MD5FUNC(II, b, c, d, a, blk[5], 21, 0xfc93a039); /* 52 */
132+
133+
MD5FUNC(II, a, b, c, d, blk[12], 6, 0x655b59c3); /* 53 */
134+
MD5FUNC(II, d, a, b, c, blk[3], 10, 0x8f0ccc92); /* 54 */
135+
MD5FUNC(II, c, d, a, b, blk[10], 15, 0xffeff47d); /* 55 */
136+
MD5FUNC(II, b, c, d, a, blk[1], 21, 0x85845dd1); /* 56 */
137+
138+
MD5FUNC(II, a, b, c, d, blk[8], 6, 0x6fa87e4f); /* 57 */
139+
MD5FUNC(II, d, a, b, c, blk[15], 10, 0xfe2ce6e0); /* 58 */
140+
MD5FUNC(II, c, d, a, b, blk[6], 15, 0xa3014314); /* 59 */
141+
MD5FUNC(II, b, c, d, a, blk[13], 21, 0x4e0811a1); /* 60 */
142+
143+
MD5FUNC(II, a, b, c, d, blk[4], 6, 0xf7537e82); /* 61 */
144+
MD5FUNC(II, d, a, b, c, blk[11], 10, 0xbd3af235); /* 62 */
145+
MD5FUNC(II, c, d, a, b, blk[2], 15, 0x2ad7d2bb); /* 63 */
146+
MD5FUNC(II, b, c, d, a, blk[9], 21, 0xeb86d391); /* 64 */
134147

135148
buf[0] += a;
136149
buf[1] += b;
@@ -168,10 +181,10 @@ static void md5_update(oapv_md5_t *md5, void *buf_t, u32 len)
168181

169182
if(len >= part_len) {
170183
oapv_mcpy(md5->msg + idx, buf, part_len);
171-
md5_trans(md5->h, (u32 *)md5->msg);
184+
md5_trans(md5->h, md5->msg);
172185

173186
for(i = part_len; i + 63 < len; i += 64) {
174-
md5_trans(md5->h, (u32 *)(buf + i));
187+
md5_trans(md5->h, buf + i);
175188
}
176189
idx = 0;
177190
}
@@ -186,43 +199,23 @@ static void md5_update(oapv_md5_t *md5, void *buf_t, u32 len)
186199

187200
static void md5_update_16(oapv_md5_t *md5, void *buf_t, u32 len)
188201
{
189-
u16 *buf;
190-
u32 i, idx, part_len, j;
191-
u8 t[512];
192-
193-
buf = (u16 *)buf_t;
194-
idx = (u32)((md5->bits[0] >> 3) & 0x3f);
195-
196-
len = len * 2;
197-
for(j = 0; j < len; j += 2) {
198-
t[j] = (u8)(*(buf));
199-
t[j + 1] = *(buf) >> 8;
200-
buf++;
201-
}
202-
203-
md5->bits[0] += (len << 3);
204-
if(md5->bits[0] < (len << 3)) {
205-
(md5->bits[1])++;
206-
}
207-
208-
md5->bits[1] += (len >> 29);
209-
part_len = 64 - idx;
210-
211-
if(len >= part_len) {
212-
oapv_mcpy(md5->msg + idx, t, part_len);
213-
md5_trans(md5->h, (u32 *)md5->msg);
214-
215-
for(i = part_len; i + 63 < len; i += 64) {
216-
md5_trans(md5->h, (u32 *)(t + i));
202+
u16 *buf = (u16 *)buf_t;
203+
u8 t[1024];
204+
u32 i, j, chunk_len;
205+
206+
i = 0;
207+
while(i < len) {
208+
chunk_len = len - i;
209+
if(chunk_len > 512)
210+
chunk_len = 512;
211+
212+
for(j = 0; j < chunk_len; j++) {
213+
t[j * 2] = (u8)(buf[i + j]);
214+
t[j * 2 + 1] = (u8)(buf[i + j] >> 8);
217215
}
218-
idx = 0;
219-
}
220-
else {
221-
i = 0;
222-
}
223216

224-
if(len - i > 0) {
225-
oapv_mcpy(md5->msg + idx, t + i, len - i);
217+
md5_update(md5, t, chunk_len * 2);
218+
i += chunk_len;
226219
}
227220
}
228221

@@ -238,18 +231,33 @@ static void md5_finish(oapv_md5_t *md5, u8 digest[16])
238231

239232
if(cnt < 8) {
240233
oapv_mset(pos, 0, cnt);
241-
md5_trans(md5->h, (u32 *)md5->msg);
234+
md5_trans(md5->h, md5->msg);
242235
oapv_mset(md5->msg, 0, 56);
243236
}
244237
else {
245238
oapv_mset(pos, 0, cnt - 8);
246239
}
247240

248-
oapv_mcpy((md5->msg + 14 * sizeof(u32)), &md5->bits[0], sizeof(u32));
249-
oapv_mcpy((md5->msg + 15 * sizeof(u32)), &md5->bits[1], sizeof(u32));
250-
251-
md5_trans(md5->h, (u32 *)md5->msg);
252-
oapv_mcpy(digest, md5->h, 16);
241+
/* Append length in bits - Little Endian */
242+
md5->msg[56] = (u8)(md5->bits[0]);
243+
md5->msg[57] = (u8)(md5->bits[0] >> 8);
244+
md5->msg[58] = (u8)(md5->bits[0] >> 16);
245+
md5->msg[59] = (u8)(md5->bits[0] >> 24);
246+
md5->msg[60] = (u8)(md5->bits[1]);
247+
md5->msg[61] = (u8)(md5->bits[1] >> 8);
248+
md5->msg[62] = (u8)(md5->bits[1] >> 16);
249+
md5->msg[63] = (u8)(md5->bits[1] >> 24);
250+
251+
md5_trans(md5->h, md5->msg);
252+
253+
/* Store state in digest - Little Endian */
254+
for (int i=0; i<4; i++) {
255+
digest[i*4+0] = (u8)(md5->h[i]);
256+
digest[i*4+1] = (u8)(md5->h[i] >> 8);
257+
digest[i*4+2] = (u8)(md5->h[i] >> 16);
258+
digest[i*4+3] = (u8)(md5->h[i] >> 24);
259+
}
260+
253261
oapv_mset(md5, 0, sizeof(oapv_md5_t));
254262
}
255263

@@ -263,14 +271,21 @@ void oapv_imgb_set_md5(oapv_imgb_t *imgb)
263271

264272
oapv_md5_t md5[N_C];
265273
int i, j;
274+
int b_depth = OAPV_CS_GET_BYTE_DEPTH(imgb->cs);
275+
266276
oapv_assert(imgb != NULL);
267277
memset(imgb->hash, 0, sizeof(imgb->hash));
268278

269279
for(i = 0; i < imgb->np; i++) {
270280
md5_init(&md5[i]);
271281

272282
for(j = 0; j < imgb->ah[i]; j++) {
273-
md5_update(&md5[i], ((u8 *)imgb->a[i]) + j * imgb->s[i], imgb->aw[i] * 2);
283+
if(b_depth >= 2) {
284+
md5_update_16(&md5[i], ((u8 *)imgb->a[i]) + j * imgb->s[i], imgb->aw[i]);
285+
}
286+
else {
287+
md5_update(&md5[i], ((u8 *)imgb->a[i]) + j * imgb->s[i], imgb->aw[i]);
288+
}
274289
}
275290

276291
md5_finish(&md5[i], imgb->hash[i]);

0 commit comments

Comments
 (0)