Skip to content

Commit 0faafdf

Browse files
zhuxiaolong37huiguangjun
zhuxiaolong37
authored and
huiguangjun
committed
crc64 optimize (#469)
1 parent 62fc2ed commit 0faafdf

File tree

2 files changed

+161
-22
lines changed

2 files changed

+161
-22
lines changed

‎src/main/java/com/aliyun/oss/common/utils/CRC64.java‎

Lines changed: 59 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -31,24 +31,45 @@ public class CRC64 implements Checksum {
3131
private final static long POLY = (long) 0xc96c5795d7870f42L; // ECMA-182
3232

3333
/* CRC64 calculation table. */
34-
private final static long[] table;
34+
private final static long[][] table;
3535

3636
/* Current CRC value. */
3737
private long value;
3838

39-
static {
40-
table = new long[256];
39+
static
40+
{
41+
/*
42+
* Nested tables as described by Mark Adler:
43+
* http://stackoverflow.com/a/20579405/58962
44+
*/
45+
table = new long[8][256];
4146

42-
for (int n = 0; n < 256; n++) {
47+
for (int n = 0; n < 256; n++)
48+
{
4349
long crc = n;
44-
for (int k = 0; k < 8; k++) {
45-
if ((crc & 1) == 1) {
50+
for (int k = 0; k < 8; k++)
51+
{
52+
if ((crc & 1) == 1)
53+
{
4654
crc = (crc >>> 1) ^ POLY;
47-
} else {
55+
}
56+
else
57+
{
4858
crc = (crc >>> 1);
4959
}
5060
}
51-
table[n] = crc;
61+
table[0][n] = crc;
62+
}
63+
64+
/* generate nested CRC table for future slice-by-8 lookup */
65+
for (int n = 0; n < 256; n++)
66+
{
67+
long crc = table[0][n];
68+
for (int k = 1; k < 8; k++)
69+
{
70+
crc = table[0][(int) (crc & 0xff)] ^ (crc >>> 8);
71+
table[k][n] = crc;
72+
}
5273
}
5374
}
5475

@@ -110,14 +131,7 @@ public long getValue() {
110131
*
111132
**/
112133
public void update(byte[] b, int len) {
113-
int idx = 0;
114-
this.value = ~this.value;
115-
while (len > 0) {
116-
this.value = table[((int) (this.value ^ b[idx])) & 0xff] ^ (this.value >>> 8);
117-
idx++;
118-
len--;
119-
}
120-
this.value = ~this.value;
134+
this.update(b, 0, len);
121135
}
122136

123137
/**
@@ -126,21 +140,44 @@ public void update(byte[] b, int len) {
126140
* the byte.
127141
**/
128142
public void update(byte b) {
129-
this.value = ~this.value;
130-
this.value = table[((int) (this.value ^ b)) & 0xff] ^ (this.value >>> 8);
131-
this.value = ~this.value;
143+
this.update(new byte[]{b}, 0, 1);
132144
}
133145

134146
@Override
135147
public void update(int b) {
136-
update((byte) (b & 0xFF));
148+
this.update(new byte[]{(byte)b}, 0, 1);
137149
}
138150

139151
@Override
140152
public void update(byte[] b, int off, int len) {
141-
for (int i = off; len > 0; len--) {
142-
update(b[i++]);
153+
this.value = ~this.value;
154+
155+
/* fast middle processing, 8 bytes (aligned!) per loop */
156+
157+
int idx = off;
158+
while (len >= 8)
159+
{
160+
value = table[7][(int) (value & 0xff ^ (b[idx] & 0xff))]
161+
^ table[6][(int) ((value >>> 8) & 0xff ^ (b[idx + 1] & 0xff))]
162+
^ table[5][(int) ((value >>> 16) & 0xff ^ (b[idx + 2] & 0xff))]
163+
^ table[4][(int) ((value >>> 24) & 0xff ^ (b[idx + 3] & 0xff))]
164+
^ table[3][(int) ((value >>> 32) & 0xff ^ (b[idx + 4] & 0xff))]
165+
^ table[2][(int) ((value >>> 40) & 0xff ^ (b[idx + 5] & 0xff))]
166+
^ table[1][(int) ((value >>> 48) & 0xff ^ (b[idx + 6] & 0xff))]
167+
^ table[0][(int) ((value >>> 56) ^ b[idx + 7] & 0xff)];
168+
idx += 8;
169+
len -= 8;
170+
}
171+
172+
/* process remaining bytes (can't be larger than 8) */
173+
while (len > 0)
174+
{
175+
value = table[0][(int) ((this.value ^ b[idx]) & 0xff)] ^ (this.value >>> 8);
176+
idx++;
177+
len--;
143178
}
179+
180+
this.value = ~this.value;
144181
}
145182

146183
@Override

‎src/test/java/com/aliyun/oss/common/utils/CRC64UtilTest.java‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,12 @@
2020
package com.aliyun.oss.common.utils;
2121

2222

23+
import org.junit.Assert;
2324
import org.junit.Test;
2425

26+
import java.util.Arrays;
27+
import java.util.Random;
28+
2529
import static org.junit.Assert.assertEquals;
2630
import static org.junit.Assert.assertTrue;
2731

@@ -72,4 +76,102 @@ public void testCRC64() {
7276

7377
assertTrue(crc4.getBytes().length > 0);
7478
}
79+
80+
@Test
81+
public void testBytes()
82+
{
83+
84+
final byte[] TEST1 = "123456789".getBytes();
85+
final int TESTLEN1 = 9;
86+
final long TESTCRC1 = 0x995dc9bbdf1939faL; // ECMA.
87+
calcAndCheck(TEST1, TESTLEN1, TESTCRC1);
88+
89+
final byte[] TEST2 = "This is a test of the emergency broadcast system.".getBytes();
90+
final int TESTLEN2 = 49;
91+
final long TESTCRC2 = 0x27db187fc15bbc72L; // ECMA.
92+
calcAndCheck(TEST2, TESTLEN2, TESTCRC2);
93+
94+
final byte[] TEST3 = "IHATEMATH".getBytes();
95+
final int TESTLEN3 = 9;
96+
final long TESTCRC3 = 0x3920e0f66b6ee0c8L; // ECMA.
97+
calcAndCheck(TEST3, TESTLEN3, TESTCRC3);
98+
}
99+
100+
101+
@Test
102+
public void testPerformance()
103+
{
104+
byte[] b = new byte[65536];
105+
new Random().nextBytes(b);
106+
107+
// warmup
108+
CRC64 crc = new CRC64();
109+
crc.update(b, b.length);
110+
111+
// start bench
112+
long bytes = 0;
113+
long start = System.currentTimeMillis();
114+
crc = new CRC64();
115+
for (int i = 0; i < 100000; i++)
116+
{
117+
crc.update(b, b.length);
118+
bytes += b.length;
119+
}
120+
121+
long duration = System.currentTimeMillis() - start;
122+
duration = (duration == 0) ? 1 : duration; // div0
123+
long bytesPerSec = (bytes / duration) * 1000;
124+
125+
System.out.println(bytes / 1024 / 1024 + " MB processed in " + duration + " ms @ " + bytesPerSec / 1024 / 1024
126+
+ " MB/s");
127+
}
128+
129+
@Test
130+
public void testUpdateAndReset()
131+
{
132+
CRC64 crc = new CRC64();
133+
134+
final byte[] TEST1 = "123456789".getBytes();
135+
final int TESTLEN1 = 9;
136+
final long TESTCRC1 = 0x995dc9bbdf1939faL; // ECMA.
137+
138+
crc.update(TEST1, TESTLEN1);
139+
140+
Assert.assertEquals("oh noes", TESTCRC1, crc.getValue());
141+
142+
crc.reset();
143+
144+
Assert.assertEquals("oh noes", 0, crc.getValue());
145+
146+
final byte[] TEST2 = "This is a test of the emergency broadcast system.".getBytes();
147+
final int TESTLEN2 = 49;
148+
final long TESTCRC2 = 0x27db187fc15bbc72L; // ECMA.
149+
150+
crc.update(TEST2, TESTLEN2);
151+
152+
Assert.assertEquals("oh noes", TESTCRC2, crc.getValue());
153+
}
154+
155+
private void calcAndCheck(byte[] b, int len, long crcValue)
156+
{
157+
158+
/* Test CRC64 default calculation. */
159+
CRC64 crc = new CRC64(b, len);
160+
if (crc.getValue() != crcValue)
161+
{
162+
throw new RuntimeException("mismatch: " + String.format("%016x", crc.getValue()) + " should be "
163+
+ String.format("%016x", crcValue));
164+
}
165+
166+
/* test combine() */
167+
CRC64 crc1 = new CRC64(b, (len + 1) >>> 1);
168+
CRC64 crc2 = new CRC64(Arrays.copyOfRange(b, (len + 1) >>> 1, b.length), len >>> 1);
169+
crc = CRC64.combine(crc1, crc2, len >>> 1);
170+
171+
if (crc.getValue() != crcValue)
172+
{
173+
throw new RuntimeException("mismatch: " + String.format("%016x", crc.getValue()) + " should be "
174+
+ String.format("%016x", crcValue));
175+
}
176+
}
75177
}

0 commit comments

Comments
 (0)