Skip to content

Commit bf60ecd

Browse files
wendigoelectrum
wendigo
authored and
electrum
committed
Add native deflate support
1 parent 96f54fb commit bf60ecd

File tree

16 files changed

+651
-154
lines changed

16 files changed

+651
-154
lines changed

‎bin/download.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,3 +155,25 @@ download_macos \
155155

156156
download_macos \
157157
xxhash 0.8.3 libxxhash.0.8.3.dylib arm64 macos-aarch64/libxxhash.dylib
158+
159+
# libdeflate
160+
download_linux \
161+
"https://deb.debian.org/debian/pool/main/libd/libdeflate/libdeflate0_1.23-2_amd64.deb" \
162+
"/usr/lib/x86_64-linux-gnu/libdeflate.so.0" \
163+
"linux-amd64/libdeflate.so"
164+
165+
download_linux \
166+
"https://deb.debian.org/debian/pool/main/libd/libdeflate/libdeflate0_1.23-2_arm64.deb" \
167+
"/usr/lib/aarch64-linux-gnu/libdeflate.so.0" \
168+
"linux-aarch64/libdeflate.so"
169+
170+
download_linux \
171+
"https://deb.debian.org/debian/pool/main/libd/libdeflate/libdeflate0_1.23-2+b1_ppc64el.deb" \
172+
"/usr/lib/powerpc64le-linux-gnu/libdeflate.so.0" \
173+
"linux-ppc64le/libdeflate.so"
174+
175+
download_macos \
176+
libdeflate 1.23 libdeflate.0.dylib amd64 macos-amd64/libdeflate.dylib
177+
178+
download_macos \
179+
libdeflate 1.23 libdeflate.0.dylib arm64 macos-aarch64/libdeflate.dylib

‎src/main/java/io/airlift/compress/v3/deflate/DeflateCompressor.java‎

Lines changed: 10 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -15,84 +15,22 @@
1515

1616
import io.airlift.compress.v3.Compressor;
1717

18-
import java.lang.foreign.MemorySegment;
19-
import java.util.zip.Deflater;
20-
21-
import static java.lang.Math.toIntExact;
22-
import static java.util.Objects.requireNonNull;
23-
import static java.util.zip.Deflater.FULL_FLUSH;
24-
25-
public class DeflateCompressor
26-
implements Compressor
18+
public interface DeflateCompressor
19+
extends Compressor
2720
{
28-
private static final int EXTRA_COMPRESSION_SPACE = 16;
29-
private static final int COMPRESSION_LEVEL = 4;
30-
31-
private final int compressionLevel;
32-
33-
public DeflateCompressor()
34-
{
35-
this(COMPRESSION_LEVEL);
36-
}
37-
38-
public DeflateCompressor(int compressionLevel)
39-
{
40-
if (compressionLevel < 0 || compressionLevel > 9) {
41-
throw new IllegalArgumentException("Invalid compression level: %d (must be 0-9)".formatted(compressionLevel));
42-
}
43-
this.compressionLevel = compressionLevel;
44-
}
45-
46-
@Override
47-
public int maxCompressedLength(int uncompressedSize)
21+
static DeflateCompressor create()
4822
{
49-
// From Mark Adler's post http://stackoverflow.com/questions/1207877/java-size-of-compression-output-bytearray
50-
return uncompressedSize + ((uncompressedSize + 7) >> 3) + ((uncompressedSize + 63) >> 6) + 5 + EXTRA_COMPRESSION_SPACE;
51-
}
52-
53-
@Override
54-
public int compress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset, int maxOutputLength)
55-
{
56-
verifyRange(input, inputOffset, inputLength);
57-
verifyRange(output, outputOffset, maxOutputLength);
58-
59-
try (Deflater deflater = new Deflater(compressionLevel, true)) {
60-
deflater.setInput(input, inputOffset, inputLength);
61-
deflater.finish();
62-
63-
int compressedDataLength = deflater.deflate(output, outputOffset, maxOutputLength, FULL_FLUSH);
64-
if (!deflater.finished()) {
65-
throw new IllegalStateException("Output buffer too small");
66-
}
67-
return compressedDataLength;
68-
}
69-
}
70-
71-
@Override
72-
public int compress(MemorySegment input, MemorySegment output)
73-
{
74-
int maxCompressedLength = maxCompressedLength(toIntExact(input.byteSize()));
75-
if (output.byteSize() < maxCompressedLength) {
76-
throw new IllegalArgumentException("Output buffer must be at least " + maxCompressedLength + " bytes");
77-
}
78-
79-
try (Deflater deflater = new Deflater(compressionLevel, true)) {
80-
deflater.setInput(input.asByteBuffer());
81-
deflater.finish();
82-
83-
int compressedDataLength = deflater.deflate(output.asByteBuffer(), FULL_FLUSH);
84-
if (!deflater.finished()) {
85-
throw new IllegalStateException("maxCompressedLength formula is incorrect, because deflate produced more data");
86-
}
87-
return compressedDataLength;
23+
if (DeflateNativeCompressor.isEnabled()) {
24+
return new DeflateNativeCompressor();
8825
}
26+
return new DeflateJavaCompressor();
8927
}
9028

91-
private static void verifyRange(byte[] data, int offset, int length)
29+
static DeflateCompressor create(int compressionLevel)
9230
{
93-
requireNonNull(data, "data is null");
94-
if (offset < 0 || length < 0 || offset + length > data.length) {
95-
throw new IllegalArgumentException("Invalid offset or length (%s, %s) in array of length %s".formatted(offset, length, data.length));
31+
if (DeflateNativeCompressor.isEnabled()) {
32+
return new DeflateNativeCompressor(compressionLevel);
9633
}
34+
return new DeflateJavaCompressor(compressionLevel);
9735
}
9836
}

‎src/main/java/io/airlift/compress/v3/deflate/DeflateDecompressor.java‎

Lines changed: 6 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -14,87 +14,15 @@
1414
package io.airlift.compress.v3.deflate;
1515

1616
import io.airlift.compress.v3.Decompressor;
17-
import io.airlift.compress.v3.MalformedInputException;
1817

19-
import java.lang.foreign.MemorySegment;
20-
import java.nio.ByteBuffer;
21-
import java.util.zip.DataFormatException;
22-
import java.util.zip.Inflater;
23-
24-
import static java.lang.String.format;
25-
import static java.util.Objects.requireNonNull;
26-
27-
public class DeflateDecompressor
28-
implements Decompressor
18+
public interface DeflateDecompressor
19+
extends Decompressor
2920
{
30-
@Override
31-
public int decompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset, int maxOutputLength)
32-
throws MalformedInputException
33-
{
34-
verifyRange(input, inputOffset, inputLength);
35-
verifyRange(output, outputOffset, maxOutputLength);
36-
37-
try (Inflater inflater = new Inflater(true)) {
38-
inflater.setInput(input, inputOffset, inputLength);
39-
40-
int uncompressedLength = 0;
41-
while (true) {
42-
uncompressedLength += inflater.inflate(output, outputOffset + uncompressedLength, maxOutputLength - uncompressedLength);
43-
if (inflater.finished() || uncompressedLength >= maxOutputLength) {
44-
break;
45-
}
46-
if (inflater.needsInput()) {
47-
throw new MalformedInputException(0, format("Premature end of input stream. Input length = %s, uncompressed length = %d", inputLength, uncompressedLength));
48-
}
49-
}
50-
51-
if (!inflater.finished()) {
52-
throw new MalformedInputException(0, "Output buffer too small");
53-
}
54-
55-
return uncompressedLength;
56-
}
57-
catch (DataFormatException e) {
58-
throw new RuntimeException("Invalid compressed stream", e);
59-
}
60-
}
61-
62-
@Override
63-
public int decompress(MemorySegment input, MemorySegment output)
64-
throws MalformedInputException
65-
{
66-
ByteBuffer inputByteBuffer = input.asByteBuffer();
67-
try (Inflater inflater = new Inflater(true)) {
68-
inflater.setInput(inputByteBuffer);
69-
70-
ByteBuffer outputByteBuffer = output.asByteBuffer();
71-
int uncompressedLength = 0;
72-
while (true) {
73-
uncompressedLength += inflater.inflate(outputByteBuffer);
74-
if (inflater.finished() || uncompressedLength >= output.byteSize()) {
75-
break;
76-
}
77-
if (inflater.needsInput()) {
78-
throw new MalformedInputException(inputByteBuffer.position(), "Premature end of input stream. Input length = " + input.byteSize() + ", uncompressed length = " + uncompressedLength);
79-
}
80-
}
81-
82-
if (!inflater.finished()) {
83-
throw new MalformedInputException(inputByteBuffer.position(), "Could not decompress all input (output buffer too small?)");
84-
}
85-
86-
return uncompressedLength;
87-
}
88-
catch (DataFormatException e) {
89-
throw new RuntimeException("Invalid compressed stream", e);
90-
}
91-
}
92-
93-
private static void verifyRange(byte[] data, int offset, int length)
21+
static DeflateDecompressor create()
9422
{
95-
requireNonNull(data, "data is null");
96-
if (offset < 0 || length < 0 || offset + length > data.length) {
97-
throw new IllegalArgumentException(format("Invalid offset or length (%s, %s) in array of length %s", offset, length, data.length));
23+
if (DeflateNativeDecompressor.isEnabled()) {
24+
return new DeflateNativeDecompressor();
9825
}
26+
return new DeflateJavaDecompressor();
9927
}
10028
}

‎src/main/java/io/airlift/compress/v3/deflate/DeflateJavaCompressor.java‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.airlift.compress.v3.deflate;
15+
16+
import java.lang.foreign.MemorySegment;
17+
import java.util.zip.Deflater;
18+
19+
import static java.lang.Math.toIntExact;
20+
import static java.util.Objects.requireNonNull;
21+
import static java.util.zip.Deflater.FULL_FLUSH;
22+
23+
public class DeflateJavaCompressor
24+
implements DeflateCompressor
25+
{
26+
private static final int EXTRA_COMPRESSION_SPACE = 16;
27+
private static final int COMPRESSION_LEVEL = 4;
28+
29+
private final int compressionLevel;
30+
31+
public DeflateJavaCompressor()
32+
{
33+
this(COMPRESSION_LEVEL);
34+
}
35+
36+
public DeflateJavaCompressor(int compressionLevel)
37+
{
38+
if (compressionLevel < 0 || compressionLevel > 9) {
39+
throw new IllegalArgumentException("Invalid compression level: %d (must be 0-9)".formatted(compressionLevel));
40+
}
41+
this.compressionLevel = compressionLevel;
42+
}
43+
44+
@Override
45+
public int maxCompressedLength(int uncompressedSize)
46+
{
47+
// From Mark Adler's post http://stackoverflow.com/questions/1207877/java-size-of-compression-output-bytearray
48+
return uncompressedSize + ((uncompressedSize + 7) >> 3) + ((uncompressedSize + 63) >> 6) + 5 + EXTRA_COMPRESSION_SPACE;
49+
}
50+
51+
@Override
52+
public int compress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset, int maxOutputLength)
53+
{
54+
verifyRange(input, inputOffset, inputLength);
55+
verifyRange(output, outputOffset, maxOutputLength);
56+
57+
try (Deflater deflater = new Deflater(compressionLevel, true)) {
58+
deflater.setInput(input, inputOffset, inputLength);
59+
deflater.finish();
60+
61+
int compressedDataLength = deflater.deflate(output, outputOffset, maxOutputLength, FULL_FLUSH);
62+
if (!deflater.finished()) {
63+
throw new IllegalStateException("Output buffer too small");
64+
}
65+
return compressedDataLength;
66+
}
67+
}
68+
69+
@Override
70+
public int compress(MemorySegment input, MemorySegment output)
71+
{
72+
int maxCompressedLength = maxCompressedLength(toIntExact(input.byteSize()));
73+
if (output.byteSize() < maxCompressedLength) {
74+
throw new IllegalArgumentException("Output buffer must be at least " + maxCompressedLength + " bytes");
75+
}
76+
77+
try (Deflater deflater = new Deflater(compressionLevel, true)) {
78+
deflater.setInput(input.asByteBuffer());
79+
deflater.finish();
80+
81+
int compressedDataLength = deflater.deflate(output.asByteBuffer(), FULL_FLUSH);
82+
if (!deflater.finished()) {
83+
throw new IllegalStateException("maxCompressedLength formula is incorrect, because deflate produced more data");
84+
}
85+
return compressedDataLength;
86+
}
87+
}
88+
89+
private static void verifyRange(byte[] data, int offset, int length)
90+
{
91+
requireNonNull(data, "data is null");
92+
if (offset < 0 || length < 0 || offset + length > data.length) {
93+
throw new IllegalArgumentException("Invalid offset or length (%s, %s) in array of length %s".formatted(offset, length, data.length));
94+
}
95+
}
96+
}

0 commit comments

Comments
 (0)