|
| 1 | +package com.clickhouse.client.api.internal; |
| 2 | + |
| 3 | +import com.clickhouse.client.api.ClientException; |
| 4 | +import com.clickhouse.data.ClickHouseByteUtils; |
| 5 | +import com.clickhouse.data.ClickHouseCityHash; |
| 6 | +import com.clickhouse.data.ClickHouseUtils; |
| 7 | +import net.jpountz.lz4.LZ4FastDecompressor; |
| 8 | +import org.slf4j.Logger; |
| 9 | +import org.slf4j.LoggerFactory; |
| 10 | + |
| 11 | +import java.io.EOFException; |
| 12 | +import java.io.IOException; |
| 13 | +import java.io.InputStream; |
| 14 | +import java.nio.ByteBuffer; |
| 15 | + |
| 16 | +public class ClickHouseLZ4InputStream extends InputStream { |
| 17 | + |
| 18 | + private static Logger LOG = LoggerFactory.getLogger(ClickHouseLZ4InputStream.class); |
| 19 | + private final LZ4FastDecompressor decompressor; |
| 20 | + |
| 21 | + private final InputStream in; |
| 22 | + |
| 23 | + private ByteBuffer buffer; |
| 24 | + |
| 25 | + private byte[] tmpBuffer = new byte[1]; |
| 26 | + |
| 27 | + |
| 28 | + public ClickHouseLZ4InputStream(InputStream in, LZ4FastDecompressor decompressor, int bufferSize) { |
| 29 | + super(); |
| 30 | + this.decompressor = decompressor; |
| 31 | + this.in = in; |
| 32 | + this.buffer = ByteBuffer.allocate(bufferSize); |
| 33 | + this.buffer.limit(0); |
| 34 | + } |
| 35 | + |
| 36 | + @Override |
| 37 | + public int read() throws IOException { |
| 38 | + int n = read(tmpBuffer, 0, 1); |
| 39 | + return n == -1 ? -1 : tmpBuffer[0] & 0xFF; |
| 40 | + } |
| 41 | + |
| 42 | + @Override |
| 43 | + public int read(byte[] b, int off, int len) throws IOException { |
| 44 | + if (b == null) { |
| 45 | + throw new NullPointerException("b is null"); |
| 46 | + } else if (off < 0) { |
| 47 | + throw new IndexOutOfBoundsException("off is negative"); |
| 48 | + } else if (len < 0) { |
| 49 | + throw new IndexOutOfBoundsException("len is negative"); |
| 50 | + } else if (off + len > b.length) { |
| 51 | + throw new IndexOutOfBoundsException("off + len is greater than b.length"); |
| 52 | + } else if (len == 0) { |
| 53 | + return 0; |
| 54 | + } |
| 55 | + |
| 56 | + int readBytes = 0; |
| 57 | + do { |
| 58 | + int remaining = Math.min(len - readBytes, buffer.remaining()); |
| 59 | + buffer.get(b, off + readBytes, remaining); |
| 60 | + readBytes += remaining; |
| 61 | + } while (readBytes < len && refill() != -1); |
| 62 | + |
| 63 | + return readBytes == 0 ? -1 : readBytes; |
| 64 | + } |
| 65 | + |
| 66 | + |
| 67 | + static final byte MAGIC = (byte) 0x82; |
| 68 | + static final int HEADER_LENGTH = 25; |
| 69 | + |
| 70 | + static final byte[] headerBuff = new byte[HEADER_LENGTH]; |
| 71 | + |
| 72 | + /** |
| 73 | + * Method ensures to read all bytes from the input stream. |
| 74 | + * In case of network connection it may be a case when not all bytes are read at once. |
| 75 | + * @throws IOException |
| 76 | + */ |
| 77 | + private boolean readFully(byte[] b, int off, int len) throws IOException { |
| 78 | + int n = 0; |
| 79 | + while (n < len) { |
| 80 | + int count = in.read(b, off + n, len - n); |
| 81 | + if (count < 0) { |
| 82 | + if (n == 0) { |
| 83 | + return false; |
| 84 | + } |
| 85 | + throw new IOException(ClickHouseUtils.format("Incomplete read: {0} of {1}", n, len)); |
| 86 | + } |
| 87 | + n += count; |
| 88 | + } |
| 89 | + |
| 90 | + return true; |
| 91 | + } |
| 92 | + |
| 93 | + private int refill() throws IOException { |
| 94 | + |
| 95 | + // read header |
| 96 | + boolean readFully = readFully(headerBuff, 0, HEADER_LENGTH); |
| 97 | + if (!readFully) { |
| 98 | + return -1; |
| 99 | + } |
| 100 | + |
| 101 | + if (headerBuff[16] != MAGIC) { |
| 102 | + // 1 byte - 0x82 (shows this is LZ4) |
| 103 | + throw new ClientException("Invalid LZ4 magic byte: '" + headerBuff[16] + "'"); |
| 104 | + } |
| 105 | + |
| 106 | + // 4 bytes - size of the compressed data including 9 bytes of the header |
| 107 | + int compressedSizeWithHeader = getInt32(headerBuff, 17); |
| 108 | + // 4 bytes - size of uncompressed data |
| 109 | + int uncompressedSize = getInt32(headerBuff, 21); |
| 110 | + |
| 111 | + int offset = 9; |
| 112 | + final byte[] block = new byte[compressedSizeWithHeader]; |
| 113 | + block[0] = MAGIC; |
| 114 | + setInt32(block, 1, compressedSizeWithHeader); |
| 115 | + setInt32(block, 5, uncompressedSize); |
| 116 | + // compressed data: compressed_size - 9 bytes |
| 117 | + int remaining = compressedSizeWithHeader - offset; |
| 118 | + |
| 119 | + readFully = readFully(block, offset, remaining); |
| 120 | + if (!readFully) { |
| 121 | + throw new EOFException("Unexpected end of stream"); |
| 122 | + } |
| 123 | + |
| 124 | + long[] real = ClickHouseCityHash.cityHash128(block, 0, compressedSizeWithHeader); |
| 125 | + if (real[0] != getInt64(headerBuff, 0) || real[1] != ClickHouseByteUtils.getInt64(headerBuff, 8)) { |
| 126 | + throw new ClientException("Corrupted stream: checksum mismatch"); |
| 127 | + } |
| 128 | + |
| 129 | + if (buffer.capacity() < uncompressedSize) { |
| 130 | + buffer = ByteBuffer.allocate(uncompressedSize); |
| 131 | + LOG.warn("Buffer size is too small, reallocate buffer with size: " + uncompressedSize); |
| 132 | + } |
| 133 | + decompressor.decompress(ByteBuffer.wrap(block), offset, buffer, 0, uncompressedSize); |
| 134 | + buffer.position(0); |
| 135 | + buffer.limit(uncompressedSize); |
| 136 | + return uncompressedSize; |
| 137 | + } |
| 138 | + |
| 139 | + /** |
| 140 | + * Read int32 Little Endian |
| 141 | + * @param bytes |
| 142 | + * @param offset |
| 143 | + * @return |
| 144 | + */ |
| 145 | + static int getInt32(byte[] bytes, int offset) { |
| 146 | + return (0xFF & bytes[offset]) | ((0xFF & bytes[offset + 1]) << 8) | ((0xFF & bytes[offset + 2]) << 16) |
| 147 | + | ((0xFF & bytes[offset + 3]) << 24); |
| 148 | + } |
| 149 | + |
| 150 | + /** |
| 151 | + * Read int64 Little Endian |
| 152 | + * @param bytes |
| 153 | + * @param offset |
| 154 | + * @return |
| 155 | + */ |
| 156 | + static long getInt64(byte[] bytes, int offset) { |
| 157 | + return (0xFFL & bytes[offset]) | ((0xFFL & bytes[offset + 1]) << 8) | ((0xFFL & bytes[offset + 2]) << 16) |
| 158 | + | ((0xFFL & bytes[offset + 3]) << 24) | ((0xFFL & bytes[offset + 4]) << 32) |
| 159 | + | ((0xFFL & bytes[offset + 5]) << 40) | ((0xFFL & bytes[offset + 6]) << 48) |
| 160 | + | ((0xFFL & bytes[offset + 7]) << 56); |
| 161 | + } |
| 162 | + |
| 163 | + static void setInt32(byte[] bytes, int offset, int value) { |
| 164 | + bytes[offset] = (byte) (0xFF & value); |
| 165 | + bytes[offset + 1] = (byte) (0xFF & (value >> 8)); |
| 166 | + bytes[offset + 2] = (byte) (0xFF & (value >> 16)); |
| 167 | + bytes[offset + 3] = (byte) (0xFF & (value >> 24)); |
| 168 | + } |
| 169 | + |
| 170 | + @Override |
| 171 | + public void close() throws IOException { |
| 172 | + super.close(); |
| 173 | + } |
| 174 | +} |
0 commit comments