Skip to content

Add reusable Huffman decoder #701

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ppkarwasz wants to merge 8 commits into
base: master
Choose a base branch
from
Open

Add reusable Huffman decoder #701

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4a9e2c4
feat: add reusable Huffman decoder
ppkarwasz Aug 26, 2025
b12da14
feat(deflate64): use shared `HuffmanDecoder` and add bit-order support
ppkarwasz Aug 29, 2025
a17a654
fix: move `HuffmanDecoder` to `compressors.support`
ppkarwasz Aug 29, 2025
6cb2d47
Fix Javadoc on JDK 8
ppkarwasz Aug 29, 2025
fe3d66c
fix: remove cast to `char`
ppkarwasz Aug 29, 2025
29e0671
Merge remote-tracking branch 'apache/master' into feat/huffman
ppkarwasz Sep 25, 2025
3e63b0c
Add missing Javadoc since tag 1.29.0
garydgregory Sep 26, 2025
f699713
Add Javadoc @since tag
garydgregory Sep 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
190 changes: 31 additions & 159 deletions src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

import org.apache.commons.compress.compressors.CompressorException;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.compressors.support.HuffmanDecoder;
import org.apache.commons.compress.utils.BitInputStream;
import org.apache.commons.compress.utils.InputStreamStatistics;
import org.apache.commons.io.input.CloseShieldInputStream;
Expand Down Expand Up @@ -59,16 +60,16 @@ static final class Data {
*/
final int[] unzftab = new int[256]; // 1024 byte

// Needs indexes from 0 to MAX_CODE_LEN inclusive.
final int[][] limit = new int[N_GROUPS][MAX_CODE_LEN + 1];
// Needs indexes from 0 to MAX_CODE_LEN + 1 inclusive.
final int[][] base = new int[N_GROUPS][MAX_CODE_LEN + 2];
final int[][] perm = new int[N_GROUPS][MAX_ALPHA_SIZE]; // 6192 byte
final int[] minLens = new int[N_GROUPS]; // 24 byte

/**
* Huffman decoding tables.
*/
final HuffmanDecoder[] huffmanDecoders = new HuffmanDecoder[N_GROUPS];
/**
* Number of non-null {@link #huffmanDecoders}.
*/
int huffmanDecodersCount;
final int[] cftab = new int[257]; // 1028 byte
final char[] getAndMoveToFrontDecode_yy = new char[256]; // 512 byte
final char[][] temp_charArray2d = new char[N_GROUPS][MAX_ALPHA_SIZE]; // 3096
// byte
final byte[] recvDecodingTables_pos = new byte[N_GROUPS]; // 6 byte
// ---------------
Expand Down Expand Up @@ -158,85 +159,6 @@ private static void checkBounds(final int checkVal, final int limitExclusive, fi
}
}

/**
* Builds the Huffman decoding tables for use by {@code recvDecodingTables()}.
*
* @param alphaSize the alphabet size, guaranteed by the caller to be in the range [2, 258]
* (RUNA, RUNB, 255 byte values, and EOB).
* @param nGroups the number of Huffman coding groups, guaranteed by the caller to be in the range [0, 6].
* @param dataShadow the data structure into which the tables are built; requires
* {@code temp_charArray2d} to be initialized.
*/
static void createHuffmanDecodingTables(final int alphaSize, final int nGroups, final Data dataShadow) {
final char[][] len = dataShadow.temp_charArray2d;
final int[] minLens = dataShadow.minLens;
final int[][] limit = dataShadow.limit;
final int[][] base = dataShadow.base;
final int[][] perm = dataShadow.perm;

for (int t = 0; t < nGroups; t++) {
final char[] len_t = len[t];
int minLen = len_t[0];
int maxLen = len_t[0];
for (int i = 1; i < alphaSize; i++) {
final char lent = len_t[i];
if (lent > maxLen) {
maxLen = lent;
}
if (lent < minLen) {
minLen = lent;
}
}
hbCreateDecodeTables(limit[t], base[t], perm[t], len[t], minLen, maxLen, alphaSize);
minLens[t] = minLen;
}
}

/**
* Called by createHuffmanDecodingTables() exclusively.
*
* @param minLen minimum code length in the range [1, {@value MAX_CODE_LEN}] guaranteed by the caller.
* @param maxLen maximum code length in the range [1, {@value MAX_CODE_LEN}] guaranteed by the caller.
*/
private static void hbCreateDecodeTables(final int[] limit, final int[] base, final int[] perm, final char[] length, final int minLen, final int maxLen,
final int alphaSize) {
for (int i = minLen, pp = 0; i <= maxLen; i++) {
for (int j = 0; j < alphaSize; j++) {
if (length[j] == i) {
perm[pp++] = j;
}
}
}
// Ensure the arrays were not reused.
Arrays.fill(base, 0);
Arrays.fill(limit, minLen, maxLen + 1, 0);
// Compute histogram of code lengths, shifted by 1.
for (int i = 0; i < alphaSize; i++) {
final int len = length[i] + 1;
base[len]++;
}
// Compute cumulative counts: base[len] = # of codes with length < len.
// In other terms: base[len] = index of the first code in the `perm` table.
for (int len = 1; len < base.length; len++) {
base[len] += base[len - 1];
}
// Compute the last code for each length.
int vec = 0;
for (int len = minLen; len <= maxLen; len++) {
// increment by the number of length `len` codes
vec += base[len + 1] - base[len];
// vec is now the last code of length `len` + 1
limit[len] = vec - 1;
vec <<= 1;
}
// Compute the bias between code value and table index.
// base[minLen] cannot be computed using this rule, since limit[minLen - 1] does not exist,
// but has already the correct value 0.
for (int len = minLen + 1; len <= maxLen; len++) {
base[len] = (limit[len - 1] + 1 << 1) - base[len];
}
}

private static void makeMaps(final Data data) throws IOException {
final boolean[] inUse = data.inUse;
final byte[] seqToUnseq = data.seqToUnseq;
Expand Down Expand Up @@ -334,27 +256,24 @@ static void recvDecodingTables(final BitInputStream bin, final Data dataShadow)
selector[i] = tmp;
}

final char[][] len = dataShadow.temp_charArray2d;

/* Now the coding tables */
/* Now the Huffman coding tables */
for (int t = 0; t < nGroups; t++) {
final int[] codeLengths = new int[alphaSize];
int curr = bsR(bin, 5);
final char[] len_t = len[t];
for (int i = 0; i < alphaSize; i++) {
while (bsGetBit(bin)) {
curr += bsGetBit(bin) ? -1 : 1;
}
// Same condition as in bzip2
if (curr < 1 || curr > MAX_CODE_LEN) {
throw new CompressorException(
"Corrupted input, code length value out of range [%d, %d]: %d", 1, MAX_CODE_LEN, curr);
}
len_t[i] = (char) curr;
codeLengths[i] = (char) curr;
}
try {
// Same limits as in the reference C implementation of bzip2
dataShadow.huffmanDecoders[t] = new HuffmanDecoder(codeLengths, alphaSize, 1, MAX_CODE_LEN);
} catch (final IllegalArgumentException e) {
throw new CompressorException("Invalid Huffman data: " + e.getMessage(), e);
}
}

// finally create the Huffman tables
createHuffmanDecodingTables(alphaSize, nGroups, dataShadow);
dataShadow.huffmanDecodersCount = nGroups;
}

// Variables used by setup* methods exclusively
Expand Down Expand Up @@ -474,10 +393,6 @@ private void getAndMoveToFrontDecode() throws IOException {
final byte[] selector = dataShadow.selector;
final byte[] seqToUnseq = dataShadow.seqToUnseq;
final char[] yy = dataShadow.getAndMoveToFrontDecode_yy;
final int[] minLens = dataShadow.minLens;
final int[][] limit = dataShadow.limit;
final int[][] base = dataShadow.base;
final int[][] perm = dataShadow.perm;
final int limitLast = this.blockSize100k * 100000;
/*
* Setting up the unzftab entries here is not strictly necessary, but it does save having to do it later in a separate pass, and so saves a block's
Expand All @@ -487,18 +402,15 @@ private void getAndMoveToFrontDecode() throws IOException {
yy[i] = (char) i;
unzftab[i] = 0;
}
int groupNo = 0;
int groupPos = G_SIZE - 1;
final int eob = dataShadow.inUseCount + 1;
int nextSym = getAndMoveToFrontDecode0();
int lastShadow = -1;

// Initialize group, selector and huffmanDecoder
int groupNo = 0;
int zt = selector[groupNo] & 0xff;
// All arrays have the same length
checkBounds(zt, base.length, "zt");
int[] base_zt = base[zt];
int[] limit_zt = limit[zt];
int[] perm_zt = perm[zt];
int minLens_zt = minLens[zt];
HuffmanDecoder currentDecoder = getHuffmanDecoder(dataShadow, zt);
int nextSym = currentDecoder.decodeSymbol(bin);
while (nextSym != eob) {
if (nextSym == RUNA || nextSym == RUNB) {
int s = -1;
Expand All @@ -514,25 +426,11 @@ private void getAndMoveToFrontDecode() throws IOException {
groupPos = G_SIZE - 1;
checkBounds(++groupNo, selector.length, "groupNo");
zt = selector[groupNo] & 0xff;
// All arrays have the same length
checkBounds(zt, base.length, "zt");
base_zt = base[zt];
limit_zt = limit[zt];
perm_zt = perm[zt];
minLens_zt = minLens[zt];
currentDecoder = getHuffmanDecoder(dataShadow, zt);
} else {
groupPos--;
}
int zn = minLens_zt;
checkBounds(zn, limit_zt.length, "zn");
int zvec = bsR(bin, zn);
while (zvec > limit_zt[zn]) {
checkBounds(++zn, limit_zt.length, "zn");
zvec = zvec << 1 | bsR(bin, 1);
}
final int tmp = zvec - base_zt[zn];
checkBounds(tmp, perm_zt.length, "zvec");
nextSym = perm_zt[tmp];
nextSym = currentDecoder.decodeSymbol(bin);
}
checkBounds(s, this.data.ll8.length, "s");
final int yy0 = yy[0];
Expand Down Expand Up @@ -570,45 +468,19 @@ private void getAndMoveToFrontDecode() throws IOException {
groupPos = G_SIZE - 1;
checkBounds(++groupNo, selector.length, "groupNo");
zt = selector[groupNo] & 0xff;
// All arrays have the same length
checkBounds(zt, base.length, "zt");
base_zt = base[zt];
limit_zt = limit[zt];
perm_zt = perm[zt];
minLens_zt = minLens[zt];
currentDecoder = getHuffmanDecoder(dataShadow, zt);
} else {
groupPos--;
}
int zn = minLens_zt;
checkBounds(zn, limit_zt.length, "zn");
int zvec = bsR(bin, zn);
while (zvec > limit_zt[zn]) {
checkBounds(++zn, limit_zt.length, "zn");
zvec = zvec << 1 | bsR(bin, 1);
}
final int idx = zvec - base_zt[zn];
checkBounds(idx, perm_zt.length, "zvec");
nextSym = perm_zt[idx];
nextSym = currentDecoder.decodeSymbol(bin);
}
}
this.last = lastShadow;
}

private int getAndMoveToFrontDecode0() throws IOException {
final Data dataShadow = this.data;
final int zt = dataShadow.selector[0] & 0xff;
checkBounds(zt, dataShadow.limit.length, "zt");
final int[] limit_zt = dataShadow.limit[zt];
int zn = dataShadow.minLens[zt];
checkBounds(zn, limit_zt.length, "zn");
int zvec = bsR(bin, zn);
while (zvec > limit_zt[zn]) {
checkBounds(++zn, limit_zt.length, "zn");
zvec = zvec << 1 | bsR(bin, 1);
}
final int tmp = zvec - dataShadow.base[zt][zn];
checkBounds(tmp, dataShadow.perm[zt].length, "zvec");
return dataShadow.perm[zt][tmp];
private static HuffmanDecoder getHuffmanDecoder(final Data dataShadow, final int zt) throws IOException {
checkBounds(zt, dataShadow.huffmanDecodersCount, "zt");
return dataShadow.huffmanDecoders[zt];
}

/**
Expand Down
6 changes: 3 additions & 3 deletions ...ava/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@
*/
public class Deflate64CompressorInputStream extends CompressorInputStream implements InputStreamStatistics {
private InputStream originalStream;
private HuffmanDecoder decoder;
private Deflate64Decoder decoder;
private long compressedBytesRead;
private final byte[] oneByte = new byte[1];

Deflate64CompressorInputStream(final HuffmanDecoder decoder) {
Deflate64CompressorInputStream(final Deflate64Decoder decoder) {
this.decoder = decoder;
}

Expand All @@ -49,7 +49,7 @@ public class Deflate64CompressorInputStream extends CompressorInputStream implem
* @param in the stream to read from
*/
public Deflate64CompressorInputStream(final InputStream in) {
this(new HuffmanDecoder(in));
this(new Deflate64Decoder(in));
originalStream = in;
}

Expand Down
Loading
Loading