Compute max compressed length for SkippableIntegratedIntegerCODEC

Author: piotrrzysko

example.java

@@ -104,11 +104,7 @@ public static void basicExampleHeadless() {
         // be processed using variable byte
         SkippableIntegratedComposition codec = new SkippableIntegratedComposition(new IntegratedBinaryPacking(),
                 new IntegratedVariableByte());
-        // output vector should be large enough...
-        int[] compressed = new int[data.length + 1024];
-        // compressed might not be large enough in some cases
-        // if you get java.lang.ArrayIndexOutOfBoundsException, try
-        // allocating more memory
+        int[] compressed = new int[codec.maxHeadlessCompressedLength(new IntWrapper(0), data.length)];
 
         /**
          *

src/main/java/me/lemire/integercompression/differential/IntegratedBinaryPacking.java

@@ -49,7 +49,8 @@
 public class IntegratedBinaryPacking implements IntegratedIntegerCODEC,
         SkippableIntegratedIntegerCODEC {
 
-    static final int BLOCK_SIZE = 32;
+    public static final int BLOCK_SIZE = 32;
+    private static final int MAX_BIT_WIDTH = Integer.SIZE;
 
     @Override
     public void compress(int[] in, IntWrapper inpos, int inlength, int[] out,
@@ -170,4 +171,13 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
         initvalue.set(initoffset);
         inpos.set(tmpinpos);
     }
+
+    @Override
+    public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+        int blockCount = inlength / BLOCK_SIZE;
+        int headersSizeInInts = blockCount / Integer.BYTES + (blockCount % Integer.BYTES);
+        int blocksSizeInInts = blockCount * MAX_BIT_WIDTH;
+        compressedPositions.add(blockCount * BLOCK_SIZE);
+        return headersSizeInInts + blocksSizeInInts;
+    }
 }

src/main/java/me/lemire/integercompression/differential/IntegratedIntCompressor.java

@@ -39,7 +39,8 @@ public IntegratedIntCompressor() {
      * @throws UncompressibleInputException if the data is too poorly compressible
      */
     public  int[] compress(int[] input) {
-        int [] compressed = new int[input.length + input.length / 100 + 1024];
+        int maxCompressedLength = codec.maxHeadlessCompressedLength(new IntWrapper(0), input.length);
+        int [] compressed = new int[maxCompressedLength + 1]; // +1 to store the length of the input
         compressed[0] = input.length;
         IntWrapper outpos = new IntWrapper(1);
         IntWrapper initvalue = new IntWrapper(0);

src/main/java/me/lemire/integercompression/differential/IntegratedVariableByte.java

@@ -24,6 +24,8 @@
 public class IntegratedVariableByte implements IntegratedIntegerCODEC, IntegratedByteIntegerCODEC,
 SkippableIntegratedIntegerCODEC  {
 
+    private static final int MAX_BYTES_PER_INT = 5;
+
     private static byte extract7bits(int i, long val) {
         return (byte)((val >> (7 * i)) & ((1 << 7) - 1));
     }
@@ -257,6 +259,14 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
         inpos.set(p + (s!=0 ? 1 : 0));        
     }
 
+    @Override
+    public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+        int maxLengthInBytes = inlength * MAX_BYTES_PER_INT;
+        int maxLengthInInts = (maxLengthInBytes + Integer.BYTES - 1) / Integer.BYTES;
+        compressedPositions.add(inlength);
+        return maxLengthInInts;
+    }
+
     /**
      * Creates a new buffer of the requested size.
      *

src/main/java/me/lemire/integercompression/differential/SkippableIntegratedComposition.java

@@ -76,4 +76,13 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
         F2.headlessUncompress(in, inpos, inlength, out, outpos,num,initvalue);
     }
 
+    @Override
+    public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+        int init = compressedPositions.get();
+        int maxLength = F1.maxHeadlessCompressedLength(compressedPositions, inlength);
+        maxLength += 1; // Add +1 for the potential F2 header. Question: is this header actually needed in the headless version?
+        inlength -= compressedPositions.get() - init;
+        maxLength += F2.maxHeadlessCompressedLength(compressedPositions, inlength);
+        return maxLength;
+    }
 }

src/main/java/me/lemire/integercompression/differential/SkippableIntegratedIntegerCODEC.java

@@ -71,4 +71,21 @@ public void headlessCompress(int[] in, IntWrapper inpos, int inlength, int[] out
     public void headlessUncompress(int[] in, IntWrapper inpos, int inlength, int[] out,
             IntWrapper outpos, int num, IntWrapper initvalue);
 
+    /**
+     * Compute the maximum number of integers that might be required to store
+     * the compressed form of a given input array segment, without headers.
+     * <p>
+     * This is useful to pre-allocate the output buffer before calling
+     * {@link #headlessCompress(int[], IntWrapper, int, int[], IntWrapper, IntWrapper)}.
+     * </p>
+     *
+     * @param compressedPositions
+     *        since not all schemes compress every input integer, this parameter
+     *        returns how many input integers will actually be compressed.
+     *        This is useful when composing multiple schemes.
+     * @param inlength
+     *            number of integers to be compressed
+     * @return the maximum number of integers needed in the output array
+     */
+    int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength);
 }

src/test/java/me/lemire/integercompression/AdhocTest.java

@@ -108,11 +108,11 @@ public void testIssue29b() {
     @Test
     public void testIssue41() {
         for (int x = 0; x < 64; x++) {
-            int[] a = { 2, 3, 4, 5 };
-            int[] b = new int[90];
-            int[] c = new int[a.length];
             SkippableIntegratedIntegerCODEC codec = new SkippableIntegratedComposition(new IntegratedBinaryPacking(),
                     new IntegratedVariableByte());
+            int[] a = { 2, 3, 4, 5 };
+            int[] b = new int[x + codec.maxHeadlessCompressedLength(new IntWrapper(0), a.length)];
+            int[] c = new int[a.length];
             IntWrapper aOffset = new IntWrapper(0);
             IntWrapper bOffset = new IntWrapper(x);
             IntWrapper initValue = new IntWrapper(0);

src/test/java/me/lemire/integercompression/ExampleTest.java

@@ -116,11 +116,7 @@ public void basicExampleHeadless() {
         // be processed using variable byte
         SkippableIntegratedComposition codec = new SkippableIntegratedComposition(new IntegratedBinaryPacking(),
                 new IntegratedVariableByte());
-        // output vector should be large enough...
-        int[] compressed = new int[data.length + 1024];
-        // compressed might not be large enough in some cases
-        // if you get java.lang.ArrayIndexOutOfBoundsException, try
-        // allocating more memory
+        int[] compressed = new int[codec.maxHeadlessCompressedLength(new IntWrapper(0), data.length)];
 
         /**
          *

src/test/java/me/lemire/integercompression/SkippableBasicTest.java

@@ -9,8 +9,13 @@
 
 import java.util.Arrays;
 
+import me.lemire.integercompression.differential.IntegratedBinaryPacking;
+import me.lemire.integercompression.differential.IntegratedVariableByte;
+import me.lemire.integercompression.differential.SkippableIntegratedComposition;
+import me.lemire.integercompression.differential.SkippableIntegratedIntegerCODEC;
 import org.junit.Test;
 
+import static org.junit.Assert.assertTrue;
 
 /**
  * Just some basic sanity tests.
@@ -147,5 +152,33 @@ public void varyingLengthTest2() {
         }
     }
 
+    @Test
+    public void testMaxHeadlessCompressedLength() {
+        testMaxHeadlessCompressedLength(new IntegratedBinaryPacking(), 16 * IntegratedBinaryPacking.BLOCK_SIZE);
+        testMaxHeadlessCompressedLength(new IntegratedVariableByte(), 128);
+        testMaxHeadlessCompressedLength(new SkippableIntegratedComposition(new IntegratedBinaryPacking(), new IntegratedVariableByte()), 16 * IntegratedBinaryPacking.BLOCK_SIZE + 10);
+    }
+
+    private static void testMaxHeadlessCompressedLength(SkippableIntegratedIntegerCODEC codec, int inlengthTo) {
+        // We test the worst-case scenario by making all deltas and the initial value negative.
+        int delta = -1;
+        int value = delta;
+
+        for (int inlength = 0; inlength < inlengthTo; ++inlength) {
+            int[] input = new int[inlength];
+            for (int i = 0; i < inlength; i++) {
+                input[i] = value;
+                value += delta;
+            }
 
+            int maxOutputLength = codec.maxHeadlessCompressedLength(new IntWrapper(), inlength);
+            int[] output = new int[maxOutputLength];
+            IntWrapper outPos = new IntWrapper();
+
+            codec.headlessCompress(input, new IntWrapper(), inlength, output, outPos, new IntWrapper());
+            // If we reach this point, no exception was thrown, which means the calculated output length was sufficient.
+
+            assertTrue(maxOutputLength <= outPos.get() + 1); // +1 because SkippableIntegratedComposition always adds one extra integer for the potential header
+        }
+    }
 }