Compute max compressed length for SkippableIntegerCODEC

Currently, only BinaryPacking and VariableByte provide real implementations
that calculate the maximum compressed length. This commit demonstrates how
the approach might work. Once we agree that the design is correct, we can
extend it to other schemes.

Author: piotrrzysko

example.java

@@ -263,10 +263,12 @@ public static void headlessDemo() {
         int[] uncompressed1 = {1,2,1,3,1};
         int[] uncompressed2 = {3,2,4,6,1};
 
-        int[] compressed = new int[uncompressed1.length+uncompressed2.length+1024];
-
         SkippableIntegerCODEC codec = new SkippableComposition(new BinaryPacking(), new VariableByte());
 
+        int maxCompressedLength = codec.maxHeadlessCompressedLength(new IntWrapper(0), uncompressed1.length)
+                                  + codec.maxHeadlessCompressedLength(new IntWrapper(0), uncompressed2.length);
+        int[] compressed = new int[maxCompressedLength];
+
         // compressing
         IntWrapper outPos = new IntWrapper();
 

src/main/java/me/lemire/integercompression/BinaryPacking.java

@@ -38,7 +38,8 @@
  */
 public final class BinaryPacking implements IntegerCODEC, SkippableIntegerCODEC {
         final static int BLOCK_SIZE = 32;
-    
+        private static final int MAX_BIT_WIDTH = Integer.SIZE;
+
         @Override
         public void compress(int[] in, IntWrapper inpos, int inlength,
                 int[] out, IntWrapper outpos) {
@@ -131,7 +132,16 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
             outpos.add(outlength);
             inpos.set(tmpinpos);
         }
-        
+
+        @Override
+        public int maxHeadlessCompressedLength(IntWrapper inpos, int inlength) {
+            int blockCount = inlength / BLOCK_SIZE;
+            int headersSizeInInts = blockCount / Integer.BYTES + (blockCount % Integer.BYTES);
+            int blocksSizeInInts = blockCount * MAX_BIT_WIDTH;
+            inpos.add(blockCount * BLOCK_SIZE);
+            return headersSizeInInts + blocksSizeInInts;
+        }
+
         @Override
         public String toString() {
                 return this.getClass().getSimpleName();

src/main/java/me/lemire/integercompression/IntCompressor.java

@@ -36,7 +36,8 @@ public IntCompressor() {
      * @throws UncompressibleInputException if the data is too poorly compressible
      */
     public  int[] compress(int[] input) {
-        int[] compressed = new int[input.length + input.length / 100 + 1024];
+        int maxCompressedLength = codec.maxHeadlessCompressedLength(new IntWrapper(0), input.length);
+        int[] compressed = new int[maxCompressedLength];
         // Store at index=0 the length of the input, hence enabling .headlessCompress
         compressed[0] = input.length;
         IntWrapper outpos = new IntWrapper(1);

src/main/java/me/lemire/integercompression/SkippableComposition.java

@@ -61,6 +61,16 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength, int[] o
         F2.headlessUncompress(in, inpos, inlength, out, outpos, num);
     }
 
+    @Override
+    public int maxHeadlessCompressedLength(IntWrapper inpos, int inlength) {
+        int init = inpos.get();
+        int maxLength = F1.maxHeadlessCompressedLength(inpos, inlength);
+        maxLength += 1; // Add +1 for the potential F2 header. Question: is this header actually needed in the headless version?
+        inlength -= inpos.get() - init;
+        maxLength += F2.maxHeadlessCompressedLength(inpos, inlength);
+        return maxLength;
+    }
+
     @Override
     public String toString() {
         return F1.toString() + "+" + F2.toString();

src/main/java/me/lemire/integercompression/SkippableIntegerCODEC.java

@@ -69,4 +69,25 @@ public void headlessCompress(int[] in, IntWrapper inpos, int inlength, int[] out
     public void headlessUncompress(int[] in, IntWrapper inpos, int inlength, int[] out,
             IntWrapper outpos, int num);
 
+    /**
+     * Compute the maximum number of integers that might be required to store
+     * the compressed form of a given input array segment, without headers.
+     * <p>
+     * This is useful to pre-allocate the output buffer before calling
+     * {@link #headlessCompress(int[], IntWrapper, int, int[], IntWrapper)}.
+     * </p>
+     *
+     * @param inpos
+     *            starting position in the input array (passed by reference)
+     * @param inlength
+     *            number of integers to be compressed
+     * @return the maximum number of integers needed in the output array
+     */
+    default int maxHeadlessCompressedLength(IntWrapper inpos, int inlength) {
+        if (inlength == 0) {
+            return 0;
+        }
+        inpos.add(inlength);
+        return inlength + inlength / 100 + 1024;
+    }
 }

src/main/java/me/lemire/integercompression/VariableByte.java

@@ -21,6 +21,8 @@
  */
 public class VariableByte implements IntegerCODEC, ByteIntegerCODEC, SkippableIntegerCODEC {
 
+    private static final int MAX_BYTES_PER_INT = 5;
+
     private static byte extract7bits(int i, long val) {
         return (byte) ((val >> (7 * i)) & ((1 << 7) - 1));
     }
@@ -208,6 +210,14 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength, int[] o
         inpos.set(p + (s!=0 ? 1 : 0));
     }
 
+    @Override
+    public int maxHeadlessCompressedLength(IntWrapper inpos, int inlength) {
+        int maxLengthInBytes = inlength * MAX_BYTES_PER_INT;
+        int maxLengthInInts = (maxLengthInBytes + Integer.BYTES - 1) / Integer.BYTES;
+        inpos.add(inlength);
+        return maxLengthInInts;
+    }
+
     /**
      * Creates a new buffer of the requested size.
      *

src/test/java/me/lemire/integercompression/AdhocTest.java

@@ -86,10 +86,10 @@ public void testIssue29() {
     @Test
     public void testIssue29b() {
         for(int x = 0; x < 64; x++) {
+            SkippableIntegerCODEC codec = new SkippableComposition(new BinaryPacking(), new VariableByte());
             int[] a = {2, 3, 4, 5};
-            int[] b = new int[90];
+            int[] b = new int[x + codec.maxHeadlessCompressedLength(new IntWrapper(0), a.length)];
             int[] c = new int[a.length];
-            SkippableIntegerCODEC codec = new SkippableComposition(new BinaryPacking(), new VariableByte());
             IntWrapper aOffset = new IntWrapper(0);
             IntWrapper bOffset = new IntWrapper(x);
             codec.headlessCompress(a, aOffset, a.length, b, bOffset);

src/test/java/me/lemire/integercompression/ExampleTest.java

@@ -276,10 +276,12 @@ public void headlessDemo() {
         int[] uncompressed1 = { 1, 2, 1, 3, 1 };
         int[] uncompressed2 = { 3, 2, 4, 6, 1 };
 
-        int[] compressed = new int[uncompressed1.length + uncompressed2.length + 1024];
-
         SkippableIntegerCODEC codec = new SkippableComposition(new BinaryPacking(), new VariableByte());
 
+        int maxCompressedLength = codec.maxHeadlessCompressedLength(new IntWrapper(0), uncompressed1.length)
+                + codec.maxHeadlessCompressedLength(new IntWrapper(0), uncompressed2.length);
+        int[] compressed = new int[maxCompressedLength];
+
         // compressing
         IntWrapper outPos = new IntWrapper();
 

src/test/java/me/lemire/integercompression/SkippableBasicTest.java

@@ -48,10 +48,11 @@ public void consistentTest() {
         for (SkippableIntegerCODEC c : codecs) {
             System.out.println("[SkippeableBasicTest.consistentTest] codec = "
                     + c);
-            int[] outBuf = new int[N + 1024];
             for (int n = 0; n <= N; ++n) {
                 IntWrapper inPos = new IntWrapper();
                 IntWrapper outPos = new IntWrapper();
+                int[] outBuf = new int[c.maxHeadlessCompressedLength(new IntWrapper(0), n)];
+
                 c.headlessCompress(data, inPos, n, outBuf, outPos);
 
                 IntWrapper inPoso = new IntWrapper();

src/test/java/me/lemire/integercompression/TestUtils.java

@@ -165,7 +165,7 @@ protected static int[] uncompress(ByteIntegerCODEC codec, byte[] data, int len)
     }
 
     protected static int[] compressHeadless(SkippableIntegerCODEC codec, int[] data) {
-        int[] outBuf = new int[data.length * 4];
+        int[] outBuf = new int[codec.maxHeadlessCompressedLength(new IntWrapper(0), data.length)];
         IntWrapper inPos = new IntWrapper();
         IntWrapper outPos = new IntWrapper();
         codec.headlessCompress(data, inPos, data.length, outBuf, outPos);