mapping improved tests

Author: Baunsgaard

src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java

@@ -829,7 +829,7 @@ protected void copyInt(MapToInt d) {
 	 * @param d The array to copy
 	 */
 	public void copyInt(int[] d) {
-		copyInt(d, 0, size());
+		copyInt(d, 0, Math.min(d.length, size()));
 	}
 
 	public abstract void copyInt(int[] d, int start, int end);
@@ -846,7 +846,7 @@ public int getMax() {
 	}
 
 	/**
-	 * Get the maximum possible value to encode in this encoding. For instance in a bit you can encode 2 values
+	 * Get the maximum possible value to encode in this encoding. For instance in a bit you can encode 2 values therefore max is 1
 	 * 
 	 * @return The maximum number of distinct values to encode
 	 */

src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java

@@ -442,7 +442,7 @@ private static int longSize(int size) {
 	}
 
 	public int getMaxPossible() {
-		return 2;
+		return 1;
 	}
 
 	@Override

src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java

@@ -284,7 +284,7 @@ public AMapToData appendN(IMapToDataGroup[] d) {
 
 	@Override
 	public int getMaxPossible() {
-		return 256;
+		return 255;
 	}
 
 	@Override

src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToChar.java

@@ -46,7 +46,7 @@ public class MapToChar extends AMapToData {
 	private final char[] _data;
 
 	protected MapToChar(int size) {
-		this(Character.MAX_VALUE, size);
+		this(Character.MAX_VALUE+1, size);
 	}
 
 	public MapToChar(int unique, int size) {
@@ -328,7 +328,7 @@ public AMapToData appendN(IMapToDataGroup[] d) {
 
 	@Override
 	public int getMaxPossible() {
-		return Character.MAX_VALUE;
+		return Character.MAX_VALUE ;
 	}
 
 	@Override

src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToCharPByte.java

@@ -38,8 +38,7 @@ public class MapToCharPByte extends AMapToData {
 
 	private static final long serialVersionUID = 6315708056775476541L;
 
-	// 8323073
-	public static final int max = 0xFFFF * 127;
+	public static final int max = (0xFFFF + 1) * 128 -1;
 	private final char[] _data_c;
 	private final byte[] _data_b; // next byte after the char
 
@@ -280,7 +279,7 @@ public AMapToData appendN(IMapToDataGroup[] d) {
 
 	@Override
 	public int getMaxPossible() {
-		return Character.MAX_VALUE * 256;
+		return (Character.MAX_VALUE+1) * 256 -1;
 	}
 
 	@Override

src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToFactory.java

@@ -75,42 +75,45 @@ public static AMapToData create(int unique, IntArrayList values) {
 		return _data;
 	}
 
-	public static AMapToData create(int size, int[] values, int nUnique, int k) {
+	public static AMapToData create(int size, int[] values, int nUnique, int k) throws Exception {
 		AMapToData _data = create(size, nUnique);
-		ExecutorService pool = CommonThreadPool.get(k);
+		final ExecutorService pool = CommonThreadPool.get(k);
 		int blk = Math.max((values.length / k), 1024);
 		blk -= blk % 64; // ensure long size
 		List<Future<?>> tasks = new ArrayList<>();
-		for(int i = 0; i < values.length; i += blk){
+		for(int i = 0; i < values.length; i += blk) {
 			int start = i;
 			int end = Math.min(i + blk, values.length);
 			tasks.add(pool.submit(() -> _data.copyInt(values, start, end)));
 		}
+
+		for(Future<?> t : tasks)
+			t.get();
 		return _data;
 	}
 
 	/**
-	 * Create and allocate a map with the given size and support for upto the num tuples argument of values
+	 * Create and allocate a map with the given size and support for up to the num tuples argument of values
 	 * 
-	 * @param size      The number of cells to allocate
-	 * @param numTuples The maximum value to be able to represent inside the map.
+	 * @param size   The number of cells to allocate
+	 * @param unique The number of unique values to support (can encode unique -1)
 	 * @return A new map
 	 */
-	public static AMapToData create(final int size, final int numTuples) {
-		if(numTuples <= 1)
+	public static AMapToData create(final int size, final int unique) {
+		if(unique <= 1)
 			return new MapToZero(size);
-		else if(numTuples == 2 && size > 32)
-			return new MapToBit(numTuples, size);
-		else if(numTuples <= 127)
-			return new MapToUByte(numTuples, size);
-		else if(numTuples <= 256)
-			return new MapToByte(numTuples, size);
-		else if(numTuples <= Character.MAX_VALUE + 1)
-			return new MapToChar(numTuples, size);
-		else if(numTuples <= MapToCharPByte.max)
-			return new MapToCharPByte(numTuples, size);
+		else if(unique == 2 && size > 32)
+			return new MapToBit(unique, size);
+		else if(unique <= 128)
+			return new MapToUByte(unique, size);
+		else if(unique <= 256)
+			return new MapToByte(unique, size);
+		else if(unique <= Character.MAX_VALUE + 1)
+			return new MapToChar(unique, size);
+		else if(unique <= MapToCharPByte.max + 1)
+			return new MapToCharPByte(unique, size);
 		else
-			return new MapToInt(numTuples, size);
+			return new MapToInt(unique, size);
 	}
 
 	/**
@@ -181,20 +184,20 @@ public static AMapToData resizeForce(AMapToData d, MAP_TYPE t) {
 	/**
 	 * Estimate the size in memory of a MapToFactory.
 	 * 
-	 * @param size      The size of the mapping
-	 * @param numTuples The number of unique values to be supported by the mapping
+	 * @param size   The size of the mapping
+	 * @param unique The number of unique values to support (can encode unique -1)
 	 * @return The size in number of bytes.
 	 */
-	public static long estimateInMemorySize(int size, int numTuples) {
-		if(numTuples <= 1)
+	public static long estimateInMemorySize(int size, int unique) {
+		if(unique <= 1)
 			return MapToZero.getInMemorySize(size);
-		else if(numTuples == 2 && size > 32)
+		else if(unique == 2 && size > 32)
 			return MapToBit.getInMemorySize(size);
-		else if(numTuples <= 256)
+		else if(unique <= 256)
 			return MapToByte.getInMemorySize(size);
-		else if(numTuples <= Character.MAX_VALUE + 1)
+		else if(unique <= Character.MAX_VALUE + 1)
 			return MapToChar.getInMemorySize(size);
-		else if(numTuples <= MapToCharPByte.max)
+		else if(unique <= MapToCharPByte.max)
 			return MapToCharPByte.getInMemorySize(size);
 		else
 			return MapToInt.getInMemorySize(size);

src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToInt.java

@@ -37,7 +37,7 @@ public class MapToInt extends AMapToData {
 	private final int[] _data;
 
 	protected MapToInt(int size) {
-		this(Character.MAX_VALUE + 1, size);
+		this(Integer.MAX_VALUE, size);
 	}
 
 	public MapToInt(int unique, int size) {
@@ -226,11 +226,11 @@ public AMapToData resize(int unique) {
 			return new MapToZero(size);
 		else if(unique == 2 && size > 32)
 			ret = new MapToBit(unique, size);
-		else if(unique <= 127)
+		else if(unique < 128)
 			ret = new MapToUByte(unique, size);
 		else if(unique < 256)
 			ret = new MapToByte(unique, size);
-		else if(unique < Character.MAX_VALUE - 1)
+		else if(unique < Character.MAX_VALUE )
 			ret = new MapToChar(unique, size);
 		else if(unique < MapToCharPByte.max)
 			ret = new MapToCharPByte(unique, size);

src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToUByte.java

@@ -31,11 +31,11 @@ public class MapToUByte extends MapToByte {
 	private static final long serialVersionUID = -2498505439667351828L;
 
 	protected MapToUByte(int size) {
-		this(127, size);
+		this(128, size);
 	}
 
 	public MapToUByte(int unique, int size) {
-		super(Math.min(unique, 127), new byte[size]);
+		super(Math.min(unique, 128), new byte[size]);
 	}
 
 	protected MapToUByte(int unique, byte[] data) {
@@ -126,7 +126,7 @@ public int[] getCounts(int[] ret) {
 
 	@Override
 	public int getMaxPossible() {
-		return 128;
+		return 127;
 	}
 
 	@Override

src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToZero.java

@@ -185,7 +185,7 @@ public AMapToData appendN(IMapToDataGroup[] d) {
 
 	@Override
 	public int getMaxPossible() {
-		return 1;
+		return 0;
 	}
 
 	@Override

src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/DDCSchemeMC.java

@@ -189,7 +189,7 @@ private Pair<ICLAScheme, AColGroup> tryUpdateAndEncode(MatrixBlock data, ReaderC
 			while((cellVals = reader.nextRow()) != null) {
 				final int row = reader.getCurrentRowIndex();
 				final int id = map.increment(cellVals);
-				if(id >= max)
+				if(id > max)
 					throw new DMLCompressionException("Failed update and encode with " + max + " possible values");
 				d.set(row, id);
 			}
@@ -204,7 +204,7 @@ private Pair<ICLAScheme, AColGroup> tryUpdateAndEncode(MatrixBlock data, ReaderC
 						d.set(r++, emptyIdx.id);
 				}
 				final int id = map.increment(cellVals);
-				if(id >= max)
+				if(id > max)
 					throw new DMLCompressionException(
 						"Failed update and encode with " + max + " possible values" + map + " " + map.size());
 				d.set(row, id);