[SYSTEMDS-3644] Compressed transform encode

Author: Baunsgaard

src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupIO.java

@@ -94,7 +94,9 @@ public static long getExactSizeOnDisk(List<AColGroup> colGroups) {
 			}
 			ret += grp.getExactSizeOnDisk();
 		}
-		LOG.error(" duplicate dicts on exact Size on Disk : " + (colGroups.size() - dicts.size()) );
+		if(LOG.isWarnEnabled())
+			LOG.warn(" duplicate dicts on exact Size on Disk : " + (colGroups.size() - dicts.size()) );
+		
 		return ret;
 	}
 

src/main/java/org/apache/sysds/runtime/compress/lib/CLALibDecompress.java

@@ -237,7 +237,7 @@ private static MatrixBlock decompressExecute(CompressedMatrixBlock cmb, int k) {
 			if(ret.isInSparseFormat()) 
 				decompressSparseSingleThread(ret, filteredGroups, nRows, blklen);
 			else 
-				decompressDenseSingleThread(ret, filteredGroups, nRows, blklen, constV, eps, nonZeros, overlapping);
+				decompressDenseSingleThread(ret, filteredGroups, nRows, blklen, constV, eps, overlapping);
 		}
 		else if(ret.isInSparseFormat()) 
 			decompressSparseMultiThread(ret, filteredGroups, nRows, blklen, k);
@@ -294,7 +294,7 @@ private static void decompressSparseSingleThread(MatrixBlock ret, List<AColGroup
 	}
 
 	private static void decompressDenseSingleThread(MatrixBlock ret, List<AColGroup> filteredGroups, int rlen,
-		int blklen, double[] constV, double eps, long nonZeros, boolean overlapping) {
+		int blklen, double[] constV, double eps, boolean overlapping) {
 
 		final DenseBlock db = ret.getDenseBlock();
 		final int nCol = ret.getNumColumns();
@@ -308,21 +308,19 @@ private static void decompressDenseSingleThread(MatrixBlock ret, List<AColGroup>
 		}
 	}
 
-	// private static void decompressDenseMultiThread(MatrixBlock ret, List<AColGroup> groups, double[] constV, int k,
-	// boolean overlapping) {
-	// final int nRows = ret.getNumRows();
-	// final double eps = getEps(constV);
-	// final int blklen = Math.max(nRows / k, 512);
-	// decompressDenseMultiThread(ret, groups, nRows, blklen, constV, eps, k, overlapping);
-	// }
-
-	protected static void decompressDenseMultiThread(MatrixBlock ret, List<AColGroup> groups, double[] constV,
+	public static void decompressDense(MatrixBlock ret, List<AColGroup> groups, double[] constV,
 		double eps, int k, boolean overlapping) {
 
 		Timing time = new Timing(true);
 		final int nRows = ret.getNumRows();
 		final int blklen = Math.max(nRows / k, 512);
-		decompressDenseMultiThread(ret, groups, nRows, blklen, constV, eps, k, overlapping);
+		if( k > 1)
+			decompressDenseMultiThread(ret, groups, nRows, blklen, constV, eps, k, overlapping);
+		else
+			decompressDenseSingleThread(ret, groups, nRows, blklen, constV, eps, overlapping);
+		
+		ret.recomputeNonZeros(k);
+		
 		if(DMLScript.STATISTICS) {
 			final double t = time.stop();
 			DMLCompressionStatistics.addDecompressTime(t, k);

src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java

@@ -189,15 +189,8 @@ private static MatrixBlock RMM(CompressedMatrixBlock m1, MatrixBlock that, int k
 			constV = mmTemp.isEmpty() ? null : mmTemp.getDenseBlockValues();
 		}
 
-		final Timing time = new Timing(true);
-
 		ret = asyncRet(f);
-		CLALibDecompress.decompressDenseMultiThread(ret, retCg, constV, 0, k, true);
-
-		if(DMLScript.STATISTICS) {
-			final double t = time.stop();
-			DMLCompressionStatistics.addDecompressTime(t, k);
-		}
+		CLALibDecompress.decompressDense(ret, retCg, constV, 0, k, true);
 
 		return ret;
 	}

src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java

@@ -1267,8 +1267,8 @@ public void copy(int rl, int ru, int cl, int cu, FrameBlock src) {
 	 * @param col is the column # from frame data which contains Recode map generated earlier.
 	 * @return map of token and code for every element in the input column of a frame containing Recode map
 	 */
-	public Map<Object, Long> getRecodeMap(int col) {
-		return _coldata[col].getRecodeMap();
+	public Map<Object, Integer> getRecodeMap(int col) {
+		return _coldata[col].getRecodeMap(4);
 	}
 
 	@Override

src/main/java/org/apache/sysds/runtime/frame/data/columns/ABooleanArray.java

@@ -21,6 +21,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
 
 public abstract class ABooleanArray extends Array<Boolean> {
 
@@ -55,13 +56,14 @@ public boolean possiblyContainsNaN() {
 	 * @param value The string array to set from.
 	 */
 	public abstract void setNullsFromString(int rl, int ru, Array<String> value);
-	
+
+
 	@Override
-	protected Map<Boolean, Long> createRecodeMap() {
-		Map<Boolean, Long> map = new HashMap<>();
-		long id = 1;
+	protected Map<Boolean, Integer> createRecodeMap(int estimate, ExecutorService pool) {
+		Map<Boolean, Integer> map = new HashMap<>();
+		int id = 1;
 		for(int i = 0; i < size() && id <= 2; i++) {
-			Long v = map.putIfAbsent(get(i), id);
+			Integer v = map.putIfAbsent(get(i), id);
 			if(v == null)
 				id++;
 		}

src/main/java/org/apache/sysds/runtime/frame/data/columns/ACompressedArray.java

@@ -59,11 +59,6 @@ public void setFromOtherType(int rl, int ru, Array<?> value) {
 		throw new DMLCompressionException("Invalid to set value in CompressedArray");
 	}
 
-	@Override
-	public void set(int rl, int ru, Array<T> value, int rlSrc) {
-		throw new DMLCompressionException("Invalid to set value in CompressedArray");
-	}
-
 	@Override
 	public void setNz(int rl, int ru, Array<T> value) {
 		throw new DMLCompressionException("Invalid to set value in CompressedArray");