comment on parallelization threshold

Author: Baunsgaard

src/main/java/org/apache/sysds/runtime/frame/data/columns/Array.java

@@ -50,6 +50,7 @@
 public abstract class Array<T> implements Writable {
 	protected static final Log LOG = LogFactory.getLog(Array.class.getName());
 
+	/** Parallelization threshold for parallelizing vector operations */
 	public static int ROW_PARALLELIZATION_THRESHOLD = 10000;
 
 	/** A soft reference to a memorization of this arrays mapping, used in transformEncode */
@@ -176,7 +177,7 @@ protected Map<T, Integer> createRecodeMap(int estimate, ExecutorService pool)
 	private Map<T, Integer> parallelCreateRecodeMap(int estimate, ExecutorService pool, final int s, int k)
 		throws InterruptedException, ExecutionException {
 
-		final int blk = Math.max(ROW_PARALLELIZATION_THRESHOLD, (s + k) / k);
+		final int blk = Math.max(ROW_PARALLELIZATION_THRESHOLD / 2, (s + k) / k);
 		final List<Future<Map<T, Integer>>> tasks = new ArrayList<>();
 		for(int i = blk; i < s; i += blk) { // start at blk for the other threads
 			final int start = i;
@@ -343,7 +344,7 @@ public double getAsNaNDouble(int i) {
 	 * @param ru    row upper (inclusive)
 	 * @param value value array to take values from (same type) offset by rl.
 	 */
-	public final void set(int rl, int ru, Array<T> value){
+	public final void set(int rl, int ru, Array<T> value) {
 		set(rl, ru, value, 0);
 	}
 

src/main/java/org/apache/sysds/runtime/transform/encode/CompressedEncode.java

@@ -247,9 +247,9 @@ private AMapToData binEncode(Array<?> a, ColumnEncoderBin b, boolean nulls)
 	private void BinEncodeParallel(Array<?> a, ColumnEncoderBin b, boolean nulls, final AMapToData m, final int rlen)
 		throws InterruptedException, ExecutionException {
 		final List<Future<?>> tasks = new ArrayList<>();
-		final int blockSize = Math.max(ROW_PARALLELIZATION_THRESHOLD, rlen + k / k);
+		final int blockSize = Math.max(ROW_PARALLELIZATION_THRESHOLD / 2, rlen + k / k);
 		final ExecutorService pool = CommonThreadPool.get(k);
-		try{
+		try {
 
 			for(int i = 0; i < rlen; i += blockSize) {
 				final int start = i;
@@ -264,7 +264,7 @@ private void BinEncodeParallel(Array<?> a, ColumnEncoderBin b, boolean nulls, fi
 			for(Future<?> t : tasks)
 				t.get();
 		}
-		finally{
+		finally {
 			pool.shutdown();
 		}
 	}
@@ -436,7 +436,7 @@ private <T> AMapToData createMappingAMapToData(Array<T> a, Map<T, Integer> map,
 
 	private <T> AMapToData CreateMappingParallel(Array<T> a, Map<T, Integer> map, boolean containsNull, final int si,
 		final int nRow, final AMapToData m) throws InterruptedException, ExecutionException {
-		final int blkz = Math.max(ROW_PARALLELIZATION_THRESHOLD, (nRow + k) / k);
+		final int blkz = Math.max(ROW_PARALLELIZATION_THRESHOLD / 2, (nRow + k) / k);
 
 		List<Future<?>> tasks = new ArrayList<>();
 		// make a thread local pool.