Skip to content

Commit afee50f

Browse files
committed
[MINOR] Add DataTypes on memory estimates for HOPS
This commit changes the memory estimates of many of the HOPS, to include the datatype, Indicating if the output is a frame or a Matrix. It is included because I had to make one modification to a unary Op in BWARE, and in general, it does not hurt to add to many of the Op Types (even if they do not return frame types at the moment)
1 parent 54c8696 commit afee50f

File tree

7 files changed

+16
-8
lines changed

7 files changed

+16
-8
lines changed

src/main/java/org/apache/sysds/hops/BinaryOp.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -562,7 +562,7 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
562562
else //e.g., for append,pow or after inference
563563
sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
564564

565-
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
565+
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
566566
}
567567
return ret;
568568
}

src/main/java/org/apache/sysds/hops/DataGenOp.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,11 +219,11 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
219219
else {
220220
//sparsity-aware estimation (dependent on sparse generation approach); for pure dense generation
221221
//we would need to disable sparsity-awareness and estimate via sparsity=1.0
222-
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, _sparsity);
222+
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, _sparsity, getDataType());
223223
}
224224
}
225225
else {
226-
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, 1.0);
226+
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, 1.0, getDataType());
227227
}
228228

229229
return ret;

src/main/java/org/apache/sysds/hops/DataOp.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -390,7 +390,7 @@ else if(dt == DataType.FRAME) {
390390
|| _op == OpOpData.TRANSIENTREAD )
391391
{
392392
double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
393-
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
393+
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
394394
}
395395
// output memory estimate is not required for "write" nodes (just input)
396396
}

src/main/java/org/apache/sysds/hops/IndexingOp.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
204204
{
205205
// only dense right indexing supported on GPU
206206
double sparsity = isGPUEnabled() ? 1.0 : OptimizerUtils.getSparsity(dim1, dim2, nnz);
207-
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
207+
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
208208
}
209209

210210
@Override

src/main/java/org/apache/sysds/hops/OptimizerUtils.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.log4j.Level;
2727
import org.apache.log4j.Logger;
2828
import org.apache.sysds.api.DMLScript;
29+
import org.apache.sysds.common.Types.DataType;
2930
import org.apache.sysds.common.Types.ExecMode;
3031
import org.apache.sysds.common.Types.ExecType;
3132
import org.apache.sysds.common.Types.FileFormat;
@@ -63,8 +64,8 @@
6364
import org.apache.sysds.runtime.meta.MatrixCharacteristics;
6465
import org.apache.sysds.runtime.util.IndexRange;
6566
import org.apache.sysds.runtime.util.UtilFunctions;
66-
import org.apache.sysds.utils.stats.InfrastructureAnalyzer;
6767
import org.apache.sysds.utils.MemoryEstimates;
68+
import org.apache.sysds.utils.stats.InfrastructureAnalyzer;
6869

6970
public class OptimizerUtils
7071
{
@@ -822,6 +823,13 @@ public static long estimateSizeExactSparsity(long nrows, long ncols, double sp)
822823
return MatrixBlock.estimateSizeInMemory(nrows,ncols,sp);
823824
}
824825

826+
public static long estimateSizeExactSparsity(long nrows, long ncols, double sp, DataType dt){
827+
if(dt == DataType.FRAME)
828+
return estimateSizeExactFrame(nrows, ncols);
829+
else
830+
return estimateSizeExactSparsity(nrows, ncols, sp, dt);
831+
}
832+
825833
/**
826834
* Estimates the footprint (in bytes) for a partitioned in-memory representation of a
827835
* matrix with the given matrix characteristics

src/main/java/org/apache/sysds/hops/ReorgOp.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ public void computeMemEstimate(MemoTable memo){
232232
protected double computeOutputMemEstimate( long dim1, long dim2, long nnz ) {
233233
//no dedicated mem estimation per op type, because always propagated via refreshSizeInformation
234234
double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
235-
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
235+
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
236236
}
237237

238238
@Override

src/main/java/org/apache/sysds/hops/UnaryOp.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -366,7 +366,7 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
366366
} else {
367367
sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
368368
}
369-
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
369+
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
370370
}
371371

372372
@Override

0 commit comments

Comments
 (0)