Skip to content

Commit 1f17b56

Browse files
committed
[MINOR] Add DataTypes on memory estimates for HOPS
This commit changes the memory estimates of many of the HOPS, to include the datatype, Indicating if the output is a frame or a Matrix. It is included because I had to make one modification to a unary Op in BWARE, and in general, it does not hurt to add to many of the Op Types (even if they do not return frame types at the moment) Closes #2221
1 parent a0c7a1f commit 1f17b56

File tree

7 files changed

+16
-8
lines changed

7 files changed

+16
-8
lines changed

src/main/java/org/apache/sysds/hops/BinaryOp.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -578,7 +578,7 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
578578
else //e.g., for append,pow or after inference
579579
sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
580580

581-
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
581+
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
582582
}
583583
return ret;
584584
}

src/main/java/org/apache/sysds/hops/DataGenOp.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,11 +219,11 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
219219
else {
220220
//sparsity-aware estimation (dependent on sparse generation approach); for pure dense generation
221221
//we would need to disable sparsity-awareness and estimate via sparsity=1.0
222-
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, _sparsity);
222+
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, _sparsity, getDataType());
223223
}
224224
}
225225
else {
226-
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, 1.0);
226+
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, 1.0, getDataType());
227227
}
228228

229229
return ret;

src/main/java/org/apache/sysds/hops/DataOp.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -390,7 +390,7 @@ else if(dt == DataType.FRAME) {
390390
|| _op == OpOpData.TRANSIENTREAD )
391391
{
392392
double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
393-
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
393+
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
394394
}
395395
// output memory estimate is not required for "write" nodes (just input)
396396
}

src/main/java/org/apache/sysds/hops/IndexingOp.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
204204
{
205205
// only dense right indexing supported on GPU
206206
double sparsity = isGPUEnabled() ? 1.0 : OptimizerUtils.getSparsity(dim1, dim2, nnz);
207-
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
207+
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
208208
}
209209

210210
@Override

src/main/java/org/apache/sysds/hops/OptimizerUtils.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.log4j.Level;
2727
import org.apache.log4j.Logger;
2828
import org.apache.sysds.api.DMLScript;
29+
import org.apache.sysds.common.Types.DataType;
2930
import org.apache.sysds.common.Types.ExecMode;
3031
import org.apache.sysds.common.Types.ExecType;
3132
import org.apache.sysds.common.Types.FileFormat;
@@ -63,8 +64,8 @@
6364
import org.apache.sysds.runtime.meta.MatrixCharacteristics;
6465
import org.apache.sysds.runtime.util.IndexRange;
6566
import org.apache.sysds.runtime.util.UtilFunctions;
66-
import org.apache.sysds.utils.stats.InfrastructureAnalyzer;
6767
import org.apache.sysds.utils.MemoryEstimates;
68+
import org.apache.sysds.utils.stats.InfrastructureAnalyzer;
6869

6970
public class OptimizerUtils
7071
{
@@ -822,6 +823,13 @@ public static long estimateSizeExactSparsity(long nrows, long ncols, double sp)
822823
return MatrixBlock.estimateSizeInMemory(nrows,ncols,sp);
823824
}
824825

826+
public static long estimateSizeExactSparsity(long nrows, long ncols, double sp, DataType dt){
827+
if(dt == DataType.FRAME)
828+
return estimateSizeExactFrame(nrows, ncols);
829+
else
830+
return estimateSizeExactSparsity(nrows, ncols, sp);
831+
}
832+
825833
/**
826834
* Estimates the footprint (in bytes) for a partitioned in-memory representation of a
827835
* matrix with the given matrix characteristics

src/main/java/org/apache/sysds/hops/ReorgOp.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ public void computeMemEstimate(MemoTable memo){
232232
protected double computeOutputMemEstimate( long dim1, long dim2, long nnz ) {
233233
//no dedicated mem estimation per op type, because always propagated via refreshSizeInformation
234234
double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
235-
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
235+
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
236236
}
237237

238238
@Override

src/main/java/org/apache/sysds/hops/UnaryOp.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -366,7 +366,7 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
366366
} else {
367367
sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
368368
}
369-
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
369+
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
370370
}
371371

372372
@Override

0 commit comments

Comments
 (0)