Skip to content

Commit 545b2b6

Browse files
BaunsgaardBaunsgaard
committed
more
1 parent 5124ca5 commit 545b2b6

File tree

3 files changed

+185
-30
lines changed

3 files changed

+185
-30
lines changed

‎src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ public IDictionary binOpRightAndAppend(BinaryOperator op, double[] v, IColIndex
289289
final int lenV = colIndexes.size();
290290
for(int i = 0; i < _values.length; i++)
291291
retVals[i] = fn.execute(_values[i], v[colIndexes.get(i % lenV)]);
292-
for(int i = _values.length; i < _values.length; i++)
292+
for(int i = _values.length; i < retVals.length; i++)
293293
retVals[i] = fn.execute(0, v[colIndexes.get(i % lenV)]);
294294

295295
return create(retVals);
@@ -329,7 +329,7 @@ public IDictionary binOpLeftAndAppend(BinaryOperator op, double[] v, IColIndex c
329329
final int lenV = colIndexes.size();
330330
for(int i = 0; i < _values.length; i++)
331331
retVals[i] = fn.execute(v[colIndexes.get(i % lenV)], _values[i]);
332-
for(int i = _values.length; i < _values.length; i++)
332+
for(int i = _values.length; i < retVals.length; i++)
333333
retVals[i] = fn.execute(v[colIndexes.get(i % lenV)], 0);
334334

335335
return create(retVals);

‎src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java‎

Lines changed: 83 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -436,28 +436,45 @@ public IDictionary applyScalarOp(ScalarOperator op) {
436436
public IDictionary applyScalarOpAndAppend(ScalarOperator op, double v0, int nCol) {
437437
// guaranteed to be densifying since this only is called if op(0) is != 0
438438
// set the entire output to v0.
439-
final MatrixBlock ret = new MatrixBlock(_data.getNumRows() + 1, _data.getNumColumns(), v0);
439+
final MatrixBlock ret = new MatrixBlock(_data.getNumRows() + 1, _data.getNumColumns(), false);
440+
ret.allocateDenseBlock();
440441
final double[] retV = ret.getDenseBlockValues();
441442

442443
if(_data.isInSparseFormat()) {
443444
final int nRow = _data.getNumRows();
444445
final SparseBlock sb = _data.getSparseBlock();
446+
final double v0r = op.executeScalar(0.0d);
445447
for(int i = 0; i < nRow; i++) {
446-
if(sb.isEmpty(i))
447-
continue;
448-
449-
final int apos = sb.pos(i);
450-
final int alen = sb.size(i) + apos;
451-
final int[] aix = sb.indexes(i);
452-
final double[] avals = sb.values(i);
453-
for(int k = apos; k < alen; k++)
454-
retV[i * nCol + aix[k]] = op.executeScalar(avals[i]);
448+
final int off = i * nCol;
449+
if(sb.isEmpty(i)) {
450+
for(int j = 0; j < nCol; j++)
451+
retV[off + j] = v0r;
452+
}
453+
else {
454+
final int apos = sb.pos(i);
455+
final int alen = sb.size(i) + apos;
456+
final int[] aix = sb.indexes(i);
457+
final double[] avals = sb.values(i);
458+
int k = apos;
459+
int j = 0;
460+
for(; j < nCol && k < alen; j++) {
461+
double v = aix[k] == j ? avals[k++] : 0;
462+
retV[off + j] = op.executeScalar(v);
463+
}
464+
for(; j < nCol; j++) {
465+
retV[off + j] = v0r;
466+
}
467+
}
455468
}
469+
for(int i = nCol * nRow; i < retV.length; i++)
470+
retV[i] = v0;
456471
}
457472
else {
458473
final double[] v = _data.getDenseBlockValues();
459474
for(int i = 0; i < v.length; i++)
460475
retV[i] = op.executeScalar(v[i]);
476+
for(int i = v.length; i < retV.length; i++)
477+
retV[i] = v0;
461478
}
462479

463480
ret.recomputeNonZeros();
@@ -472,14 +489,12 @@ public IDictionary applyUnaryOp(UnaryOperator op) {
472489

473490
@Override
474491
public IDictionary applyUnaryOpAndAppend(UnaryOperator op, double v0, int nCol) {
475-
// guaranteed to be densifying since this only is called if op(0) is != 0
476-
// set the entire output to v0.
477-
final MatrixBlock ret = new MatrixBlock(_data.getNumRows() + 1, _data.getNumColumns(), v0);
478-
final double[] retV = ret.getDenseBlockValues();
479-
480-
if(_data.isInSparseFormat()) {
481-
final int nRow = _data.getNumRows();
492+
final int nRow = _data.getNumRows();
493+
final MatrixBlock ret = new MatrixBlock(nRow + 1, nCol, op.sparseSafe && _data.isInSparseFormat());
494+
if(op.sparseSafe && _data.isInSparseFormat()) {
495+
ret.allocateSparseRowsBlock();
482496
final SparseBlock sb = _data.getSparseBlock();
497+
final SparseBlock sbr = ret.getSparseBlock();
483498
for(int i = 0; i < nRow; i++) {
484499
if(sb.isEmpty(i))
485500
continue;
@@ -489,16 +504,56 @@ public IDictionary applyUnaryOpAndAppend(UnaryOperator op, double v0, int nCol)
489504
final int[] aix = sb.indexes(i);
490505
final double[] avals = sb.values(i);
491506
for(int k = apos; k < alen; k++)
492-
retV[i * nCol + aix[k]] = op.fn.execute(avals[i]);
507+
sbr.append(i, aix[k], op.fn.execute(avals[k]));
493508
}
509+
510+
for(int i = 0; i < nCol; i++)
511+
sbr.append(nRow, i, v0);
512+
}
513+
else if(_data.isInSparseFormat()) {
514+
ret.allocateDenseBlock();
515+
final double[] retV = ret.getDenseBlockValues();
516+
final SparseBlock sb = _data.getSparseBlock();
517+
double v0r = op.fn.execute(0);
518+
for(int i = 0; i < nRow; i++) {
519+
final int off = i * nCol;
520+
if(sb.isEmpty(i)) {
521+
for(int j = 0; j < nCol; j++)
522+
retV[off + j] = v0r;
523+
}
524+
else {
525+
526+
final int apos = sb.pos(i);
527+
final int alen = sb.size(i) + apos;
528+
final int[] aix = sb.indexes(i);
529+
final double[] avals = sb.values(i);
530+
int k = apos;
531+
int j = 0;
532+
for(; j < nCol && k < alen; j++) {
533+
double v = aix[k] == j ? avals[k++] : 0;
534+
retV[off + j] = op.fn.execute(v);
535+
}
536+
for(; j < nCol; j++) {
537+
retV[off + j] = v0r;
538+
}
539+
}
540+
541+
}
542+
for(int i = nRow * nCol; i < retV.length; i++)
543+
retV[i] = v0;
494544
}
495545
else {
546+
ret.allocateDenseBlock();
547+
final double[] retV = ret.getDenseBlockValues();
496548
final double[] v = _data.getDenseBlockValues();
497549
for(int i = 0; i < v.length; i++)
498550
retV[i] = op.fn.execute(v[i]);
551+
for(int i = nRow * nCol; i < retV.length; i++)
552+
retV[i] = v0;
499553
}
500554

501555
ret.recomputeNonZeros();
556+
ret.examSparsity();
502557
return MatrixBlockDictionary.create(ret);
503558
}
504559

@@ -655,7 +710,7 @@ public IDictionary binOpLeftAndAppend(BinaryOperator op, double[] v, IColIndex c
655710
for(int i = 0; i < nRow; i++) {
656711
if(sb.isEmpty(i))
657712
for(int j = 0; j < nCol; j++)
658-
retV[off++] = op.fn.execute(v[j], 0);
713+
retV[off++] = op.fn.execute(v[colIndexes.get(j)], 0);
659714
else {
660715
final int apos = sb.pos(i);
661716
final int alen = sb.size(i) + apos;
@@ -664,24 +719,24 @@ public IDictionary binOpLeftAndAppend(BinaryOperator op, double[] v, IColIndex c
664719
int j = 0;
665720
for(int k = apos; j < nCol && k < alen; j++) {
666721
final double vx = aix[k] == j ? avals[k++] : 0;
667-
retV[off++] = op.fn.execute(v[j], vx);
722+
retV[off++] = op.fn.execute(v[colIndexes.get(j)], vx);
668723
}
669724
for(; j < nCol; j++)
670-
retV[off++] = op.fn.execute(v[j], 0);
725+
retV[off++] = op.fn.execute(v[colIndexes.get(j)], 0);
671726
}
672727
}
673728
}
674729
else {
675730
final double[] values = _data.getDenseBlockValues();
676731
for(int i = 0; i < nRow; i++) {
677732
for(int j = 0; j < nCol; j++) {
678-
retV[off] = op.fn.execute(v[j], values[off]);
733+
retV[off] = op.fn.execute(v[colIndexes.get(j)], values[off]);
679734
off++;
680735
}
681736
}
682737
}
683738
for(int j = 0; j < nCol; j++) {
684-
retV[off] = op.fn.execute(v[j], 0);
739+
retV[off] = op.fn.execute(v[colIndexes.get(j)], 0);
685740
off++;
686741
}
687742

@@ -769,7 +824,7 @@ public IDictionary binOpRightAndAppend(BinaryOperator op, double[] v, IColIndex
769824
for(int i = 0; i < nRow; i++) {
770825
if(sb.isEmpty(i))
771826
for(int j = 0; j < nCol; j++)
772-
retV[off++] = op.fn.execute(0, v[j]);
827+
retV[off++] = op.fn.execute(0, v[colIndexes.get(j)]);
773828
else {
774829
final int apos = sb.pos(i);
775830
final int alen = sb.size(i) + apos;
@@ -778,24 +833,24 @@ public IDictionary binOpRightAndAppend(BinaryOperator op, double[] v, IColIndex
778833
int j = 0;
779834
for(int k = apos; j < nCol && k < alen; j++) {
780835
final double vx = aix[k] == j ? avals[k++] : 0;
781-
retV[off++] = op.fn.execute(vx, v[j]);
836+
retV[off++] = op.fn.execute(vx, v[colIndexes.get(j)]);
782837
}
783838
for(; j < nCol; j++)
784-
retV[off++] = op.fn.execute(0, v[j]);
839+
retV[off++] = op.fn.execute(0, v[colIndexes.get(j)]);
785840
}
786841
}
787842
}
788843
else {
789844
final double[] values = _data.getDenseBlockValues();
790845
for(int i = 0; i < nRow; i++) {
791846
for(int j = 0; j < nCol; j++) {
792-
retV[off] = op.fn.execute(values[off], v[j]);
847+
retV[off] = op.fn.execute(values[off], v[colIndexes.get(j)]);
793848
off++;
794849
}
795850
}
796851
}
797852
for(int j = 0; j < nCol; j++) {
798-
retV[off] = op.fn.execute(0, v[j]);
853+
retV[off] = op.fn.execute(0, v[colIndexes.get(j)]);
799854
off++;
800855
}
801856

‎src/test/java/org/apache/sysds/test/component/compress/dictionary/DictionaryTests.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,13 @@
5858
import org.apache.sysds.runtime.functionobjects.Builtin.BuiltinCode;
5959
import org.apache.sysds.runtime.functionobjects.Divide;
6060
import org.apache.sysds.runtime.functionobjects.Minus;
61+
import org.apache.sysds.runtime.functionobjects.Multiply;
62+
import org.apache.sysds.runtime.functionobjects.Plus;
6163
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
6264
import org.apache.sysds.runtime.matrix.operators.BinaryOperator;
65+
import org.apache.sysds.runtime.matrix.operators.RightScalarOperator;
66+
import org.apache.sysds.runtime.matrix.operators.ScalarOperator;
67+
import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
6368
import org.apache.sysds.test.TestUtils;
6469
import org.junit.Test;
6570
import org.junit.runner.RunWith;
@@ -847,6 +852,16 @@ private void binOp(BinaryOperator op, double[] vals, IColIndex cols) {
847852
bb = b.binOpLeft(op, vals, cols);
848853
compare(aa, bb, nRow, nCol);
849854

855+
double[] app = TestUtils.generateTestVector(nCol, 0, 10, 1.0, 33);
856+
857+
aa = a.binOpLeftAndAppend(op, app, cols);
858+
bb = b.binOpLeftAndAppend(op, app, cols);
859+
compare(aa, bb, nRow + 1, nCol);
860+
861+
aa = a.binOpRightAndAppend(op, app, cols);
862+
bb = b.binOpRightAndAppend(op, app, cols);
863+
compare(aa, bb, nRow + 1, nCol);
864+
850865
}
851866
catch(Exception e) {
852867
e.printStackTrace();
@@ -1513,7 +1528,13 @@ public void colProduct() {
15131528
a.colProduct(aa, counts, cols);
15141529
double[] bb = new double[nCol + 3];
15151530
b.colProduct(bb, counts, cols);
1531+
TestUtils.compareMatrices(aa, bb, 0.001);
15161532

1533+
double[] ref = TestUtils.generateTestVector(nCol, 0, 10, 1, 3215555);
1534+
aa = new double[nCol + 3];
1535+
a.colProductWithReference(aa, counts, cols, ref);
1536+
bb = new double[nCol + 3];
1537+
b.colProductWithReference(bb, counts, cols, ref);
15171538
TestUtils.compareMatrices(aa, bb, 0.001);
15181539
}
15191540

@@ -1531,6 +1552,7 @@ public void productWithDefault(double retV, double[] def) {
15311552

15321553
TestUtils.compareMatricesBitAvgDistance(//
15331554
aRet, bRet, 10, 10, "Not Equivalent values from product");
1555+
15341556
}
15351557

15361558
private static int[] getCounts(int nRows, int seed) {
@@ -1671,4 +1693,82 @@ public void testClone() {
16711693
assertFalse(cb == b);
16721694
assertEquals(cb, b);
16731695
}
1696+
1697+
@Test
1698+
public void round() {
1699+
unaryOp(new UnaryOperator(Builtin.getBuiltinFnObject(BuiltinCode.ROUND)));
1700+
}
1701+
1702+
@Test
1703+
public void mod() {
1704+
unaryOp(new UnaryOperator(Builtin.getBuiltinFnObject(BuiltinCode.ABS)));
1705+
}
1706+
1707+
@Test
1708+
public void floor() {
1709+
unaryOp(new UnaryOperator(Builtin.getBuiltinFnObject(BuiltinCode.FLOOR)));
1710+
}
1711+
1712+
@Test
1713+
public void sin() {
1714+
unaryOp(new UnaryOperator(Builtin.getBuiltinFnObject(BuiltinCode.SIN)));
1715+
}
1716+
1717+
@Test
1718+
public void cos() {
1719+
unaryOp(new UnaryOperator(Builtin.getBuiltinFnObject(BuiltinCode.COS)));
1720+
}
1721+
1722+
public void unaryOp(UnaryOperator op) {
1723+
IDictionary aa;
1724+
IDictionary bb;
1725+
1726+
aa = a.applyUnaryOp(op);
1727+
bb = b.applyUnaryOp(op);
1728+
compare(aa, bb, nCol);
1729+
1730+
aa = a.applyUnaryOpAndAppend(op, 32, nCol);
1731+
bb = b.applyUnaryOpAndAppend(op, 32, nCol);
1732+
compare(aa, bb, nCol);
1733+
1734+
double[] ref1 = TestUtils.generateTestVector(nCol, 0, 10, 1, 333);
1735+
double[] ref2 = TestUtils.generateTestVector(nCol, 0, 10, 1, 32);
1736+
aa = a.applyUnaryOpWithReference(op, ref1, ref2);
1737+
bb = b.applyUnaryOpWithReference(op, ref1, ref2);
1738+
compare(aa, bb, nCol);
1739+
}
1740+
1741+
@Test
1742+
public void plus() {
1743+
scalarOp(new RightScalarOperator(Plus.getPlusFnObject(), 1));
1744+
}
1745+
1746+
@Test
1747+
public void mult() {
1748+
scalarOp(new RightScalarOperator(Multiply.getMultiplyFnObject(), 1));
1749+
}
1750+
1751+
@Test
1752+
public void div() {
1753+
scalarOp(new RightScalarOperator(Divide.getDivideFnObject(), 1));
1754+
}
1755+
1756+
public void scalarOp(ScalarOperator op) {
1757+
IDictionary aa;
1758+
IDictionary bb;
1759+
1760+
aa = a.applyScalarOp(op);
1761+
bb = b.applyScalarOp(op);
1762+
compare(aa, bb, nCol);
1763+
1764+
aa = a.applyScalarOpAndAppend(op, 32, nCol);
1765+
bb = b.applyScalarOpAndAppend(op, 32, nCol);
1766+
compare(aa, bb, nCol);
1767+
1768+
double[] ref1 = TestUtils.generateTestVector(nCol, 0, 10, 1, 3213);
1769+
double[] ref2 = TestUtils.generateTestVector(nCol, 0, 10, 1, 23232);
1770+
aa = a.applyScalarOpWithReference(op, ref1, ref2);
1771+
bb = b.applyScalarOpWithReference(op, ref1, ref2);
1772+
compare(aa, bb, nCol);
1773+
}
16741774
}

0 commit comments

Comments
 (0)