[SYSTEMDS-3778] New determinant function, kernels, rewrites

DIA WiSe 24/25 project
Closes #2196.

Co-authored-by: Lou Frizzi Maria Wagner <[email protected]>
Co-authored-by: Laurits	Sartorius <[email protected]>

Author: 3 people

src/main/java/org/apache/sysds/common/Builtins.java

@@ -113,6 +113,7 @@ public enum Builtins {
 	DECISIONTREEPREDICT("decisionTreePredict", true),
 	DECOMPRESS("decompress", false),
 	DEEPWALK("deepWalk", true),
+	DET("det", false),
 	DETECTSCHEMA("detectSchema", false),
 	DENIALCONSTRAINTS("denialConstraints", true),
 	DIFFERENCESTATISTICS("differenceStatistics", true),

src/main/java/org/apache/sysds/common/Opcodes.java

@@ -165,6 +165,7 @@ public enum Opcodes {
 	STOP("stop", CPType.Unary),
 	INVERSE("inverse", CPType.Unary),
 	CHOLESKY("cholesky", CPType.Unary),
+	DET("det", CPType.Unary),
 	SPROP("sprop", CPType.Unary),
 	SIGMOID("sigmoid", CPType.Unary),
 	TYPEOF("typeOf", CPType.Unary),

src/main/java/org/apache/sysds/common/Types.java

@@ -539,7 +539,7 @@ public enum OpOp1 {
 		CAST_AS_FRAME, CAST_AS_LIST, CAST_AS_MATRIX, CAST_AS_SCALAR,
 		CAST_AS_BOOLEAN, CAST_AS_DOUBLE, CAST_AS_INT,
 		CEIL, CHOLESKY, COS, COSH, CUMMAX, CUMMIN, CUMPROD, CUMSUM,
-		CUMSUMPROD, DETECTSCHEMA, COLNAMES, EIGEN, EXISTS, EXP, FLOOR, INVERSE,
+		CUMSUMPROD, DET, DETECTSCHEMA, COLNAMES, EIGEN, EXISTS, EXP, FLOOR, INVERSE,
 		IQM, ISNA, ISNAN, ISINF, LENGTH, LINEAGE, LOG, NCOL, NOT, NROW,
 		MEDIAN, PREFETCH, PRINT, ROUND, SIN, SINH, SIGN, SOFTMAX, SQRT, STOP, _EVICT,
 		SVD, TAN, TANH, TYPEOF, TRIGREMOTE, SQRT_MATRIX_JAVA,
@@ -558,6 +558,7 @@ public enum OpOp1 {
 
 		public boolean isScalarOutput() {
 			return this == CAST_AS_SCALAR
+				|| this == DET
 				|| this == NROW || this == NCOL
 				|| this == LENGTH || this == EXISTS
 				|| this == IQM || this == LINEAGE
@@ -579,6 +580,7 @@ public String toString() {
 				case CUMPROD:         return Opcodes.UCUMM.toString();
 				case CUMSUM:          return Opcodes.UCUMKP.toString();
 				case CUMSUMPROD:      return Opcodes.UCUMKPM.toString();
+				case DET:             return Opcodes.DET.toString();
 				case DETECTSCHEMA:    return Opcodes.DETECTSCHEMA.toString();
 				case MULT2:           return Opcodes.MULT2.toString();
 				case NOT:             return Opcodes.NOT.toString();

src/main/java/org/apache/sysds/hops/UnaryOp.java

@@ -512,7 +512,8 @@ && getInput().get(0).getParent().size()==1 ) //unary is only parent
 
 		//ensure cp exec type for single-node operations
 		if( _op == OpOp1.PRINT || _op == OpOp1.ASSERT || _op == OpOp1.STOP || _op == OpOp1.TYPEOF
-			|| _op == OpOp1.INVERSE || _op == OpOp1.EIGEN || _op == OpOp1.CHOLESKY || _op == OpOp1.SVD || _op == OpOp1.SQRT_MATRIX_JAVA
+			|| _op == OpOp1.INVERSE || _op == OpOp1.EIGEN || _op == OpOp1.CHOLESKY || _op == OpOp1.DET
+			||_op == OpOp1.SVD || _op == OpOp1.SQRT_MATRIX_JAVA
 			|| getInput().get(0).getDataType() == DataType.LIST || isMetadataOperation() )
 		{
 			_etype = ExecType.CP;

src/main/java/org/apache/sysds/hops/rewrite/RewriteAlgebraicSimplificationStatic.java

@@ -159,12 +159,15 @@ private void rule_AlgebraicSimplification(Hop hop, boolean descendFirst)
 			if(OptimizerUtils.ALLOW_OPERATOR_FUSION)
 				hi = simplifyMultiBinaryToBinaryOperation(hi);       //e.g., 1-X*Y -> X 1-* Y
 			hi = simplifyDistributiveBinaryOperation(hop, hi, i);//e.g., (X-Y*X) -> (1-Y)*X
+			hi = simplifyTransposeInDetOperation(hop, hi, i);    //e.g., det(t(X)) -> det(X)
 			hi = simplifyBushyBinaryOperation(hop, hi, i);       //e.g., (X*(Y*(Z%*%v))) -> (X*Y)*(Z%*%v)
 			hi = simplifyUnaryAggReorgOperation(hop, hi, i);     //e.g., sum(t(X)) -> sum(X)
 			hi = removeUnnecessaryAggregates(hi);                //e.g., sum(rowSums(X)) -> sum(X)
 			hi = simplifyBinaryMatrixScalarOperation(hop, hi, i);//e.g., as.scalar(X*s) -> as.scalar(X)*s;
 			hi = pushdownUnaryAggTransposeOperation(hop, hi, i); //e.g., colSums(t(X)) -> t(rowSums(X))
 			hi = pushdownCSETransposeScalarOperation(hop, hi, i);//e.g., a=t(X), b=t(X^2) -> a=t(X), b=t(X)^2 for CSE t(X)
+			hi = pushdownDetMultOperation(hop, hi, i);           //e.g., det(X%*%Y) -> det(X)*det(Y)
+			hi = pushdownDetScalarMatrixMultOperation(hop, hi, i);  //e.g., det(lambda*X) -> lambda^nrow(X)*det(X)
 			hi = pushdownSumBinaryMult(hop, hi, i);              //e.g., sum(lambda*X) -> lambda*sum(X)
 			hi = pullupAbs(hop, hi, i);                          //e.g., abs(X)*abs(Y) --> abs(X*Y)
 			hi = simplifyUnaryPPredOperation(hop, hi, i);        //e.g., abs(ppred()) -> ppred(), others: round, ceil, floor
@@ -922,6 +925,29 @@ private static Hop simplifyDistributiveBinaryOperation( Hop parent, Hop hi, int
 		return hi;
 	}
 
+	/**
+	 * det(t(X)) -> det(X)
+	 *
+	 * @param parent parent high-level operator
+	 * @param hi high-level operator
+	 * @param pos position
+	 * @return high-level operator
+	 */
+	private static Hop simplifyTransposeInDetOperation(Hop parent, Hop hi, int pos)
+	{
+		if(HopRewriteUtils.isUnary(hi, OpOp1.DET)
+				&& HopRewriteUtils.isReorg(hi.getInput(0), ReOrgOp.TRANS))
+		{
+			Hop operand = hi.getInput(0).getInput(0);
+			Hop uop = HopRewriteUtils.createUnary(operand, OpOp1.DET);
+			HopRewriteUtils.replaceChildReference(parent, hi, uop, pos);
+
+			LOG.debug("Applied simplifyTransposeInDetOperation.");
+			return uop;
+		}
+		return hi;
+	}
+
 	/**
 	 * t(Z)%*%(X*(Y*(Z%*%v))) -> t(Z)%*%(X*Y)*(Z%*%v)
 	 * t(Z)%*%(X+(Y+(Z%*%v))) -> t(Z)%*%((X+Y)+(Z%*%v))
@@ -1163,6 +1189,65 @@ private static Hop pushdownCSETransposeScalarOperation( Hop parent, Hop hi, int
 		return hi;
 	}
 
+	/**
+	 * det(X%*%Y) -> det(X)*det(Y)
+	 *
+	 * @param parent parent high-level operator
+	 * @param hi high-level operator
+	 * @param pos position
+	 * @return high-level operator
+	 */
+	private static Hop pushdownDetMultOperation(Hop parent, Hop hi, int pos) {
+		if( HopRewriteUtils.isUnary(hi, OpOp1.DET)
+				&& HopRewriteUtils.isMatrixMultiply(hi.getInput(0))
+				&& hi.getInput(0).getInput(0).isMatrix()
+				&& hi.getInput(0).getInput(1).isMatrix())
+		{
+			Hop operand1 = hi.getInput(0).getInput(0);
+			Hop operand2 = hi.getInput(0).getInput(1);
+			Hop uop1 = HopRewriteUtils.createUnary(operand1, OpOp1.DET);
+			Hop uop2 = HopRewriteUtils.createUnary(operand2, OpOp1.DET);
+			Hop bop = HopRewriteUtils.createBinary(uop1, uop2, OpOp2.MULT);
+			HopRewriteUtils.replaceChildReference(parent, hi, bop, pos);
+
+			LOG.debug("Applied pushdownDetMultOperation.");
+			return bop;
+		}
+		return hi;
+	}
+
+	/**
+	 * det(lambda*X) -> lambda^nrow*det(X)
+	 *
+	 * @param parent parent high-level operator
+	 * @param hi high-level operator
+	 * @param pos position
+	 * @return high-level operator
+	 */
+	private static Hop pushdownDetScalarMatrixMultOperation(Hop parent, Hop hi, int pos) {
+		if( HopRewriteUtils.isUnary(hi, OpOp1.DET)
+				&& HopRewriteUtils.isBinary(hi.getInput(0), OpOp2.MULT)
+				&& ((hi.getInput(0).getInput(0).isMatrix() && hi.getInput(0).getInput(1).isScalar())
+					|| (hi.getInput(0).getInput(0).isScalar() && hi.getInput(0).getInput(1).isMatrix())))
+		{
+			Hop operand1 = hi.getInput(0).getInput(0);
+			Hop operand2 = hi.getInput(0).getInput(1);
+
+			Hop lambda = (operand1.isScalar()) ? operand1 : operand2;
+			Hop matrix = (operand1.isMatrix()) ? operand1 : operand2;
+
+			Hop uopDet = HopRewriteUtils.createUnary(matrix, OpOp1.DET);
+			Hop uopNrow = HopRewriteUtils.createUnary(matrix, OpOp1.NROW);
+			Hop bopPow = HopRewriteUtils.createBinary(lambda, uopNrow, OpOp2.POW);
+			Hop bopMult = HopRewriteUtils.createBinary(bopPow, uopDet, OpOp2.MULT);
+			HopRewriteUtils.replaceChildReference(parent, hi, bopMult, pos);
+
+			LOG.debug("Applied pushdownDetScalarMatrixMultOperation.");
+			return bopMult;
+		}
+		return hi;
+	}
+
 	private static Hop pushdownSumBinaryMult(Hop parent, Hop hi, int pos ) {
 		//pattern:  sum(lamda*X) -> lamda*sum(X)
 		if( hi instanceof AggUnaryOp && ((AggUnaryOp)hi).getDirection()==Direction.RowCol

src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java

@@ -1302,6 +1302,17 @@ else if( getOpCode() == Builtins.RBIND ) {
 			output.setBlocksize(id.getBlocksize());
 			output.setValueType(id.getValueType());
 			break;
+		case DET:
+			checkNumParameters(1);
+			checkMatrixParam(getFirstExpr());
+			if ( id.getDim2() == -1 || id.getDim1() != id.getDim2() ) {
+				raiseValidateError("det requires a square matrix as first argument.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
+			}
+			output.setDataType(DataType.SCALAR);
+			output.setDimensions(0, 0);
+			output.setBlocksize(0);
+			output.setValueType(ValueType.FP64);
+			break;
 		case NROW:
 		case NCOL:
 		case LENGTH:

src/main/java/org/apache/sysds/parser/DMLTranslator.java

@@ -2755,6 +2755,7 @@ else if ( in.length == 2 )
 		case SQRT_MATRIX_JAVA:
 		case CHOLESKY:
 		case TYPEOF:
+		case DET:
 		case DETECTSCHEMA:
 		case COLNAMES:
 			currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(),

src/main/java/org/apache/sysds/resource/cost/CPCostUtils.java

@@ -501,6 +501,7 @@ public static long getInstNFLOP(
 					case "cholesky":
 						costs = (1.0 / 3.0) * output.getCellsWithSparsity() * output.getCellsWithSparsity();
 						break;
+					case "det":
 					case "detectschema":
 					case "colnames":
 						throw new RuntimeException("Specific Frame operation with opcode '" + opcode + "' is not supported yet");

src/main/java/org/apache/sysds/runtime/instructions/cp/UnaryMatrixCPInstruction.java

@@ -19,6 +19,7 @@
 
 package org.apache.sysds.runtime.instructions.cp;
 
+import org.apache.sysds.common.Types.ValueType;
 import org.apache.sysds.runtime.controlprogram.caching.CacheableData;
 import org.apache.sysds.runtime.controlprogram.caching.MatrixObject;
 import org.apache.sysds.runtime.controlprogram.context.ExecutionContext;
@@ -57,7 +58,13 @@ public void processInstruction(ExecutionContext ec) {
 		LineageItem lin = (!inObj.hasValidLineage() || !inObj.getCacheLineage().isLeaf() ||
 			CacheableData.isBelowCachingThreshold(retBlock)) ? null : 
 			getCacheLineageItem(inObj.getCacheLineage());
-		ec.setMatrixOutputAndLineage(output, retBlock, lin);
+		if (getOpcode().equals("det")){
+			var temp = ScalarObjectFactory.createScalarObject(ValueType.FP64, retBlock.get(0,0));
+			ec.setVariable(output.getName(), temp);
+		}
+		else {
+			ec.setMatrixOutputAndLineage(output, retBlock, lin);
+		}
 	}
 
 	public LineageItem getCacheLineageItem(LineageItem input) {