[SYSTEMDS-3819] New sliceLineExtract builtin function

This new sliceLineExtract builtin functions allows to take the output
of sliceLine and extract the rows from X and e which belong to the top
k2 <= k slices.

Author: mboehm7

scripts/builtin/sliceLineDebug.dml

@@ -26,7 +26,7 @@
 # INPUT:
 # ------------------------------------------------------------------------------
 # TK      top-k slices (k x ncol(X) if successful)
-# TKC     score, size, error of slices (k x 3)
+# TKC     score, total/max error, size of slices (k x 4)
 # tfmeta  transformencode meta data
 # tfspec  transform specification
 # ------------------------------------------------------------------------------

scripts/builtin/sliceLineExtract.dml

@@ -0,0 +1,60 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# This builtin function takes the outputs of SliceLine and allows
+#
+#
+# INPUT:
+# ------------------------------------------------------------------------------
+# X       Feature matrix in recoded/binned representation
+# e       Error vector of trained model
+# TK      top-k slices (k x ncol(X) if successful)
+# TKC     score, total/max error, size of slices (k x 4)
+# k2      fist k2 slices to extract with k2 <= k
+# ------------------------------------------------------------------------------
+#
+# OUTPUT:
+# ------------------------------------------------------------------------------
+# Xtk     Selected rows from X which belong to k2 top slices
+# etk     Selected rows from e which belong to k2 top slices
+# ------------------------------------------------------------------------------
+
+m_sliceLineExtract = function(Matrix[Double] X, Matrix[Double] e,
+  Matrix[Double] TK, Matrix[Double] TKC, Integer k2 = -1)
+  return(Matrix[Double] Xtk, Matrix[Double] etk)
+{
+  # check valid parameters
+  if( k2 > nrow(TK) )
+    stop("sliceLineExtract: invalid number of slices to extract: "+k2+" > "+nrow(TK)).
+  if( k2 <= 0 )
+    k2 = nrow(TK);
+
+  # extract first k2 slices from X and e
+  I = matrix(0, k2, nrow(X));
+  parfor(i in 1:k2) {
+    I[i,] = t(rowSums(X == TK[i,]) == sum(TK[i,]))
+  }
+  I = t(colSums(I)); #union
+
+  Xtk = removeEmpty(target=X, margin="rows", select=I);
+  etk = removeEmpty(target=e, margin="rows", select=I);
+}
+

src/main/java/org/apache/sysds/common/Builtins.java

@@ -312,6 +312,7 @@ public enum Builtins {
 	SLICEFINDER("slicefinder", true), //TODO remove
 	SLICELINE("sliceLine", true),
 	SLICELINE_DEBUG("sliceLineDebug", true),
+	SLICELINE_EXTRACT("sliceLineExtract", true),
 	SKEWNESS("skewness", true),
 	SMAPE("smape", true),
 	SMOTE("smote", true),

src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSliceLineRealDataTest.java

@@ -39,7 +39,7 @@ public class BuiltinSliceLineRealDataTest extends AutomatedTestBase {
 	@Override
 	public void setUp() {
 		for(int i=1; i<=1; i++)
-			addTestConfiguration(TEST_NAME+i, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"R"}));
+			addTestConfiguration(TEST_NAME+i, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"R","V"}));
 	}
 
 	@Test
@@ -55,12 +55,14 @@ private void runSliceLine(int test, String data, String tfspec, double minAcc, E
 			String HOME = SCRIPT_DIR + TEST_DIR;
 			fullDMLScriptName = HOME + TEST_NAME + ".dml";
 			programArgs = new String[] {"-stats",
-				"-args", data, tfspec, output("R")};
+				"-args", data, tfspec, output("R"), output("V")};
 
 			runTest(true, false, null, -1);
 
 			double acc = readDMLMatrixFromOutputDir("R").get(new CellIndex(1,1));
+			double val = readDMLMatrixFromOutputDir("V").get(new CellIndex(1,1));
 			Assert.assertTrue(acc >= minAcc);
+			Assert.assertTrue(val >= 0.99);
 			Assert.assertEquals(0, Statistics.getNoOfExecutedSPInst());
 		}
 		finally {

src/test/scripts/functions/builtin/sliceLineRealData.dml

@@ -45,9 +45,14 @@ acc = lmPredictStats(yhat, y, TRUE);
 e = (y-yhat)^2;
 
 # model debugging via sliceline
-[TK,TKC,D] = slicefinder(X=X, e=e, k=4, alpha=0.95, minSup=32, tpBlksz=16, verbose=TRUE)
+[TK,TKC,D] = sliceLine(X=X, e=e, k=4, alpha=0.95, minSup=32, tpBlksz=16, verbose=TRUE)
 tfspec2 = "{ ids:true, recode:[1,2,5], bin:[{id:3, method:equi-width, numbins:10},{id:4, method:equi-width, numbins:10}]}"
 XYZ = sliceLineDebug(TK=TK, TKC=TKC, tfmeta=meta, tfspec=tfspec2)
+[Xtk,etk] = sliceLineExtract(X=X, e=e, TK=TK, TKC=TKC, k2=3);
 
 acc = acc[3,1];
+val = as.matrix((sum(TKC[1,4]) >= nrow(Xtk)) & (nrow(Xtk) == nrow(etk)))
+
 write(acc, $3);
+write(val, $4);
+