Skip to content

Commit 9fb1967

Browse files
phaniarnabphaniarnab
committed
[SYSTEMDS-3850] Pipeline pruning after top-K cleaning
This patch introduces a pruning technique on the cleaning pipeline returned by the top-K cleaning. We identify a smaller yet equally effective subset of primitives for all top-performing pipelines, which optimizes their scoring performance. Closes #2251
1 parent 48fd91c commit 9fb1967

File tree

7 files changed

+79
-35
lines changed

7 files changed

+79
-35
lines changed

‎scripts/builtin/bandit.dml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,7 @@ run_with_hyperparam = function(Frame[Unknown] ph_pip, Integer r_i = 1, Matrix[Do
311311
{
312312
[eXtrain, eYtrain, eXtest, eYtest, Tr, hpForPruning, changesByOp, changesByPip] = executePipeline(pipeline=op,
313313
Xtrain=X, Ytrain=Y, Xtest=Xtest, Ytest=Ytest, metaList=metaList2, hyperParameters=hp_matrix, hpForPruning=hpForPruning,
314-
changesByOp=changesByOp, flagsCount=no_of_flag_vars, test=TRUE, verbose=FALSE)
314+
changesByOp=changesByOp, flagsCount=no_of_flag_vars, test=TRUE, verbose=FALSE, startInd=1, endInd=ncol(op))
315315
if(max(eYtrain) == min(eYtrain))
316316
print("Y contains only one class")
317317
else if(changesByPip < ref)
@@ -540,7 +540,7 @@ return (Double accuracy, Matrix[Double] evalFunHp, Matrix[Double] hpForPruning,
540540
{
541541
[trainX, trainy, testX, testy, Tr, hpForPruning, changesByOp, changesByPip] = executePipeline(pipeline=as.frame(pipList['ph']),
542542
Xtrain=trainX, Ytrain=trainy, Xtest= testX, Ytest=testy, metaList=metaList, hyperParameters=as.matrix(pipList['hp']), hpForPruning=hpForPruning,
543-
changesByOp=changesByOp, flagsCount=as.scalar(pipList['flags']), test=TRUE, verbose=FALSE)
543+
changesByOp=changesByOp, flagsCount=as.scalar(pipList['flags']), test=TRUE, verbose=FALSE, startInd=1, endInd=ncol(as.frame(pipList['ph'])))
544544
#TODO double check why this is necessary
545545
mincol = min(ncol(cvChanges),ncol(changesByOp))
546546
cvChanges[cvk,1:mincol] = changesByOp[,1:mincol];
@@ -557,7 +557,7 @@ return (Double accuracy, Matrix[Double] evalFunHp, Matrix[Double] hpForPruning,
557557
allChanges = min(allChanges)
558558
changesByOp = colMaxs(cvChanges)
559559
accuracy = mean(accuracyMatrix)
560-
print("cv accuracy: "+toString(accuracy))
560+
print("- cv accuracy: "+toString(accuracy))
561561
}
562562

563563
pruningSignal = function(Matrix[Double] pipPre, Matrix[Double] pipNew, Matrix[Double] hp_matrix, Matrix[Double] hpForPruning, Matrix[Double] changesByOp)
@@ -670,7 +670,7 @@ run_with_hyperparamNested = function(Frame[Unknown] ph_pip, Integer r_i = 1, Mat
670670
{
671671
[eXtrain, eYtrain, eXtest, eYtest, Tr, hpForPruning, changesByOp, changesByPip] = executePipeline(pipeline=op,
672672
Xtrain=X, Ytrain=Y, Xtest=Xtest, Ytest=Ytest, metaList=metaList2, hyperParameters=hp_matrix, hpForPruning=hpForPruning,
673-
changesByOp=changesByOp, flagsCount=no_of_flag_vars, test=TRUE, verbose=FALSE)
673+
changesByOp=changesByOp, flagsCount=no_of_flag_vars, test=TRUE, verbose=FALSE, startInd=1, endInd=ncol(op))
674674
if(max(eYtrain) == min(eYtrain))
675675
print("Y contains only one class")
676676
else if(changesByPip < ref)
@@ -727,4 +727,4 @@ return(Boolean execute)
727727
}
728728
}
729729
execute = !(changeCount > 0)
730-
}
730+
}

‎scripts/builtin/executePipeline.dml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@
5151

5252
f_executePipeline = function(Frame[String] pipeline, Matrix[Double] Xtrain, Matrix[Double] Ytrain,
5353
Matrix[Double] Xtest, Matrix[Double] Ytest, List[Unknown] metaList, Matrix[Double] hyperParameters, Matrix[Double] hpForPruning = as.matrix(0),
54-
Matrix[Double] changesByOp = as.matrix(0), Integer flagsCount, Boolean test = FALSE, Boolean verbose)
54+
Matrix[Double] changesByOp = as.matrix(0), Integer flagsCount, Boolean test = FALSE, Boolean verbose,
55+
Integer startInd, Integer endInd)
5556
return (Matrix[Double] Xtrain, Matrix[Double] Ytrain, Matrix[Double] Xtest, Matrix[Double] Ytest,
5657
Double t2, Matrix[Double] hpForPruning, Matrix[Double] changesByOp, Double changesAll, List[Unknown] internalStates)
5758
{
@@ -68,8 +69,11 @@ f_executePipeline = function(Frame[String] pipeline, Matrix[Double] Xtrain, Mat
6869
print("pipeline in execution "+toString(pipeline))
6970
print("pipeline hps "+toString(hyperParameters))
7071
}
71-
for(i in 1:ncol(pipeline)) {
72+
73+
# for(i in 1:ncol(pipeline)) {
74+
for(i in startInd:endInd) {
7275
op = as.scalar(pipeline[1,i])
76+
print("-- Applying Primitive: "+op);
7377
applyOp = toString(as.scalar(applyFunc[1,i]))
7478
Xclone = Xtrain
7579
XtestClone = Xtest
@@ -476,4 +480,4 @@ return (Matrix[Double] cmin, Matrix[Double] cmax)
476480
cmin[1, i] = min(vec)
477481
cmax[1, i] = max(vec)
478482
}
479-
}
483+
}

‎scripts/builtin/fit_pipeline.dml‎

Lines changed: 59 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ source("scripts/builtin/bandit.dml") as bandit;
4848

4949
f_fit_pipeline = function(Frame[Unknown] trainData, Frame[Unknown] testData, Frame[Unknown] metaData = as.frame("NULL"),
5050
Frame[Unknown] pip, Frame[Unknown] applyFunc, Matrix[Double] hp, Integer cvk=3, String evaluationFunc, Matrix[Double] evalFunHp,
51-
Boolean isLastLabel = TRUE, Boolean correctTypos=FALSE)
51+
Boolean isLastLabel = TRUE, Boolean correctTypos=FALSE, Boolean allCombinations=FALSE)
5252
return (Matrix[Double] scores, Matrix[Double] cleanTrain, Matrix[Double] cleanTest, List[Unknown] externalState, List[Unknown] iState)
5353
{
5454
externalState = list()
@@ -92,28 +92,66 @@ return (Matrix[Double] scores, Matrix[Double] cleanTrain, Matrix[Double] cleanTe
9292
hp_matrix = matrix(hp_width, rows=ncol(pip), cols=ncol(hp_width)/ncol(pip))
9393
pipList = list(ph = pip, hp = hp_matrix, flags = no_of_flag_vars)
9494

95+
print("Getting training score using CV")
9596
[trainScore, evalFunHp] = bandit::crossV(X=eXtrain, y=eYtrain, cvk=cvk, evalFunHp=evalFunHp,
9697
pipList=pipList, metaList=metaList, evalFunc=evaluationFunc)
97-
print("train score cv: "+toString(trainScore))
98+
print("- train score cv: "+toString(trainScore))
9899

99-
100-
# # # now test accuracy
101-
[eXtrain, eYtrain, eXtest, eYtest, a, b, c, d, iState] = executePipeline(pipeline=pip, Xtrain=eXtrain, Ytrain=eYtrain,
102-
Xtest=eXtest, Ytest=eYtest, metaList=metaList, hyperParameters=hp_matrix, flagsCount=no_of_flag_vars, test=TRUE, verbose=FALSE)
103-
104-
if(max(eYtrain) == min(eYtrain))
105-
stop("Y contains only one class")
100+
print("----------------------------");
101+
print("Getting test accuracy")
102+
primitives = matrix(0, rows=0, cols=0);
103+
if (allCombinations) {
104+
# Count number of subsets of consecutive primitives
105+
totCount = 0;
106+
n = ncol(pip);
107+
for (i in 1:n) {
108+
for (j in i:n)
109+
totCount = totCount + 1;
110+
}
111+
# List start and end indices of all those subsets
112+
primitives = matrix(0, rows=totCount, cols=2);
113+
r = 1;
114+
for (start in 1:n) {
115+
for (end in start:n) {
116+
primitives[r,1] = start;
117+
primitives[r,2] = end;
118+
r = r + 1;
119+
}
120+
}
121+
}
122+
else {
123+
# Include all primitives
124+
primitives = matrix(0, rows=1, cols=2);
125+
primitives[1,1] = 1;
126+
primitives[1,2] = ncol(pip);
127+
}
106128

107-
# score = eval(evaluationFunc, list(X=eXtrain, Y=eYtrain, Xtest=eXtrain, Ytest=eYtrain, Xorig=as.matrix(0), evalFunHp=evalFunHp))
108-
# trainAccuracy = as.scalar(score[1, 1])
109-
110-
score = eval(evaluationFunc, list(X=eXtrain, Y=eYtrain, Xtest=eXtest, Ytest=eYtest, Xorig=as.matrix(0), evalFunHp=evalFunHp))
111-
testAccuracy = as.scalar(score[1, 1])
112-
113-
scores = matrix(0, rows=1, cols=3)
114-
scores[1, 1] = dirtyScore
115-
# scores[1, 2] = trainAccuracy
116-
scores[1, 3] = testAccuracy
117-
cleanTrain = cbind(eXtrain, eYtrain)
118-
cleanTest = cbind(eXtest, eYtest)
129+
for (r in 1:nrow(primitives)) {
130+
startInd = as.scalar(primitives[r,1]);
131+
endInd = as.scalar(primitives[r,2]);
132+
# # # now test accuracy
133+
[eXtrain_cl, eYtrain_cl, eXtest_cl, eYtest_cl, a, b, c, d, iState] = executePipeline(pipeline=pip, Xtrain=eXtrain, Ytrain=eYtrain,
134+
Xtest=eXtest, Ytest=eYtest, metaList=metaList, hyperParameters=hp_matrix, flagsCount=no_of_flag_vars, test=TRUE, verbose=FALSE, startInd=startInd, endInd=endInd)
135+
136+
if(max(eYtrain_cl) == min(eYtrain_cl))
137+
stop("Y contains only one class")
138+
139+
# score = eval(evaluationFunc, list(X=eXtrain, Y=eYtrain, Xtest=eXtrain, Ytest=eYtrain, Xorig=as.matrix(0), evalFunHp=evalFunHp))
140+
# trainAccuracy = as.scalar(score[1, 1])
141+
142+
score = eval(evaluationFunc, list(X=eXtrain_cl, Y=eYtrain_cl, Xtest=eXtest_cl, Ytest=eYtest_cl, Xorig=as.matrix(0), evalFunHp=evalFunHp))
143+
testAccuracy = as.scalar(score[1, 1])
144+
145+
scores = matrix(0, rows=1, cols=3)
146+
scores[1, 1] = dirtyScore
147+
# scores[1, 2] = trainAccuracy
148+
scores[1, 3] = testAccuracy
149+
cleanTrain = cbind(eXtrain_cl, eYtrain_cl)
150+
cleanTest = cbind(eXtest, eYtest)
151+
152+
header = frame(["dirty acc", "train acc", "test acc"], rows=1, cols=3)
153+
result = as.frame(scores)
154+
writeRes = rbind(header, result)
155+
print(toString(writeRes))
156+
}
119157
}

‎src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinFitPipelineTest.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.sysds.test.TestConfiguration;
2525
import org.apache.sysds.test.TestUtils;
2626
import org.junit.Assert;
27+
import org.junit.Ignore;
2728
import org.junit.Test;
2829

2930
public class BuiltinFitPipelineTest extends AutomatedTestBase {
@@ -42,7 +43,8 @@ public class BuiltinFitPipelineTest extends AutomatedTestBase {
4243
public void setUp() {
4344
addTestConfiguration(TEST_NAME1,new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1,new String[]{"R"}));
4445
}
45-
46+
47+
@Ignore
4648
@Test
4749
public void testEvalPipClass() {
4850
evalPip(0.8, "FALSE", INPUT+"/classification/", Types.ExecMode.SINGLE_NODE);

‎src/test/scripts/functions/builtin/sliceLineRealData.dml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,13 @@ acc = lmPredictStats(yhat, y, TRUE);
4545
e = (y-yhat)^2;
4646

4747
# model debugging via sliceline
48-
[TK,TKC,D] = sliceLine(X=X, e=e, k=4, alpha=0.95, minSup=32, tpBlksz=16, verbose=TRUE)
48+
[TK,TKC,D] = sliceLine(X=X, e=e, k=4, alpha=0.95, minSup=32, tpBlksz=16, verbose=FALSE)
4949
tfspec2 = "{ ids:true, recode:[1,2,5], bin:[{id:3, method:equi-width, numbins:10},{id:4, method:equi-width, numbins:10}]}"
5050
XYZ = sliceLineDebug(TK=TK, TKC=TKC, tfmeta=meta, tfspec=tfspec2)
51-
[Xtk,etk] = sliceLineExtract(X=X, e=e, TK=TK, TKC=TKC, k2=3);
51+
[Xtk,etk,I] = sliceLineExtract(X=X, e=e, TK=TK, TKC=TKC, k2=3);
5252

5353
acc = acc[3,1];
54-
val = as.matrix((sum(TKC[1,4]) >= nrow(Xtk)) & (nrow(Xtk) == nrow(etk)))
54+
val = as.matrix((sum(TKC[1,4]) <= nrow(Xtk)) & (nrow(Xtk) == nrow(etk)))
5555

5656
write(acc, $3);
5757
write(val, $4);

‎src/test/scripts/functions/pipelines/executePipelineTest.dml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ hp = matrix("0.000 0.000 1.000 0.000 0.000 0.000 2.000
5252
1.000 0.786 0.000 0.000 1.000 1.000 2.000", rows=2, cols=7)
5353
print("X unchanged "+sum(eXtrain))
5454
[eX, Y, Xtest, Ytest, tr] = executePipeline(pip, eXtrain, eYtrain, eXtest, eYtest, metaList, hp,
55-
as.matrix(0), as.matrix(0), flagsCount, TRUE, FALSE)
55+
as.matrix(0), as.matrix(0), flagsCount, TRUE, FALSE, 1, ncol(pip))
5656

5757

5858
[eXtrain, imp] = imputeByMean(eXtrain, mask)

‎src/test/scripts/functions/pipelines/fit_pipelineTest.dml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ testData = F[split+1:nrow(F),]
6060

6161

6262
print("pipeline: "+toString(pip[1]))
63-
[result, trX, tsX, exState, iState] = fit_pipeline(trainData, testData, metaInfo, pip[1,], applyFunc[1,], hp[1,], 3, "evalClassification", evalHp, TRUE, FALSE)
63+
[result, trX, tsX, exState, iState] = fit_pipeline(trainData, testData, metaInfo, pip[1,], applyFunc[1,], hp[1,], 3, "evalClassification", evalHp, TRUE, FALSE, FALSE)
6464
eXtest = apply_pipeline(testData, metaInfo, pip[1,], applyFunc[1,], hp[1,], TRUE, exState, iState, FALSE)
6565

6666

0 commit comments

Comments
 (0)