Skip to content

Commit 4d6c6b4

Browse files
committed
[SYSTEMDS-3846] Perftest: Fix parfor forced CP compilation
The perftest runMSVM_10k_1k_dense_k5 reproducible failed on serializing the parfor body program for a remote spark parfor job, because there were remaining spark instructions (checkpoints). The reason was because the forced CP compilation before such remote jobs was only applied in a subset of cases and special hop properties were not correctly cleaned up. This patch fixes the general issue.
1 parent 56c7823 commit 4d6c6b4

File tree

2 files changed

+6
-3
lines changed

2 files changed

+6
-3
lines changed

src/main/java/org/apache/sysds/hops/recompile/Recompiler.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1509,8 +1509,12 @@ public static void rReplaceLiterals( Hop hop, LocalVariableMap vars, boolean sca
15091509
public static void rSetExecType( Hop hop, ExecType etype ) {
15101510
if( hop.isVisited() )
15111511
return;
1512-
//update function names
1512+
//update hop properties (exec type, spark lops)
15131513
hop.setForcedExecType(etype);
1514+
if(etype == ExecType.CP) {
1515+
hop.setRequiresCheckpoint(false);
1516+
hop.setRequiresReblock(false);
1517+
}
15141518
if( hop.getInput() != null )
15151519
for( Hop c : hop.getInput() )
15161520
rSetExecType(c, etype);

src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -834,8 +834,7 @@ private void executeRemoteSparkParFor(ExecutionContext ec, IntObject from, IntOb
834834
{
835835
// Step 0) check and compile to CP (if forced remote parfor)
836836
boolean flagForced = false;
837-
if( FORCE_CP_ON_REMOTE_SPARK && (_optMode == POptMode.NONE
838-
|| (_optMode == POptMode.CONSTRAINED && _execMode==PExecMode.REMOTE_SPARK)) ) {
837+
if( FORCE_CP_ON_REMOTE_SPARK && _execMode==PExecMode.REMOTE_SPARK ) {
839838
//tid = 0 because replaced in remote parworker
840839
flagForced = checkSparkAndRecompileToCP(0);
841840
}

0 commit comments

Comments
 (0)