Skip to content

Commit 5ff6274

Browse files
committed
[minor] Compression SP Schema apply
Closes #2215
1 parent dc3947a commit 5ff6274

File tree

2 files changed

+26
-1
lines changed

2 files changed

+26
-1
lines changed

src/main/java/org/apache/sysds/runtime/instructions/SPInstructionParser.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.apache.sysds.lops.WeightedUnaryMM;
4040
import org.apache.sysds.lops.WeightedUnaryMMR;
4141
import org.apache.sysds.runtime.DMLRuntimeException;
42+
import org.apache.sysds.runtime.instructions.cp.CPInstruction.CPType;
4243
import org.apache.sysds.runtime.instructions.cp.CPOperand;
4344
import org.apache.sysds.runtime.instructions.spark.AggregateTernarySPInstruction;
4445
import org.apache.sysds.runtime.instructions.spark.AggregateUnarySPInstruction;
@@ -195,6 +196,7 @@ public class SPInstructionParser extends InstructionParser
195196
String2SPInstructionType.put( "freplicate", SPType.Binary);
196197
String2SPInstructionType.put( "mapdropInvalidLength", SPType.Binary);
197198
String2SPInstructionType.put( "valueSwap", SPType.Binary);
199+
String2SPInstructionType.put( "applySchema" , SPType.Binary);
198200
String2SPInstructionType.put( "_map", SPType.Ternary); // _map refers to the operation map
199201
// Relational Instruction Opcodes
200202
String2SPInstructionType.put( "==" , SPType.Binary);

src/main/java/org/apache/sysds/runtime/instructions/spark/BinaryFrameFrameSPInstruction.java

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@ else if(getOpcode().equals("valueSwap")) {
5959
// Attach result frame with FrameBlock associated with output_name
6060
sec.releaseFrameInput(input2.getName());
6161
}
62+
else if(getOpcode().equals("applySchema")){
63+
Broadcast<FrameBlock> fb = sec.getSparkContext().broadcast(sec.getFrameInput(input2.getName()));
64+
out = in1.mapValues(new applySchema(fb.getValue()));
65+
sec.releaseFrameInput(input2.getName());
66+
}
6267
else {
6368
JavaPairRDD<Long, FrameBlock> in2 = sec.getFrameBinaryBlockRDDHandleForVariable(input2.getName());
6469
// create output frame
@@ -70,7 +75,9 @@ else if(getOpcode().equals("valueSwap")) {
7075
//set output RDD and maintain dependencies
7176
sec.setRDDHandleForVariable(output.getName(), out);
7277
sec.addLineageRDD(output.getName(), input1.getName());
73-
if( !getOpcode().equals("dropInvalidType") && !getOpcode().equals("valueSwap"))
78+
if(!getOpcode().equals("dropInvalidType") && //
79+
!getOpcode().equals("valueSwap") && //
80+
!getOpcode().equals("applySchema"))
7481
sec.addLineageRDD(output.getName(), input2.getName());
7582
}
7683

@@ -116,4 +123,20 @@ public FrameBlock call(FrameBlock arg0) throws Exception {
116123
return arg0.valueSwap(schema_frame);
117124
}
118125
}
126+
127+
128+
private static class applySchema implements Function<FrameBlock, FrameBlock>{
129+
private static final long serialVersionUID = 58504021316402L;
130+
131+
private FrameBlock schema;
132+
133+
public applySchema(FrameBlock schema ) {
134+
this.schema = schema;
135+
}
136+
137+
@Override
138+
public FrameBlock call(FrameBlock arg0) throws Exception {
139+
return arg0.applySchema(schema);
140+
}
141+
}
119142
}

0 commit comments

Comments
 (0)