Skip to content

Commit 4f75d4c

Browse files
committed
compressed writer
1 parent 6744bf8 commit 4f75d4c

File tree

1 file changed

+5
-7
lines changed

1 file changed

+5
-7
lines changed

src/main/java/org/apache/sysds/runtime/io/FrameWriterCompressed.java

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,13 @@
1919

2020
package org.apache.sysds.runtime.io;
2121

22-
import java.io.IOException;
22+
import java.util.List;
2323

24-
import org.apache.hadoop.fs.Path;
25-
import org.apache.hadoop.mapred.JobConf;
2624
import org.apache.sysds.hops.OptimizerUtils;
27-
import org.apache.sysds.runtime.DMLRuntimeException;
2825
import org.apache.sysds.runtime.frame.data.FrameBlock;
26+
import org.apache.sysds.runtime.frame.data.columns.Array;
2927
import org.apache.sysds.runtime.frame.data.lib.FrameLibCompress;
28+
import org.apache.sysds.runtime.matrix.data.Pair;
3029

3130
public class FrameWriterCompressed extends FrameWriterBinaryBlockParallel {
3231

@@ -37,11 +36,10 @@ public FrameWriterCompressed(boolean parallel) {
3736
}
3837

3938
@Override
40-
protected void writeBinaryBlockFrameToHDFS(Path path, JobConf job, FrameBlock src, long rlen, long clen)
41-
throws IOException, DMLRuntimeException {
39+
protected Pair<List<Pair<Integer, Array<?>>>, FrameBlock> extractDictionaries(FrameBlock src) {
4240
int k = parallel ? OptimizerUtils.getParallelBinaryWriteParallelism() : 1;
4341
FrameBlock compressed = FrameLibCompress.compress(src, k);
44-
super.writeBinaryBlockFrameToHDFS(path, job, compressed, rlen, clen);
42+
return super.extractDictionaries(compressed);
4543
}
4644

4745
}

0 commit comments

Comments
 (0)