Skip to content

Commit 61a4573

Browse files
committed
fix to improve SWAP intent detection logic
this includes new tests to lock in the behavior with toJson
1 parent 8a43e16 commit 61a4573

File tree

2 files changed

+57
-1
lines changed

2 files changed

+57
-1
lines changed

tessellate-main/src/main/java/io/clusterless/tessellate/pipeline/Transformer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ private PipelineContext discardAndEval(PipelineContext context) {
6262

6363
Fields selector;
6464

65-
if ((argsIsAll && result.results().isNone()) || resultsEqualsDeclared) {
65+
if ((argsIsAll && result.results().isNone()) || (argsIsAll && resultsEqualsDeclared)) {
6666
selector = Fields.RESULTS;
6767
} else if (argsIsAll) {
6868
selector = result.results();

tessellate-main/src/test/java/io/clusterless/tessellate/pipeline/PipelineTest.java

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -847,6 +847,62 @@ void toJsonAndBack(
847847
assertFilenameParts(output, "test-", "-guid", ".csv", 1);
848848
}
849849

850+
@Test
851+
void toJsonPartialSwap(@PathForResource("/data/delimited-header.csv") URI input, @PathForOutput("output") URI output) throws IOException {
852+
baseTransformPartial(input, output, "first+third ^toJson{} -> json", 4);
853+
}
854+
855+
@Test
856+
void toJsonPartialAppend(@PathForResource("/data/delimited-header.csv") URI input, @PathForOutput("output") URI output) throws IOException {
857+
baseTransformPartial(input, output, "first+third ^toJson{} +> json", 6);
858+
}
859+
860+
private static void baseTransformPartial(URI input, URI output, String transform, int numFields) throws IOException {
861+
PipelineOptions pipelineOptions = new PipelineOptions();
862+
863+
PipelineDef writeJsonDef = PipelineDef.builder()
864+
.withName("test")
865+
.withSource(Source.builder()
866+
.withInputs(List.of(input))
867+
.withSchema(Schema.builder()
868+
.withFormat(Format.csv)
869+
.withEmbedsSchema(true)
870+
.build())
871+
.build())
872+
.withTransform(
873+
new Transform(
874+
transform
875+
)
876+
)
877+
.withSink(Sink.builder()
878+
.withOutput(output)
879+
.withSchema(Schema.builder()
880+
.withFormat(Format.csv)
881+
.withEmbedsSchema(true)
882+
.build())
883+
.withFilename(Filename.builder()
884+
.withPrefix("test")
885+
.withIncludeGuid(true)
886+
.withProvidedGuid("guid")
887+
.build())
888+
.build())
889+
.build();
890+
891+
Pipeline writeJson = new Pipeline(pipelineOptions, writeJsonDef);
892+
893+
writeJson.run();
894+
895+
CascadingTesting.validateEntries(
896+
writeJson.flow().openSink(),
897+
l -> assertEquals(13, l, "wrong file length"), // headers are declared so aren't counted
898+
l -> assertEquals(numFields, l, "wrong tuple size"),
899+
l -> {
900+
}
901+
);
902+
903+
assertFilenameParts(output, "test-", "-guid", ".csv", 1);
904+
}
905+
850906
private static void assertFilenameParts(URI output, String prefix, String guid, String extension, int fileCount) throws IOException {
851907
final int[] count = {0};
852908
try (Stream<Path> pathStream = Files.find(Paths.get(output), 10, (path, attr) -> !path.toString().startsWith(".") && path.toString().endsWith(extension))) {

0 commit comments

Comments
 (0)