Skip to content

Commit 2bad499

Browse files
blackwinterblackwinter
authored
Merge pull request #244 from metafacture/addFlattenFunction
Add `flatten()` Fix function.
2 parents 93b83e9 + 27203ae commit 2bad499

File tree

9 files changed

+190
-1
lines changed

9 files changed

+190
-1
lines changed

‎README.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,14 @@ Only keeps field values that match the regular expression pattern.
489489
filter("<sourceField>", "<regexp>")
490490
```
491491

492+
##### `flatten`
493+
494+
Flattens a nested array field.
495+
496+
```perl
497+
flatten("<sourceField>")
498+
```
499+
492500
##### `index`
493501

494502
Returns the index position of a substring in a field and replaces the field value with this number.

‎metafix/src/main/java/org/metafacture/metafix/FixMethod.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -414,6 +414,14 @@ public void apply(final Metafix metafix, final Record record, final List<String>
414414
);
415415
}
416416
},
417+
flatten {
418+
@Override
419+
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
420+
record.transform(params.get(0), (m, c) -> m
421+
.ifArray(a -> c.accept(newArray(flatten(a.stream()))))
422+
);
423+
}
424+
},
417425
index {
418426
@Override
419427
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {

‎metafix/src/main/java/org/metafacture/metafix/api/FixFunction.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,11 @@ default Stream<Value> unique(final Stream<Value> stream) {
5555
return stream.filter(set::add);
5656
}
5757

58+
default Stream<Value> flatten(final Stream<Value> stream) {
59+
return stream.flatMap(v -> v.extractType((m, c) -> m
60+
.ifArray(a -> c.accept(flatten(a.stream())))
61+
.orElse(w -> c.accept(Stream.of(w)))
62+
));
63+
}
64+
5865
}

‎metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1109,6 +1109,156 @@ public void shouldFilterArrayObjectValues() {
11091109
);
11101110
}
11111111

1112+
@Test
1113+
public void shouldFlattenFlatArray() {
1114+
MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
1115+
"flatten('flat')"
1116+
),
1117+
i -> {
1118+
i.startRecord("1");
1119+
i.literal("flat", "1");
1120+
i.literal("flat", "2");
1121+
i.literal("flat", "3");
1122+
i.endRecord();
1123+
},
1124+
o -> {
1125+
o.get().startRecord("1");
1126+
o.get().literal("flat", "1");
1127+
o.get().literal("flat", "2");
1128+
o.get().literal("flat", "3");
1129+
o.get().endRecord();
1130+
}
1131+
);
1132+
}
1133+
1134+
@Test
1135+
public void shouldFlattenFlatHash() {
1136+
MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
1137+
"flatten('flat')"
1138+
),
1139+
i -> {
1140+
i.startRecord("1");
1141+
i.startEntity("flat");
1142+
i.literal("a", "1");
1143+
i.literal("a", "2");
1144+
i.literal("a", "3");
1145+
i.endEntity();
1146+
i.endRecord();
1147+
},
1148+
o -> {
1149+
o.get().startRecord("1");
1150+
o.get().startEntity("flat");
1151+
o.get().literal("a", "1");
1152+
o.get().literal("a", "2");
1153+
o.get().literal("a", "3");
1154+
o.get().endEntity();
1155+
o.get().endRecord();
1156+
}
1157+
);
1158+
}
1159+
1160+
@Test
1161+
public void shouldFlattenFlatString() {
1162+
MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
1163+
"flatten('flat')"
1164+
),
1165+
i -> {
1166+
i.startRecord("1");
1167+
i.literal("flat", "1");
1168+
i.endRecord();
1169+
},
1170+
o -> {
1171+
o.get().startRecord("1");
1172+
o.get().literal("flat", "1");
1173+
o.get().endRecord();
1174+
}
1175+
);
1176+
}
1177+
1178+
@Test
1179+
public void shouldFlattenNestedArray() {
1180+
MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
1181+
"flatten('deep[]')"
1182+
),
1183+
i -> {
1184+
i.startRecord("1");
1185+
i.startEntity("deep[]");
1186+
i.literal("1", "1");
1187+
i.startEntity("2[]");
1188+
i.literal("1", "2");
1189+
i.literal("2", "3");
1190+
i.endEntity();
1191+
i.startEntity("3[]");
1192+
i.startEntity("1[]");
1193+
i.literal("1", "4");
1194+
i.literal("2", "5");
1195+
i.endEntity();
1196+
i.literal("2", "6");
1197+
i.endEntity();
1198+
i.literal("4", "7");
1199+
i.endEntity();
1200+
i.endRecord();
1201+
},
1202+
o -> {
1203+
o.get().startRecord("1");
1204+
o.get().startEntity("deep[]");
1205+
o.get().literal("1", "1");
1206+
o.get().literal("2", "2");
1207+
o.get().literal("3", "3");
1208+
o.get().literal("4", "4");
1209+
o.get().literal("5", "5");
1210+
o.get().literal("6", "6");
1211+
o.get().literal("7", "7");
1212+
o.get().endEntity();
1213+
o.get().endRecord();
1214+
}
1215+
);
1216+
}
1217+
1218+
@Test
1219+
public void shouldFlattenNestedArrayWithHashes() {
1220+
MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
1221+
"flatten('deep[]')"
1222+
),
1223+
i -> {
1224+
i.startRecord("1");
1225+
i.startEntity("deep[]");
1226+
i.literal("1", "1");
1227+
i.startEntity("2");
1228+
i.literal("a", "2");
1229+
i.literal("a", "3");
1230+
i.endEntity();
1231+
i.startEntity("3[]");
1232+
i.startEntity("1");
1233+
i.literal("a", "4");
1234+
i.literal("a", "5");
1235+
i.endEntity();
1236+
i.literal("2", "6");
1237+
i.endEntity();
1238+
i.literal("4", "7");
1239+
i.endEntity();
1240+
i.endRecord();
1241+
},
1242+
o -> {
1243+
o.get().startRecord("1");
1244+
o.get().startEntity("deep[]");
1245+
o.get().literal("1", "1");
1246+
o.get().startEntity("2");
1247+
o.get().literal("a", "2");
1248+
o.get().literal("a", "3");
1249+
o.get().endEntity();
1250+
o.get().startEntity("3");
1251+
o.get().literal("a", "4");
1252+
o.get().literal("a", "5");
1253+
o.get().endEntity();
1254+
o.get().literal("4", "6");
1255+
o.get().literal("5", "7");
1256+
o.get().endEntity();
1257+
o.get().endRecord();
1258+
}
1259+
);
1260+
}
1261+
11121262
@Test
11131263
public void shouldGetFirstIndexOfSubstring() {
11141264
MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(

‎metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/todo.txt

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/expected.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"zoo" : [ {
3+
"animals" : [ "ant", "dog", "cat", "fish", "zebra", "horse", "hippo", "giraffe" ]
4+
} ]
5+
}

‎metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/input.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"zoo": [ {"animals" : [ [ "ant", "dog" ], "cat", [ "fish", [ "zebra", "horse" ], "hippo" ], "giraffe"] } ]
3+
}

‎metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/test.fix

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
flatten("zoo[].*.animals[]")

‎metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/test.flux

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
FLUX_DIR + "input.json"
2+
|open-file
3+
|as-records
4+
|decode-json
5+
|fix(FLUX_DIR + "test.fix")
6+
|encode-json(prettyPrinting="true")
7+
|write(FLUX_DIR + "output-metafix.json")
8+
;

0 commit comments

Comments
 (0)