Skip to content

Commit a7acef1

Browse files
suddendustsuddendust
authored
Handle nested null arrays correctly (#265)
1 parent 9b4ec54 commit a7acef1

File tree

7 files changed

+109
-28
lines changed

7 files changed

+109
-28
lines changed

‎document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1915,6 +1915,26 @@ void testNotContainsAndUnnestFilters(String dataStoreName) throws IOException {
19151915
dataStoreName, iterator, "query/unwind_not_contains_filter_response.json", 2);
19161916
}
19171917

1918+
@ParameterizedTest
1919+
@ArgumentsSource(PostgresProvider.class)
1920+
public void testUnnestTopLevelArrayWithNullValue(String dataStoreName) throws IOException {
1921+
Collection collection = getFlatCollection(dataStoreName);
1922+
1923+
Query query =
1924+
Query.builder()
1925+
.addSelection(IdentifierExpression.of("item"))
1926+
.addSelection(IdentifierExpression.of("categoryTags"))
1927+
.addFromClause(UnnestExpression.of(IdentifierExpression.of("categoryTags"), true))
1928+
.setFilter(
1929+
RelationalExpression.of(
1930+
IdentifierExpression.of("item"), EQ, ConstantExpression.of("Bottle")))
1931+
.build();
1932+
1933+
Iterator<Document> iterator = collection.aggregate(query);
1934+
assertDocsAndSizeEqualWithoutOrder(
1935+
dataStoreName, iterator, "query/unnest_null_top_level_array_response.json", 1);
1936+
}
1937+
19181938
@ParameterizedTest
19191939
@ArgumentsSource(AllProvider.class)
19201940
public void testQueryV1DistinctCountWithSortingSpecs(String dataStoreName) throws IOException {
@@ -5575,6 +5595,49 @@ void testNotInOnUnnestedArray(String dataStoreName) throws Exception {
55755595
assertEquals(12, count, "Should return unnested locations not matching the filter");
55765596
}
55775597

5598+
@ParameterizedTest
5599+
@ArgumentsSource(PostgresProvider.class)
5600+
public void testUnnestNestedArrayWithNullValue(String dataStoreName) throws IOException {
5601+
Collection collection = getFlatCollection(dataStoreName);
5602+
5603+
Query query =
5604+
Query.builder()
5605+
.addSelection(IdentifierExpression.of("item"))
5606+
.addFromClause(
5607+
UnnestExpression.of(JsonIdentifierExpression.of("props", "source-loc"), true))
5608+
.setFilter(
5609+
RelationalExpression.of(
5610+
IdentifierExpression.of("_id"), EQ, ConstantExpression.of(1)))
5611+
.build();
5612+
5613+
Iterator<Document> iterator = collection.aggregate(query);
5614+
assertDocsAndSizeEqualWithoutOrder(
5615+
dataStoreName, iterator, "query/unnest_null_nested_array_response.json", 2);
5616+
}
5617+
5618+
@ParameterizedTest
5619+
@ArgumentsSource(PostgresProvider.class)
5620+
public void testArrayFilterAnyWithJsonNullArray(String dataStoreName) {
5621+
Collection collection = getFlatCollection(dataStoreName);
5622+
5623+
Query query =
5624+
Query.builder()
5625+
.addSelection(IdentifierExpression.of("item"))
5626+
.setFilter(
5627+
ArrayRelationalFilterExpression.builder()
5628+
.operator(ArrayOperator.ANY)
5629+
.filter(
5630+
RelationalExpression.of(
5631+
JsonIdentifierExpression.of("props", "colors"),
5632+
EQ,
5633+
ConstantExpression.of("Blue")))
5634+
.build())
5635+
.build();
5636+
5637+
long count = collection.count(query);
5638+
assertEquals(2, count, "Should find 2 items with 'Blue' color");
5639+
}
5640+
55785641
@ParameterizedTest
55795642
@ArgumentsSource(PostgresProvider.class)
55805643
void testExistsOnArrays(String dataStoreName) throws JsonProcessingException {

‎document-store/src/integrationTest/resources/query/pg_flat_collection_insert.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
"INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n4, 'Shampoo', 5, 20, '2014-04-04T11:21:39.736Z', false,\n'{\"hair-care\", \"budget\", \"bulk\"}',\n'{\"HairCare\"}',\nNULL,\nNULL,\n'{1, 2}',\n'{5.0, 10.0}',\n'{true, true}'\n)",
77
"INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n5, 'Soap', 20, 5, '2014-04-04T21:23:13.331Z', true,\n'{\"hygiene\", \"antibacterial\", \"family-pack\"}',\n'{\"Hygiene\"}',\n'{\"colors\": [\"Orange\", \"Blue\"], \"brand\": \"Lifebuoy\", \"size\": \"S\", \"product-code\": \"SOAP-LIF-005\", \"source-loc\": [\"warehouse-C\"], \"seller\": {\"name\": \"Hans and Co.\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\n'{3, 6, 9}',\n'{7.5}',\n'{false}'\n)",
88
"INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n6, 'Comb', 7.5, 5, '2015-06-04T05:08:13Z', true,\n'{\"grooming\", \"plastic\", \"essential\"}',\n'{\"Grooming\"}',\nNULL,\nNULL,\n'{20, 30}',\n'{6.0, 8.0}',\n'{true, false}'\n)",
9-
"INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n7, 'Comb', 7.5, 10, '2015-09-10T08:43:00Z', false,\n'{\"grooming\", \"bulk\", \"wholesale\"}',\n'{\"Grooming\"}',\n'{\"colors\": [], \"product-code\": null, \"source-loc\": [], \"seller\": {\"name\": \"Go Go Plastics\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\n'{10}',\n'{3.0}',\n'{false, false, false}'\n)",
9+
"INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n7, 'Comb', 7.5, 10, '2015-09-10T08:43:00Z', false,\n'{\"grooming\", \"bulk\", \"wholesale\"}',\n'{\"Grooming\"}',\n'{\"colors\": [], \"product-code\": null, \"source-loc\": null, \"seller\": {\"name\": \"Go Go Plastics\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\n'{10}',\n'{3.0}',\n'{false, false, false}'\n)",
1010
"INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n8, 'Soap', 10, 5, '2016-02-06T20:20:13Z', true,\n'{\"hygiene\", \"budget\", \"basic\"}',\n'{\"Hygiene\"}',\nNULL,\nNULL,\n'{1, 10, 20}',\n'{2.5, 5.0}',\n'{true}'\n)",
1111
"INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n9, 'Bottle', 15, 3, '2016-03-01T10:00:00Z', false,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)",
1212
"INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n10, 'Cup', 8, 2, '2016-04-01T10:00:00Z', true,\n'{}',\n'{}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)"

‎document-store/src/integrationTest/resources/query/unnest_null_nested_array_response.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[
2+
{
3+
"item": "Soap"
4+
},
5+
{
6+
"item": "Soap"
7+
}
8+
]

‎document-store/src/integrationTest/resources/query/unnest_null_top_level_array_response.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
[
2+
{
3+
"item": "Bottle"
4+
}
5+
]

‎document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresFilterTypeExpressionVisitor.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -222,8 +222,8 @@ private String getFilterStringForAnyOperator(final ArrayRelationalFilterExpressi
222222
} else {
223223
// For nested collections OR JSONB arrays in flat collections, use jsonb_array_elements()
224224
return String.format(
225-
"EXISTS (SELECT 1 FROM jsonb_array_elements(COALESCE(%s, '[]'::jsonb)) AS \"%s\" WHERE %s)",
226-
parsedLhs, alias, parsedFilter);
225+
"EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(%s) = 'array' THEN %s ELSE '[]'::jsonb END) AS \"%s\" WHERE %s)",
226+
parsedLhs, parsedLhs, alias, parsedFilter);
227227
}
228228
}
229229

@@ -331,9 +331,10 @@ private String getFilterStringForAnyOperator(final DocumentArrayFilterExpression
331331
parsedLhs, arrayTypeCast, alias, parsedFilter);
332332
} else {
333333
// For nested collections OR JSONB arrays in flat collections, use jsonb_array_elements()
334+
// Use jsonb_typeof() to handle JSON null values - COALESCE only handles SQL NULL
334335
return String.format(
335-
"EXISTS (SELECT 1 FROM jsonb_array_elements(COALESCE(%s, '[]'::jsonb)) AS \"%s\" WHERE %s)",
336-
parsedLhs, alias, parsedFilter);
336+
"EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(%s) = 'array' THEN %s ELSE '[]'::jsonb END) AS \"%s\" WHERE %s)",
337+
parsedLhs, parsedLhs, alias, parsedFilter);
337338
}
338339
}
339340
}

‎document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresFromTypeExpressionVisitor.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ public class PostgresFromTypeExpressionVisitor implements FromTypeExpressionVisi
2424
"%s as (SELECT * from %s %s, %s %s)";
2525
private static final String PRESERVE_NULL_AND_EMPTY_TABLE_QUERY_FMT =
2626
"%s as (SELECT * from %s %s LEFT JOIN LATERAL %s %s on TRUE)";
27-
private static final String JSONB_UNWIND_EXP_FMT = "jsonb_array_elements(%s)";
27+
private static final String JSONB_UNWIND_EXP_FMT =
28+
"jsonb_array_elements(CASE WHEN jsonb_typeof(%s) = 'array' THEN %s ELSE '[]'::jsonb END)";
2829
private static final String NATIVE_UNWIND_EXP_FMT = "unnest(%s)";
2930
private static final String UNWIND_EXP_ALIAS_FMT = "p%s(%s)";
3031

@@ -80,7 +81,10 @@ public String visit(UnnestExpression unnestExpression) {
8081
String preTable = "table" + preIndex;
8182
String newTable = "table" + nextIndex;
8283
String tableAlias = "t" + preIndex;
83-
String unwindExpr = String.format(unnestFunction, transformedFieldName);
84+
String unwindExpr =
85+
unnestFunction.equals(JSONB_UNWIND_EXP_FMT)
86+
? String.format(unnestFunction, transformedFieldName, transformedFieldName)
87+
: String.format(unnestFunction, transformedFieldName);
8488

8589
// we'll quote the col name to prevent folding to lower case for top-level array fields
8690
String unwindExprAlias =

0 commit comments

Comments
 (0)