Support for Unnest on Top-Level Array Fields in Flat Postgres Collection (#239)

suddendust · web-flow · commit 5c65ef64720c · 2025-10-15T11:16:37.000+05:30
diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java
diff --git a/document-store/src/integrationTest/resources/query/flat_array_relational_filter_response.json b/document-store/src/integrationTest/resources/query/flat_array_relational_filter_response.json
@@ -0,0 +1,14 @@
+[
+  {
+    "item": "Soap",
+    "price": 10
+  },
+  {
+    "item": "Soap",
+    "price": 20
+  },
+  {
+    "item": "Soap",
+    "price": 10
+  }
+]
diff --git a/document-store/src/integrationTest/resources/query/flat_boolean_array_filter_response.json b/document-store/src/integrationTest/resources/query/flat_boolean_array_filter_response.json
@@ -0,0 +1,10 @@
+[
+  {
+    "item": "Soap",
+    "price": 10
+  },
+  {
+    "item": "Shampoo",
+    "price": 5
+  }
+]
diff --git a/document-store/src/integrationTest/resources/query/flat_double_array_filter_response.json b/document-store/src/integrationTest/resources/query/flat_double_array_filter_response.json
@@ -0,0 +1,6 @@
+[
+  {
+    "item": "Shampoo",
+    "price": 5
+  }
+]
diff --git a/document-store/src/integrationTest/resources/query/flat_integer_array_filter_response.json b/document-store/src/integrationTest/resources/query/flat_integer_array_filter_response.json
@@ -0,0 +1,6 @@
+[
+  {
+    "item": "Mirror",
+    "price": 20
+  }
+]
diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_complex_query_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_complex_query_response.json
@@ -0,0 +1,37 @@
+[
+  {
+    "avg_price": "13.3333333333333333",
+    "tag_count": 3,
+    "tags": "hygiene"
+  },
+  {
+    "avg_price": "7.5000000000000000",
+    "tag_count": 2,
+    "tags": "premium"
+  },
+  {
+    "avg_price": "7.5000000000000000",
+    "tag_count": 2,
+    "tags": "personal-care"
+  },
+  {
+    "avg_price": "8.0000000000000000",
+    "tag_count": 2,
+    "tags": "grooming"
+  },
+  {
+    "avg_price": "7.5000000000000000",
+    "tag_count": 2,
+    "tags": "budget"
+  },
+  {
+    "avg_price": "6.5000000000000000",
+    "tag_count": 2,
+    "tags": "bulk"
+  },
+  {
+    "avg_price": "5.0000000000000000",
+    "tag_count": 2,
+    "tags": "hair-care"
+  }
+]
diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_not_preserving_empty_array_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_not_preserving_empty_array_response.json
@@ -0,0 +1,5 @@
+[
+  {
+    "count": 25
+  }
+]
diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_only_main_filter_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_only_main_filter_response.json
@@ -0,0 +1,26 @@
+[
+  {
+    "item": "Mirror",
+    "tags": "home-decor"
+  },
+  {
+    "item": "Mirror",
+    "tags": "reflective"
+  },
+  {
+    "item": "Mirror",
+    "tags": "glass"
+  },
+  {
+    "item": "Soap",
+    "tags": "hygiene"
+  },
+  {
+    "item": "Soap",
+    "tags": "antibacterial"
+  },
+  {
+    "item": "Soap",
+    "tags": "family-pack"
+  }
+]
diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_only_unnest_filter_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_only_unnest_filter_response.json
@@ -0,0 +1,10 @@
+[
+  {
+    "item": "Soap",
+    "tags": "premium"
+  },
+  {
+    "item": "Shampoo",
+    "tags": "premium"
+  }
+]
diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_preserving_empty_array_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_preserving_empty_array_response.json
@@ -0,0 +1,5 @@
+[
+  {
+    "count": 27
+  }
+]
diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_tags_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_tags_response.json
@@ -0,0 +1,70 @@
+[
+  {
+    "tags": "hygiene",
+    "count": 3
+  },
+  {
+    "tags": "personal-care",
+    "count": 2
+  },
+  {
+    "tags": "premium",
+    "count": 2
+  },
+  {
+    "tags": "home-decor",
+    "count": 1
+  },
+  {
+    "tags": "reflective",
+    "count": 1
+  },
+  {
+    "tags": "glass",
+    "count": 1
+  },
+  {
+    "tags": "hair-care",
+    "count": 2
+  },
+  {
+    "tags": "herbal",
+    "count": 1
+  },
+  {
+    "tags": "budget",
+    "count": 2
+  },
+  {
+    "tags": "bulk",
+    "count": 2
+  },
+  {
+    "tags": "antibacterial",
+    "count": 1
+  },
+  {
+    "tags": "family-pack",
+    "count": 1
+  },
+  {
+    "tags": "grooming",
+    "count": 2
+  },
+  {
+    "tags": "plastic",
+    "count": 1
+  },
+  {
+    "tags": "essential",
+    "count": 1
+  },
+  {
+    "tags": "wholesale",
+    "count": 1
+  },
+  {
+    "tags": "basic",
+    "count": 1
+  }
+]
diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_with_filters_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_with_filters_response.json
@@ -0,0 +1,10 @@
+[
+  {
+    "item": "Comb",
+    "tags": "grooming"
+  },
+  {
+    "item": "Comb",
+    "tags": "grooming"
+  }
+]
diff --git a/document-store/src/integrationTest/resources/query/pg_flat_collection_insert.json b/document-store/src/integrationTest/resources/query/pg_flat_collection_insert.json
@@ -1,12 +1,14 @@
 {
   "statements": [
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\"\n) VALUES (\n1, 'Soap', 10, 2, '2014-03-01T08:00:00Z',\n'{\"hygiene\", \"personal-care\", \"premium\"}',\n'{\"colors\": [\"Blue\", \"Green\"], \"brand\": \"Dettol\", \"size\": \"M\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\"\n) VALUES (\n2, 'Mirror', 20, 1, '2014-03-01T09:00:00Z',\n'{\"home-decor\", \"reflective\", \"glass\"}',\nNULL,\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\"\n) VALUES (\n3, 'Shampoo', 5, 10, '2014-03-15T09:00:00Z',\n'{\"hair-care\", \"personal-care\", \"premium\", \"herbal\"}',\n'{\"colors\": [\"Black\"], \"brand\": \"Sunsilk\", \"size\": \"L\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\"\n) VALUES (\n4, 'Shampoo', 5, 20, '2014-04-04T11:21:39.736Z',\n'{\"hair-care\", \"budget\", \"bulk\"}',\nNULL,\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\"\n) VALUES (\n5, 'Soap', 20, 5, '2014-04-04T21:23:13.331Z',\n'{\"hygiene\", \"antibacterial\", \"family-pack\"}',\n'{\"colors\": [\"Orange\", \"Blue\"], \"brand\": \"Lifebuoy\", \"size\": \"S\", \"seller\": {\"name\": \"Hans and Co.\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\"\n) VALUES (\n6, 'Comb', 7.5, 5, '2015-06-04T05:08:13Z',\n'{\"grooming\", \"plastic\", \"essential\"}',\nNULL,\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\"\n) VALUES (\n7, 'Comb', 7.5, 10, '2015-09-10T08:43:00Z',\n'{\"grooming\", \"bulk\", \"wholesale\"}',\n'{\"colors\": [], \"seller\": {\"name\": \"Go Go Plastics\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\"\n) VALUES (\n8, 'Soap', 10, 5, '2016-02-06T20:20:13Z',\n'{\"hygiene\", \"budget\", \"basic\"}',\nNULL,\nNULL\n)"
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n1, 'Soap', 10, 2, '2014-03-01T08:00:00Z',\n'{\"hygiene\", \"personal-care\", \"premium\"}',\n'{\"colors\": [\"Blue\", \"Green\"], \"brand\": \"Dettol\", \"size\": \"M\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\n'{1, 2, 3}',\n'{4.5, 9.2}',\n'{true, false}'\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n2, 'Mirror', 20, 1, '2014-03-01T09:00:00Z',\n'{\"home-decor\", \"reflective\", \"glass\"}',\nNULL,\nNULL,\n'{10, 20}',\nNULL,\nNULL\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n3, 'Shampoo', 5, 10, '2014-03-15T09:00:00Z',\n'{\"hair-care\", \"personal-care\", \"premium\", \"herbal\"}',\n'{\"colors\": [\"Black\"], \"brand\": \"Sunsilk\", \"size\": \"L\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\nNULL,\n'{3.14, 2.71}',\nNULL\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n4, 'Shampoo', 5, 20, '2014-04-04T11:21:39.736Z',\n'{\"hair-care\", \"budget\", \"bulk\"}',\nNULL,\nNULL,\nNULL,\nNULL,\n'{true, true}'\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n5, 'Soap', 20, 5, '2014-04-04T21:23:13.331Z',\n'{\"hygiene\", \"antibacterial\", \"family-pack\"}',\n'{\"colors\": [\"Orange\", \"Blue\"], \"brand\": \"Lifebuoy\", \"size\": \"S\", \"seller\": {\"name\": \"Hans and Co.\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\nNULL,\nNULL,\nNULL\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n6, 'Comb', 7.5, 5, '2015-06-04T05:08:13Z',\n'{\"grooming\", \"plastic\", \"essential\"}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n7, 'Comb', 7.5, 10, '2015-09-10T08:43:00Z',\n'{\"grooming\", \"bulk\", \"wholesale\"}',\n'{\"colors\": [], \"seller\": {\"name\": \"Go Go Plastics\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\nNULL,\nNULL,\nNULL\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n8, 'Soap', 10, 5, '2016-02-06T20:20:13Z',\n'{\"hygiene\", \"budget\", \"basic\"}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n9, 'Bottle', 15, 3, '2016-03-01T10:00:00Z',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n10, 'Cup', 8, 2, '2016-04-01T10:00:00Z',\n'{}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)"
   ]
 }
diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/FlatPostgresFieldTransformer.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/FlatPostgresFieldTransformer.java
@@ -13,8 +13,10 @@ public class FlatPostgresFieldTransformer implements PostgresColTransformer {
 
   @Override
   public FieldToPgColumn transform(String orgFieldName, Map<String, String> pgColMapping) {
-    // In flat structure mode, all fields are direct PostgreSQL columns as-is
-    return new FieldToPgColumn(null, PostgresUtils.wrapFieldNamesWithDoubleQuotes(orgFieldName));
+    // Check if this field has been unnested (e.g., "tags" -> "tags_unnested")
+    String pgColumnName = pgColMapping.getOrDefault(orgFieldName, orgFieldName);
+    // In flat structure mode, all fields are direct PostgreSQL columns
+    return new FieldToPgColumn(null, PostgresUtils.wrapFieldNamesWithDoubleQuotes(pgColumnName));
   }
 
   @Override
diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresFilterTypeExpressionVisitor.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresFilterTypeExpressionVisitor.java
diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresFromTypeExpressionVisitor.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresFromTypeExpressionVisitor.java
diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresUnnestFilterTypeExpressionVisitor.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresUnnestFilterTypeExpressionVisitor.java

-Original file line number
+Diff line change
@@ @@ -0,0 +1,6 @@ @@
 +[
 +  {
 +    "item": "Shampoo",
 +    "price": 5
 +  }
 +]