Skip to content

Commit abadac5

Browse files
shangyianshangyian
authored
Add better metric metadata to materializations (#1322)
* Add better metric metadata to materializations * Fix tests * Add display name to cube
1 parent ea6d490 commit abadac5

File tree

6 files changed

+76
-5
lines changed

6 files changed

+76
-5
lines changed

‎datajunction-server/datajunction_server/internal/cube_materializations.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
from sqlalchemy.ext.asyncio import AsyncSession
66

7+
from datajunction_server.sql.parsing.backends.antlr4 import parse
78
from datajunction_server.construction.build_v2 import get_measures_query
89
from datajunction_server.database.node import Column, NodeRevision
910
from datajunction_server.errors import DJInvalidInputException
@@ -137,6 +138,20 @@ def _combine_measures_join_criteria(left_table, right_table, query_grain):
137138
)
138139

139140

141+
def _extract_expression(metric_query: str) -> str:
142+
"""
143+
Extract only the derived metric expression from a metric query.
144+
"""
145+
expression = parse(metric_query).select.projection[0]
146+
return str(
147+
expression.child
148+
if isinstance(expression, ast.Alias)
149+
else expression.without_aliases()
150+
if isinstance(expression, ast.Expression)
151+
else expression,
152+
)
153+
154+
140155
async def build_cube_materialization(
141156
session: AsyncSession,
142157
current_revision: NodeRevision,
@@ -247,12 +262,14 @@ async def build_cube_materialization(
247262
cube=NodeNameVersion(
248263
name=current_revision.name,
249264
version=current_revision.version,
265+
display_name=current_revision.display_name,
250266
),
251267
metrics=[
252268
CubeMetric(
253269
metric=NodeNameVersion(
254270
name=metric.name,
255271
version=metric.current_version,
272+
display_name=metric.current.display_name,
256273
),
257274
required_measures=[
258275
MeasureKey(
@@ -262,6 +279,9 @@ async def build_cube_materialization(
262279
for measure in metrics_mapping.get(metric.name)[1][0] # type: ignore
263280
],
264281
derived_expression=metrics_mapping.get(metric.name)[1][1], # type: ignore
282+
metric_expression=_extract_expression(
283+
metrics_mapping.get(metric.name)[1][1], # type: ignore
284+
),
265285
)
266286
for metric in current_revision.cube_metrics()
267287
],

‎datajunction-server/datajunction_server/models/cube_materialization.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,13 @@ class CubeMetric(BaseModel):
207207
)
208208
derived_expression: str = Field(
209209
description=(
210-
"The expression for rewriting the original metric query "
210+
"The query for rewriting the original metric query "
211+
"using the materialized measures."
212+
),
213+
)
214+
metric_expression: str = Field(
215+
description=(
216+
"SQL expression for rewriting the original metric query "
211217
"using the materialized measures."
212218
),
213219
)

‎datajunction-server/datajunction_server/models/node_type.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ class NodeNameVersion(BaseModel):
4343

4444
name: str
4545
version: str
46+
display_name: str | None
4647

4748
class Config:
4849
orm_mode = True

‎datajunction-server/tests/api/cubes_test.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2213,6 +2213,7 @@ async def test_cube_materialization_metadata(
22132213
assert results["cube"] == {
22142214
"name": "default.example_repairs_cube",
22152215
"version": "v1.0",
2216+
"display_name": "Example Repairs Cube",
22162217
}
22172218
assert results["job"] == "DRUID_CUBE"
22182219
assert results["lookback_window"] == "1 DAY"
@@ -2232,98 +2233,116 @@ async def test_cube_materialization_metadata(
22322233
"derived_expression": "SELECT CAST(sum(discount_sum_62846f49) AS DOUBLE) / "
22332234
"SUM(count_3389dae3) AS default_DOT_discounted_orders_rate FROM "
22342235
"default.repair_orders_fact",
2236+
"metric_expression": "CAST(sum(discount_sum_62846f49) AS DOUBLE) / "
2237+
"SUM(count_3389dae3)",
22352238
"metric": {
22362239
"name": "default.discounted_orders_rate",
22372240
"version": "v1.0",
2241+
"display_name": "Discounted Orders Rate",
22382242
},
22392243
"required_measures": [
22402244
{
22412245
"measure_name": "discount_sum_62846f49",
22422246
"node": {
22432247
"name": "default.repair_orders_fact",
22442248
"version": "v1.0",
2249+
"display_name": "Repair Orders Fact",
22452250
},
22462251
},
22472252
{
22482253
"measure_name": "count_3389dae3",
22492254
"node": {
22502255
"name": "default.repair_orders_fact",
22512256
"version": "v1.0",
2257+
"display_name": "Repair Orders Fact",
22522258
},
22532259
},
22542260
],
22552261
},
22562262
{
22572263
"derived_expression": "SELECT SUM(repair_order_id_count_0b7dfba0) FROM "
22582264
"default.repair_orders_fact",
2265+
"metric_expression": "SUM(repair_order_id_count_0b7dfba0)",
22592266
"metric": {
22602267
"name": "default.num_repair_orders",
22612268
"version": "v1.0",
2269+
"display_name": "Num Repair Orders",
22622270
},
22632271
"required_measures": [
22642272
{
22652273
"measure_name": "repair_order_id_count_0b7dfba0",
22662274
"node": {
22672275
"name": "default.repair_orders_fact",
22682276
"version": "v1.0",
2277+
"display_name": "Repair Orders Fact",
22692278
},
22702279
},
22712280
],
22722281
},
22732282
{
22742283
"derived_expression": "SELECT SUM(price_sum_78a5eb43) / SUM(price_count_78a5eb43) FROM "
22752284
"default.repair_orders_fact",
2285+
"metric_expression": "SUM(price_sum_78a5eb43) / SUM(price_count_78a5eb43)",
22762286
"metric": {
22772287
"name": "default.avg_repair_price",
22782288
"version": "v1.0",
2289+
"display_name": "Avg Repair Price",
22792290
},
22802291
"required_measures": [
22812292
{
22822293
"measure_name": "price_count_78a5eb43",
22832294
"node": {
22842295
"name": "default.repair_orders_fact",
22852296
"version": "v1.0",
2297+
"display_name": "Repair Orders Fact",
22862298
},
22872299
},
22882300
{
22892301
"measure_name": "price_sum_78a5eb43",
22902302
"node": {
22912303
"name": "default.repair_orders_fact",
22922304
"version": "v1.0",
2305+
"display_name": "Repair Orders Fact",
22932306
},
22942307
},
22952308
],
22962309
},
22972310
{
22982311
"derived_expression": "SELECT sum(total_repair_cost_sum_9bdaf803) FROM "
22992312
"default.repair_orders_fact",
2313+
"metric_expression": "sum(total_repair_cost_sum_9bdaf803)",
23002314
"metric": {
23012315
"name": "default.total_repair_cost",
23022316
"version": "v1.0",
2317+
"display_name": "Total Repair Cost",
23032318
},
23042319
"required_measures": [
23052320
{
23062321
"measure_name": "total_repair_cost_sum_9bdaf803",
23072322
"node": {
23082323
"name": "default.repair_orders_fact",
23092324
"version": "v1.0",
2325+
"display_name": "Repair Orders Fact",
23102326
},
23112327
},
23122328
],
23132329
},
23142330
{
23152331
"derived_expression": "SELECT sum(price_discount_sum_017d55a8) FROM "
23162332
"default.repair_orders_fact",
2333+
"metric_expression": "sum(price_discount_sum_017d55a8)",
23172334
"metric": {
23182335
"name": "default.total_repair_order_discounts",
23192336
"version": "v1.0",
2337+
"display_name": "Total Repair Order Discounts",
23202338
},
23212339
"required_measures": [
23222340
{
23232341
"measure_name": "price_discount_sum_017d55a8",
23242342
"node": {
23252343
"name": "default.repair_orders_fact",
23262344
"version": "v1.0",
2345+
"display_name": "Repair Orders Fact",
23272346
},
23282347
},
23292348
],
@@ -2332,16 +2351,19 @@ async def test_cube_materialization_metadata(
23322351
"derived_expression": "SELECT sum(price_sum_78a5eb43) + sum(price_sum_78a5eb43) AS "
23332352
"default_DOT_double_total_repair_cost FROM "
23342353
"default.repair_order_details",
2354+
"metric_expression": "sum(price_sum_78a5eb43) + sum(price_sum_78a5eb43)",
23352355
"metric": {
23362356
"name": "default.double_total_repair_cost",
23372357
"version": "v1.0",
2358+
"display_name": "Double Total Repair Cost",
23382359
},
23392360
"required_measures": [
23402361
{
23412362
"measure_name": "price_sum_78a5eb43",
23422363
"node": {
23432364
"name": "default.repair_order_details",
23442365
"version": "v1.0",
2366+
"display_name": "default.roads.repair_order_details",
23452367
},
23462368
},
23472369
],
@@ -2550,6 +2572,7 @@ async def test_cube_materialization_metadata(
25502572
"node": {
25512573
"name": "default.repair_orders_fact",
25522574
"version": "v1.0",
2575+
"display_name": "Repair Orders Fact",
25532576
},
25542577
"output_table_name": "default_repair_orders_fact_v1_0_c9390406463b348e",
25552578
"query": mock.ANY,
@@ -2666,6 +2689,7 @@ async def test_cube_materialization_metadata(
26662689
"node": {
26672690
"name": "default.repair_order_details",
26682691
"version": "v1.0",
2692+
"display_name": "default.roads.repair_order_details",
26692693
},
26702694
"output_table_name": "default_repair_order_details_v1_0_5bf367d2fc7c255d",
26712695
"query": mock.ANY,
@@ -3081,6 +3105,7 @@ async def test_cube_materialization_metadata(
30813105
},
30823106
],
30833107
"node": {
3108+
"display_name": "Example Repairs Cube",
30843109
"name": "default.example_repairs_cube",
30853110
"version": "v1.0",
30863111
},

‎datajunction-server/tests/api/materializations_test.py‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -803,6 +803,7 @@ async def test_druid_cube_incremental(
803803
assert mat.cube == NodeNameVersion(
804804
name="default.repairs_cube__default_incremental",
805805
version="v1.0",
806+
display_name="Repairs Cube Default Incremental",
806807
)
807808
assert mat.dimensions == [
808809
"default.repair_orders_fact.order_date",
@@ -812,37 +813,50 @@ async def test_druid_cube_incremental(
812813
]
813814
assert mat.metrics == [
814815
CubeMetric(
815-
metric=NodeNameVersion(name="default.num_repair_orders", version="v1.0"),
816+
metric=NodeNameVersion(
817+
name="default.num_repair_orders",
818+
version="v1.0",
819+
display_name="Num Repair Orders",
820+
),
816821
required_measures=[
817822
MeasureKey(
818823
node=NodeNameVersion(
819824
name="default.repair_orders_fact",
820825
version=ANY_STRING,
826+
display_name="Repair Orders Fact",
821827
),
822828
measure_name="repair_order_id_count_0b7dfba0",
823829
),
824830
],
825831
derived_expression="SELECT SUM(repair_order_id_count_0b7dfba0)"
826832
" FROM default.repair_orders_fact",
833+
metric_expression="SUM(repair_order_id_count_0b7dfba0)",
827834
),
828835
CubeMetric(
829-
metric=NodeNameVersion(name="default.total_repair_cost", version="v1.0"),
836+
metric=NodeNameVersion(
837+
name="default.total_repair_cost",
838+
version="v1.0",
839+
display_name="Total Repair Cost",
840+
),
830841
required_measures=[
831842
MeasureKey(
832843
node=NodeNameVersion(
833844
name="default.repair_orders_fact",
834845
version=ANY_STRING,
846+
display_name="Repair Orders Fact",
835847
),
836848
measure_name="total_repair_cost_sum_9bdaf803",
837849
),
838850
],
839851
derived_expression="SELECT sum(total_repair_cost_sum_9bdaf803)"
840852
" FROM default.repair_orders_fact",
853+
metric_expression="sum(total_repair_cost_sum_9bdaf803)",
841854
),
842855
]
843856
assert mat.measures_materializations[0].node == NodeNameVersion(
844857
name="default.repair_orders_fact",
845858
version=ANY_STRING,
859+
display_name="Repair Orders Fact",
846860
)
847861
assert mat.measures_materializations[0].grain == [
848862
"default_DOT_repair_orders_fact_DOT_order_date",
@@ -942,6 +956,7 @@ async def test_druid_cube_incremental(
942956
assert mat.combiners[0].node == NodeNameVersion(
943957
name="default.repair_orders_fact",
944958
version=ANY_STRING,
959+
display_name="Repair Orders Fact",
945960
)
946961
assert mat.combiners[0].query is None
947962
assert mat.combiners[0].columns == [

‎datajunction-server/tests/service_clients_test.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -790,6 +790,7 @@ def test_materialize_cube(self, mocker: MockerFixture) -> None:
790790
metric=NodeNameVersion(
791791
name="default.num_repair_orders",
792792
version="v1.0",
793+
display_name=None,
793794
),
794795
required_measures=[
795796
MeasureKey(
@@ -800,12 +801,14 @@ def test_materialize_cube(self, mocker: MockerFixture) -> None:
800801
measure_name="count",
801802
),
802803
],
803-
derived_expression="SUM(count)",
804+
derived_expression="SELECT SUM(count) FROM default.repair_orders",
805+
metric_expression="SUM(count)",
804806
),
805807
CubeMetric(
806808
metric=NodeNameVersion(
807809
name="default.avg_repair_price",
808810
version="v1.0",
811+
display_name=None,
809812
),
810813
required_measures=[
811814
MeasureKey(
@@ -816,7 +819,8 @@ def test_materialize_cube(self, mocker: MockerFixture) -> None:
816819
measure_name="sum_price_123abc",
817820
),
818821
],
819-
derived_expression="SUM(sum_price_123abc)",
822+
derived_expression="SELECT SUM(sum_price_123abc) FROM default.repair_orders",
823+
metric_expression="SUM(sum_price_123abc)",
820824
),
821825
],
822826
measures_materializations=[],

0 commit comments

Comments
 (0)