Hydrate metrics in semantic manifest and manifest (#5556)

GitOrigin-RevId: e1c1655b29da3c1f26878c664640659d69993c8d

Author: eddowh

crates/dbt-parser/src/resolve/resolve_metrics.rs

@@ -41,6 +41,7 @@ pub struct MinimalProperties {
     pub unit_tests: BTreeMap<String, MinimalPropertiesEntry>,
     pub tests: BTreeMap<String, MinimalPropertiesEntry>,
     pub exposures: BTreeMap<String, MinimalPropertiesEntry>,
+    pub metrics: BTreeMap<String, MinimalPropertiesEntry>,
     pub saved_queries: BTreeMap<String, MinimalPropertiesEntry>,
     pub groups: BTreeMap<String, MinimalPropertiesEntry>,
 }
@@ -263,6 +264,37 @@ impl MinimalProperties {
                 );
             }
         }
+        if let Some(metrics) = other.metrics {
+            for metric_value in metrics {
+                let metric = into_typed_with_jinja::<MinimalSchemaValue, _>(
+                    io_args,
+                    metric_value.clone(),
+                    false,
+                    jinja_env,
+                    base_ctx,
+                    &[],
+                    dependency_package_name_from_ctx(jinja_env, base_ctx),
+                )?;
+                if let Some(existing_metric) = self.metrics.get_mut(&metric.name) {
+                    existing_metric
+                        .duplicate_paths
+                        .push(properties_path.to_path_buf());
+                } else {
+                    self.metrics.insert(
+                        metric.name.clone(),
+                        MinimalPropertiesEntry {
+                            name: validate_resource_name(&metric.name)?,
+                            name_span: Span::default(),
+                            relative_path: properties_path.to_path_buf(),
+                            schema_value: metric_value,
+                            table_value: None,
+                            version_info: None,
+                            duplicate_paths: vec![],
+                        },
+                    );
+                }
+            }
+        }
         if let Some(saved_queries) = other.saved_queries {
             for saved_query_value in saved_queries {
                 let saved_query = into_typed_with_jinja::<MinimalSchemaValue, _>(

crates/dbt-parser/src/resolve/resolve_properties.rs

@@ -9,15 +9,14 @@ use dbt_schemas::schemas::common::{DbtChecksum, Dimension, DimensionTypeParams,
 use dbt_schemas::schemas::dbt_column::{
     ColumnPropertiesDimension, ColumnPropertiesDimensionConfig, Entity, EntityConfig,
 };
+use dbt_schemas::schemas::manifest::metric::{MeasureAggregationParameters, NonAdditiveDimension};
 use dbt_schemas::schemas::manifest::semantic_model::{
-    DbtSemanticModel, DbtSemanticModelAttr, MeasureAggregationParameters, NodeRelation,
-    SemanticEntity, SemanticMeasure, SemanticModelDefaults,
+    DbtSemanticModel, DbtSemanticModelAttr, NodeRelation, SemanticEntity, SemanticMeasure,
+    SemanticModelDefaults,
 };
 use dbt_schemas::schemas::project::{DefaultTo, ModelConfig, SemanticModelConfig};
 use dbt_schemas::schemas::properties::ModelProperties;
-use dbt_schemas::schemas::properties::metrics_properties::{
-    AggregationType, MetricExpr, PercentileType,
-};
+use dbt_schemas::schemas::properties::metrics_properties::{AggregationType, PercentileType};
 use dbt_schemas::schemas::ref_and_source::DbtRef;
 use dbt_schemas::schemas::semantic_layer::semantic_manifest::SemanticLayerElementConfig;
 use dbt_schemas::schemas::{CommonAttributes, DbtModel, InternalDbtNodeAttributes};
@@ -159,10 +158,7 @@ pub async fn resolve_semantic_models(
             .map(|metric| {
                 SemanticMeasure {
                     name: metric.name.clone(),
-                    expr: metric.expr.clone().map(|expr| match expr {
-                        MetricExpr::String(ref str) => str.clone(),
-                        MetricExpr::Integer(ref int) => int.to_string(),
-                    }),
+                    expr: metric.expr.clone().map(String::from),
                     description: metric.description.clone(),
                     label: metric.label.clone(),
                     config: metric.config.as_ref().map(|c| SemanticLayerElementConfig {
@@ -186,7 +182,10 @@ pub async fn resolve_semantic_models(
                     } else {
                         None
                     },
-                    non_additive_dimension: metric.non_additive_dimension.clone(),
+                    non_additive_dimension: metric
+                        .non_additive_dimension
+                        .clone()
+                        .map(NonAdditiveDimension::from),
                     agg_time_dimension: Some(
                         metric
                             .agg_time_dimension

crates/dbt-parser/src/resolve/resolve_semantic_models.rs

@@ -558,8 +558,6 @@ pub async fn resolve_inner(
     nodes.exposures.extend(exposures);
     disabled_nodes.exposures.extend(disabled_exposures);
 
-    // dbg!(&nodes.clone().models);
-
     let (semantic_models, disabled_semantic_models) = resolve_semantic_models(
         arg,
         package,
@@ -578,7 +576,15 @@ pub async fn resolve_inner(
         .extend(disabled_semantic_models);
 
     let (metrics, disabled_metrics) = resolve_metrics(
-        // TODO: pass in typed_models_properties
+        arg,
+        package,
+        root_project_configs,
+        &mut min_properties.models.clone(),
+        &mut min_properties.metrics.clone(),
+        &typed_models_properties,
+        package_name,
+        &jinja_env,
+        &base_ctx,
     )
     .await?;
     nodes.metrics.extend(metrics);

crates/dbt-parser/src/resolver.rs

@@ -33,17 +33,17 @@ pub fn coalesce<T: Clone>(values: Vec<Option<T>>) -> Option<T> {
     None
 }
 
-/// generate the unique id for a model (can be made more extensible for each type of node)
+/// generate the unique id for a dbt resource (can be made more extensible for each type of node)
 pub fn get_unique_id(
-    model_name: &str,
+    resource_name: &str,
     package_name: &str,
     version: Option<String>,
     node_type: &str,
 ) -> String {
     if let Some(version) = version {
-        format!("{node_type}.{package_name}.{model_name}.v{version}")
+        format!("{node_type}.{package_name}.{resource_name}.v{version}")
     } else {
-        format!("{node_type}.{package_name}.{model_name}")
+        format!("{node_type}.{package_name}.{resource_name}")
     }
 }
 

crates/dbt-parser/src/utils.rs

@@ -58,7 +58,7 @@ pub mod schemas {
         #[allow(clippy::module_inception)]
         mod manifest;
         mod manifest_nodes;
-        pub(crate) mod metric;
+        pub mod metric;
         mod operation;
         pub mod postgres;
         pub mod saved_query;

crates/dbt-schemas/src/lib.rs

@@ -33,19 +33,6 @@ impl From<Vec<String>> for WhereFilterIntersection {
     }
 }
 
-impl From<StringOrArrayOfStrings> for WhereFilterIntersection {
-    fn from(source: StringOrArrayOfStrings) -> Self {
-        match source {
-            StringOrArrayOfStrings::String(s) => Self {
-                where_filters: vec![WhereFilter {
-                    where_sql_template: s,
-                }],
-            },
-            StringOrArrayOfStrings::ArrayOfStrings(arr) => arr.into(),
-        }
-    }
-}
-
 #[skip_serializing_none]
 #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, JsonSchema)]
 #[serde(rename_all = "snake_case")]

crates/dbt-schemas/src/schemas/manifest/common.rs

@@ -185,7 +185,12 @@ pub fn build_manifest(invocation_id: &str, resolver_state: &ResolverState) -> Db
             .iter()
             .map(|(id, semantic_model)| (id.clone(), (**semantic_model).clone().into()))
             .collect(),
-        metrics: BTreeMap::new(), // TODO: map from resolver_state.nodes
+        metrics: resolver_state
+            .nodes
+            .metrics
+            .iter()
+            .map(|(id, metric)| (id.clone(), (**metric).clone().into()))
+            .collect(),
         saved_queries: resolver_state
             .nodes
             .saved_queries
@@ -423,6 +428,24 @@ fn build_parent_and_child_maps(
         all_nodes.push((id.clone(), unit_test.__base_attr__.depends_on.clone()));
     }
 
+    for (id, semantic_model) in &nodes.semantic_models {
+        all_nodes.push((
+            id.clone(),
+            semantic_model.__semantic_model_attr__.depends_on.clone(),
+        ));
+    }
+
+    for (id, metric) in &nodes.metrics {
+        all_nodes.push((id.clone(), metric.__metric_attr__.depends_on.clone()));
+    }
+
+    for (id, saved_query) in &nodes.saved_queries {
+        all_nodes.push((
+            id.clone(),
+            saved_query.__saved_query_attr__.depends_on.clone(),
+        ));
+    }
+
     // Process all collected nodes
     for (node_id, depends_on) in all_nodes {
         // Initialize parent list for this node

crates/dbt-schemas/src/schemas/manifest/manifest.rs

@@ -14,12 +14,10 @@ use crate::schemas::{
     },
     dbt_column::DbtColumnRef,
     manifest::{
-        DbtOperation, DbtSavedQuery, DbtSemanticModel,
+        DbtMetric, DbtOperation, DbtSavedQuery, DbtSemanticModel,
         common::{DbtOwner, SourceFileMetadata, WhereFilterIntersection},
-        semantic_model::{
-            MeasureAggregationParameters, NodeRelation, SemanticEntity, SemanticMeasure,
-            SemanticModelDefaults,
-        },
+        metric::{MeasureAggregationParameters, MetricTypeParams, NonAdditiveDimension},
+        semantic_model::{NodeRelation, SemanticEntity, SemanticMeasure, SemanticModelDefaults},
     },
     nodes::TestMetadata,
     project::{
@@ -28,7 +26,7 @@ use crate::schemas::{
     },
     properties::{
         ModelConstraint, UnitTestOverrides,
-        metrics_properties::{AggregationType, NonAdditiveDimension, WindowChoice},
+        metrics_properties::{AggregationType, MetricType},
     },
     ref_and_source::{DbtRef, DbtSourceWrapper},
     semantic_layer::semantic_manifest::SemanticLayerElementConfig,
@@ -637,8 +635,8 @@ pub struct ManifestMetric {
     // Metric Specific Attributes
     pub label: String,
     #[serde(rename = "type")]
-    pub metric_type: crate::schemas::manifest::metric::MetricType,
-    pub type_params: crate::schemas::manifest::metric::MetricTypeParams,
+    pub metric_type: MetricType,
+    pub type_params: MetricTypeParams,
     pub filter: Option<WhereFilterIntersection>,
     pub metadata: Option<SourceFileMetadata>,
     pub time_granularity: Option<String>,
@@ -652,6 +650,41 @@ pub struct ManifestMetric {
     pub __other__: BTreeMap<String, YmlValue>,
 }
 
+impl From<DbtMetric> for ManifestMetric {
+    fn from(metric: DbtMetric) -> Self {
+        Self {
+            __common_attr__: ManifestCommonAttributes {
+                unique_id: metric.__common_attr__.unique_id,
+                name: metric.__common_attr__.name,
+                package_name: metric.__common_attr__.package_name,
+                fqn: metric.__common_attr__.fqn,
+                path: metric.__common_attr__.path,
+                original_file_path: metric.__common_attr__.original_file_path,
+                description: metric.__common_attr__.description,
+                tags: metric.__common_attr__.tags,
+                meta: metric.__common_attr__.meta,
+            },
+            __base_attr__: ManifestMetricNodeBaseAttributes {
+                depends_on: metric.__metric_attr__.depends_on,
+                refs: metric.__metric_attr__.refs,
+                sources: metric.__metric_attr__.sources,
+                unrendered_config: metric.__metric_attr__.unrendered_config,
+                created_at: metric.__metric_attr__.created_at,
+            },
+            label: metric.__metric_attr__.label.unwrap_or_default(),
+            metric_type: metric.__metric_attr__.metric_type,
+            type_params: metric.__metric_attr__.type_params,
+            filter: metric.__metric_attr__.filter,
+            metadata: metric.__metric_attr__.metadata,
+            time_granularity: metric.__metric_attr__.time_granularity.clone(),
+            group: metric.__metric_attr__.group.clone(),
+            config: metric.deprecated_config.into(),
+            __other__: metric.__other__,
+            metrics: vec![], // TODO: metric.__metric_attr__.metrics.clone(),
+        }
+    }
+}
+
 #[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
 pub struct ManifestMetricConfig {
     pub enabled: bool,
@@ -779,23 +812,6 @@ impl From<DbtSemanticModel> for ManifestSemanticModel {
     }
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ManifestNonAdditiveDimension {
-    pub name: String,
-    pub window_choice: WindowChoice,
-    pub window_groupings: Option<Vec<String>>,
-}
-
-impl From<NonAdditiveDimension> for ManifestNonAdditiveDimension {
-    fn from(dim: NonAdditiveDimension) -> Self {
-        Self {
-            name: dim.name,
-            window_choice: dim.window_agg,
-            window_groupings: dim.group_by,
-        }
-    }
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ManifestSemanticModelMeasure {
     pub name: String,
@@ -805,7 +821,7 @@ pub struct ManifestSemanticModelMeasure {
     pub create_metric: Option<bool>,
     pub expr: Option<String>,
     pub agg_params: Option<MeasureAggregationParameters>,
-    pub non_additive_dimension: Option<ManifestNonAdditiveDimension>,
+    pub non_additive_dimension: Option<NonAdditiveDimension>,
     pub agg_time_dimension: Option<String>,
     pub config: Option<SemanticLayerElementConfig>,
 }
@@ -820,9 +836,7 @@ impl From<SemanticMeasure> for ManifestSemanticModelMeasure {
             create_metric: measure.create_metric,
             expr: measure.expr,
             agg_params: measure.agg_params,
-            non_additive_dimension: measure
-                .non_additive_dimension
-                .map(ManifestNonAdditiveDimension::from),
+            non_additive_dimension: measure.non_additive_dimension,
             agg_time_dimension: measure.agg_time_dimension,
             config: measure.config,
         }