Add new-syle YAML (Parsed) Metric

Author: theyostalservice

core/dbt/artifacts/resources/v1/metric.py

@@ -83,9 +83,11 @@ class ConstantPropertyInput(dbtClassMixin):
 
 @dataclass
 class ConversionTypeParams(dbtClassMixin):
-    base_measure: MetricInputMeasure
-    conversion_measure: MetricInputMeasure
     entity: str
+    base_measure: Optional[MetricInputMeasure] = None
+    conversion_measure: Optional[MetricInputMeasure] = None
+    base_metric: Optional[MetricInput] = None
+    conversion_metric: Optional[MetricInput] = None
     calculation: ConversionCalculationType = ConversionCalculationType.CONVERSION_RATE
     window: Optional[MetricTimeWindow] = None
     constant_properties: Optional[List[ConstantPropertyInput]] = None
@@ -106,12 +108,13 @@ class MetricAggregationParams(dbtClassMixin):
     agg_params: Optional[MeasureAggregationParameters] = None
     agg_time_dimension: Optional[str] = None
     non_additive_dimension: Optional[NonAdditiveDimension] = None
-    expr: Optional[str] = None
 
 
 @dataclass
 class MetricTypeParams(dbtClassMixin):
+    # Only used in old-style YAML
     measure: Optional[MetricInputMeasure] = None
+    # Only used in new-style JSON
     input_measures: List[MetricInputMeasure] = field(default_factory=list)
     numerator: Optional[MetricInput] = None
     denominator: Optional[MetricInput] = None
@@ -123,7 +126,12 @@ class MetricTypeParams(dbtClassMixin):
     metrics: Optional[List[MetricInput]] = None
     conversion_type_params: Optional[ConversionTypeParams] = None
     cumulative_type_params: Optional[CumulativeTypeParams] = None
+
+    # Only used in v2 YAML
     metric_aggregation_params: Optional[MetricAggregationParams] = None
+    fill_nulls_with: Optional[int] = None
+    join_to_timespine: Optional[bool] = None
+    is_private: Optional[bool] = None  # populated by "hidden" field in YAML
 
 
 @dataclass
@@ -148,8 +156,6 @@ class Metric(GraphResource):
     metadata: Optional[SourceFileMetadata] = None
     time_granularity: Optional[str] = None
     resource_type: Literal[NodeType.Metric]
-    meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta())
-    tags: List[str] = field(default_factory=list)
     config: MetricConfig = field(default_factory=MetricConfig)
     unrendered_config: Dict[str, Any] = field(default_factory=dict)
     sources: List[List[str]] = field(default_factory=list)
@@ -159,14 +165,18 @@ class Metric(GraphResource):
     created_at: float = field(default_factory=lambda: time.time())
     group: Optional[str] = None
 
+    # These fields are only used in v1 metrics.
+    meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta())
+    tags: List[str] = field(default_factory=list)
+
+    @property
+    def input_metrics(self) -> List[MetricInput]:
+        return self.type_params.metrics or []
+
     @property
     def input_measures(self) -> List[MetricInputMeasure]:
         return self.type_params.input_measures
 
     @property
     def measure_references(self) -> List[MeasureReference]:
         return [x.measure_reference() for x in self.input_measures]
-
-    @property
-    def input_metrics(self) -> List[MetricInput]:
-        return self.type_params.metrics or []

core/dbt/contracts/graph/unparsed.py

@@ -580,9 +580,12 @@ class UnparsedMetricInput(dbtClassMixin):
 
 @dataclass
 class UnparsedConversionTypeParams(dbtClassMixin):
-    base_measure: Union[UnparsedMetricInputMeasure, str]
-    conversion_measure: Union[UnparsedMetricInputMeasure, str]
     entity: str
+    # TODO : add validation that we have input measures XOR input metrics
+    base_measure: Optional[Union[UnparsedMetricInputMeasure, str]] = None
+    conversion_measure: Optional[Union[UnparsedMetricInputMeasure, str]] = None
+    base_metric: Optional[Union[UnparsedMetricInput, str]] = None
+    conversion_metric: Optional[Union[UnparsedMetricInput, str]] = None
     calculation: str = (
         ConversionCalculationType.CONVERSION_RATE.value
     )  # ConversionCalculationType Enum
@@ -674,30 +677,30 @@ class UnparsedMetricV2(UnparsedMetricBase):
 
     join_to_timespine: Optional[bool] = None
     fill_nulls_with: Optional[int] = None
-    expr: Optional[Union[str, int]] = None
+    expr: Optional[Union[str, bool]] = None
 
     non_additive_dimension: Optional[UnparsedNonAdditiveDimensionV2] = None
     agg_time_dimension: Optional[str] = None
 
     # For cumulative metrics
     window: Optional[str] = None
     grain_to_date: Optional[str] = None
-    period_agg: Optional[str] = None
+    period_agg: str = PeriodAggregation.FIRST.value
     input_metric: Optional[Union[str, Dict[str, Any]]] = None
 
     # For ratio metrics
-    numerator: Optional[Union[str, Dict[str, Any]]] = None
-    denominator: Optional[Union[str, Dict[str, Any]]] = None
+    numerator: Optional[Union[UnparsedMetricInput, str]] = None
+    denominator: Optional[Union[UnparsedMetricInput, str]] = None
 
     # For derived metrics
-    input_metrics: Optional[List[Dict[str, Any]]] = None
+    input_metrics: Optional[List[Union[UnparsedMetricInput, str]]] = None
 
     # For conversion metrics
     entity: Optional[str] = None
     calculation: Optional[str] = None
-    base_metric: Optional[Union[str, Dict[str, Any]]] = None
-    conversion_metric: Optional[Union[str, Dict[str, Any]]] = None
-    constant_properties: Optional[List[Dict[str, Any]]] = None
+    base_metric: Optional[Union[UnparsedMetricInput, str]] = None
+    conversion_metric: Optional[Union[UnparsedMetricInput, str]] = None
+    constant_properties: Optional[List[ConstantPropertyInput]] = None
 
 
 @dataclass

core/dbt/parser/schema_yaml_readers.py

@@ -46,9 +46,11 @@
     UnparsedGroup,
     UnparsedMeasure,
     UnparsedMetric,
+    UnparsedMetricBase,
     UnparsedMetricInput,
     UnparsedMetricInputMeasure,
     UnparsedMetricTypeParams,
+    UnparsedMetricV2,
     UnparsedNonAdditiveDimension,
     UnparsedQueryParams,
     UnparsedSavedQuery,
@@ -72,7 +74,7 @@
 
 
 def parse_where_filter(
-    where: Optional[Union[List[str], str]]
+    where: Optional[Union[List[str], str]],
 ) -> Optional[WhereFilterIntersection]:
     if where is None:
         return None
@@ -306,11 +308,19 @@ def _get_metric_inputs(
 
         return metric_inputs
 
-    def _get_optional_conversion_type_params(
+    def _get_optional_v1_conversion_type_params(
         self, unparsed: Optional[UnparsedConversionTypeParams]
     ) -> Optional[ConversionTypeParams]:
         if unparsed is None:
             return None
+        if unparsed.base_measure is None:
+            raise ValidationError(
+                "base_measure is required for conversion metrics that use type_params."
+            )
+        if unparsed.conversion_measure is None:
+            raise ValidationError(
+                "conversion_measure is required for conversion metrics that use type_params."
+            )
         return ConversionTypeParams(
             base_measure=self._get_input_measure(unparsed.base_measure),
             conversion_measure=self._get_input_measure(unparsed.conversion_measure),
@@ -320,7 +330,30 @@ def _get_optional_conversion_type_params(
             constant_properties=unparsed.constant_properties,
         )
 
-    def _get_optional_cumulative_type_params(
+    def _get_optional_v2_conversion_type_params(
+        self,
+        unparsed_metric: UnparsedMetricV2,
+    ) -> Optional[ConversionTypeParams]:
+        if MetricType(unparsed_metric.type) is not MetricType.CONVERSION:
+            return None
+        # TODO: shoudl we also make a best effort to validate this in the UNPARSED class?
+        # It'd violate DRY, but it is probably the more appropriate place to do the validation.
+        if unparsed_metric.base_metric is None:
+            raise ValidationError("base_metric is required for cumulative metrics.")
+        if unparsed_metric.conversion_metric is None:
+            raise ValidationError("conversion_metric is required for cumulative metrics.")
+        if unparsed_metric.entity is None:
+            raise ValidationError("entity is required for conversion metrics.")
+        return ConversionTypeParams(
+            base_metric=self._get_metric_input(unparsed_metric.base_metric),
+            conversion_metric=self._get_metric_input(unparsed_metric.conversion_metric),
+            entity=unparsed_metric.entity,
+            calculation=ConversionCalculationType(unparsed_metric.calculation),
+            window=self._get_optional_time_window(unparsed_metric.window),
+            constant_properties=unparsed_metric.constant_properties,
+        )
+
+    def _get_optional_v1_cumulative_type_params(
         self, unparsed_metric: UnparsedMetric
     ) -> Optional[CumulativeTypeParams]:
         unparsed_type_params = unparsed_metric.type_params
@@ -353,36 +386,71 @@ def _get_optional_cumulative_type_params(
 
         return None
 
-    def _get_metric_type_params(self, unparsed_metric: UnparsedMetric) -> MetricTypeParams:
-        type_params = unparsed_metric.type_params
-
-        grain_to_date: Optional[TimeGranularity] = None
-        if type_params.grain_to_date is not None:
-            # This should've been changed to a string (to support custom grain), but since this
-            # is a legacy field waiting to be deprecated, we will not support custom grain here
-            # in order to force customers off of using this field. The field to use should be
-            # `cumulative_type_params.grain_to_date`
-            grain_to_date = TimeGranularity(type_params.grain_to_date)
-
-        return MetricTypeParams(
-            measure=self._get_optional_input_measure(type_params.measure),
-            numerator=self._get_optional_metric_input(type_params.numerator),
-            denominator=self._get_optional_metric_input(type_params.denominator),
-            expr=str(type_params.expr) if type_params.expr is not None else None,
-            window=self._get_optional_time_window(type_params.window),
-            grain_to_date=grain_to_date,
-            metrics=self._get_metric_inputs(type_params.metrics),
-            conversion_type_params=self._get_optional_conversion_type_params(
-                type_params.conversion_type_params
-            ),
-            cumulative_type_params=self._get_optional_cumulative_type_params(
-                unparsed_metric=unparsed_metric,
-            ),
-            # input measures are calculated via metric processing post parsing
-            # input_measures=?,
+    def _get_optional_v2_cumulative_type_params(
+        self,
+        unparsed_metric: UnparsedMetricV2,
+    ) -> Optional[CumulativeTypeParams]:
+        if MetricType(unparsed_metric.type) is not MetricType.CUMULATIVE:
+            return None
+        return CumulativeTypeParams(
+            window=self._get_optional_time_window(unparsed_metric.window),
+            grain_to_date=unparsed_metric.grain_to_date,
+            period_agg=self._get_period_agg(unparsed_metric.period_agg),
         )
 
-    def parse_metric(self, unparsed: UnparsedMetric, generated_from: Optional[str] = None) -> None:
+    def _get_metric_type_params(self, unparsed_metric: UnparsedMetricBase) -> MetricTypeParams:
+        if isinstance(unparsed_metric, UnparsedMetric):
+            type_params = unparsed_metric.type_params
+
+            grain_to_date: Optional[TimeGranularity] = None
+            if type_params.grain_to_date is not None:
+                # This should've been changed to a string (to support custom grain), but since this
+                # is a legacy field waiting to be deprecated, we will not support custom grain here
+                # in order to force customers off of using this field. The field to use should be
+                # `cumulative_type_params.grain_to_date`
+                grain_to_date = TimeGranularity(type_params.grain_to_date)
+
+            return MetricTypeParams(
+                measure=self._get_optional_input_measure(type_params.measure),
+                numerator=self._get_optional_metric_input(type_params.numerator),
+                denominator=self._get_optional_metric_input(type_params.denominator),
+                expr=str(type_params.expr) if type_params.expr is not None else None,
+                window=self._get_optional_time_window(type_params.window),
+                grain_to_date=grain_to_date,
+                metrics=self._get_metric_inputs(type_params.metrics),
+                conversion_type_params=self._get_optional_v1_conversion_type_params(
+                    type_params.conversion_type_params
+                ),
+                cumulative_type_params=self._get_optional_v1_cumulative_type_params(
+                    unparsed_metric=unparsed_metric,
+                ),
+                # input measures are calculated via metric processing post parsing
+                # input_measures=?,
+            )
+        elif isinstance(unparsed_metric, UnparsedMetricV2):
+            # if unparsed_metric.type.lower() == MetricType.CONVERSION.value:
+            return MetricTypeParams(
+                numerator=self._get_optional_metric_input(unparsed_metric.numerator),
+                denominator=self._get_optional_metric_input(unparsed_metric.denominator),
+                expr=str(unparsed_metric.expr) if unparsed_metric.expr is not None else None,
+                window=self._get_optional_time_window(unparsed_metric.window),
+                metrics=self._get_metric_inputs(unparsed_metric.input_metrics),
+                conversion_type_params=self._get_optional_v2_conversion_type_params(
+                    unparsed_metric=unparsed_metric,
+                ),
+                cumulative_type_params=self._get_optional_v2_cumulative_type_params(
+                    unparsed_metric=unparsed_metric,
+                ),
+            )
+        else:
+            raise DbtInternalError(
+                f"Tried to parse type params for a {type(unparsed_metric)}, but expected "
+                "an UnparsedMetric or UnparsedMetricV2",
+            )
+
+    def parse_metric(
+        self, unparsed: UnparsedMetricBase, generated_from: Optional[str] = None
+    ) -> None:
         package_name = self.project.project_name
         unique_id = f"{NodeType.Metric}.{package_name}.{unparsed.name}"
         path = self.yaml.path.relative_path
@@ -411,10 +479,23 @@ def parse_metric(self, unparsed: UnparsedMetric, generated_from: Optional[str] =
                 f"Calculated a {type(config)} for a metric, but expected a MetricConfig"
             )
 
-        # If we have meta in the config, copy to node level, for backwards
-        # compatibility with earlier node-only config.
-        if "meta" in config and config["meta"]:
-            unparsed.meta = config["meta"]
+        if isinstance(unparsed, UnparsedMetric):
+            # If we have meta in the config, copy to node level, for backwards
+            # compatibility with earlier node-only config.
+            if "meta" in config and config["meta"]:
+                unparsed.meta = config["meta"]
+            meta = unparsed.meta
+            tags = unparsed.tags
+        elif isinstance(unparsed, UnparsedMetricV2):
+            # V2 Metrics do not have a top-level meta field; this should be part of
+            # the config.
+            meta = {}
+            tags = []
+        else:
+            raise DbtInternalError(
+                f"Tried to parse a {type(unparsed)} into a metric, but expected "
+                "an UnparsedMetric or UnparsedMetricV2",
+            )
 
         parsed = Metric(
             resource_type=NodeType.Metric,
@@ -425,13 +506,13 @@ def parse_metric(self, unparsed: UnparsedMetric, generated_from: Optional[str] =
             fqn=fqn,
             name=unparsed.name,
             description=unparsed.description,
-            label=unparsed.label,
+            label=unparsed.label or unparsed.name,
             type=MetricType(unparsed.type),
             type_params=self._get_metric_type_params(unparsed),
             time_granularity=unparsed.time_granularity,
             filter=parse_where_filter(unparsed.filter),
-            meta=unparsed.meta,
-            tags=unparsed.tags,
+            meta=meta,
+            tags=tags,
             config=config,
             unrendered_config=unrendered_config,
             group=config.group,
@@ -445,7 +526,7 @@ def parse_metric(self, unparsed: UnparsedMetric, generated_from: Optional[str] =
             self.manifest.add_disabled(self.yaml.file, parsed)
 
     def _generate_metric_config(
-        self, target: UnparsedMetric, fqn: List[str], package_name: str, rendered: bool
+        self, target: UnparsedMetricBase, fqn: List[str], package_name: str, rendered: bool
     ):
         generator: BaseContextConfigGenerator
         if rendered:
@@ -470,12 +551,20 @@ def _generate_metric_config(
 
     def parse(self) -> None:
         for data in self.get_key_dicts():
-            try:
-                UnparsedMetric.validate(data)
-                unparsed = UnparsedMetric.from_dict(data)
-
-            except (ValidationError, JSONValidationError) as exc:
-                raise YamlParseDictError(self.yaml.path, self.key, data, exc)
+            # The main differentiator of old-style yaml and new-style is "type_params",
+            # so if that is missing, we'll assume you're using the newer yaml.
+            if "type_params" in data:
+                try:
+                    UnparsedMetric.validate(data)
+                    unparsed = UnparsedMetric.from_dict(data)
+                except (ValidationError, JSONValidationError) as exc:
+                    raise YamlParseDictError(self.yaml.path, self.key, data, exc)
+            else:
+                try:
+                    UnparsedMetricV2.validate(data)
+                    unparsed = UnparsedMetricV2.from_dict(data)
+                except (ValidationError, JSONValidationError) as exc:
+                    raise YamlParseDictError(self.yaml.path, self.key, data, exc)
             self.parse_metric(unparsed)
 
 

tests/functional/semantic_models/test_semantic_model_parsing.py

@@ -15,6 +15,8 @@
     schema_yml,
 )
 
+# TODO ADD TESTS HERE FOR V2 Semantic Models
+
 
 class TestSemanticModelParsing:
     @pytest.fixture(scope="class")

tests/unit/contracts/graph/test_unparsed.py

@@ -1016,6 +1016,7 @@ def test_ok(self):
         dct = self.get_ok_dict()
         # add defaults:
         dct["hidden"] = False
+        dct["period_agg"] = "first"
         self.assert_symmetric(metric, dct)
         pickle.loads(pickle.dumps(metric))