Emiel/add support for opt in metric properties (#31)

Add support for `include_experiment_computation` parameter

Author: emielver

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023-2024 Eppo
+Copyright (c) 2023-2025 Eppo
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -104,6 +104,14 @@ pip install -r requirements.txt
 pytest tests
 ```
 
+### Running the package
+
+```bash
+export EPPO_API_KEY="your-api-key"
+export EPPO_SYNC_TAG="your-sync-tag"
+python -m eppo_metrics_sync path/to/yaml/directory
+```
+
 ### Building and Publishing
 
 For package maintainers:

eppo_metrics_sync/eppo_metrics_sync.py

@@ -7,7 +7,8 @@
     unique_names,
     valid_fact_references,
     metric_aggregation_is_valid,
-    valid_guardrail_cutoff_signs
+    valid_guardrail_cutoff_signs,
+    valid_experiment_computation
 )
 
 from eppo_metrics_sync.dbt_model_parser import DbtModelParser
@@ -112,6 +113,7 @@ def validate(self):
         valid_fact_references(self)
         metric_aggregation_is_valid(self)
         valid_guardrail_cutoff_signs(self)
+        valid_experiment_computation(self)
 
         if self.validation_errors:
             error_count = len(self.validation_errors)

eppo_metrics_sync/schema/eppo_metric_schema.json

@@ -65,7 +65,7 @@
                                 },
                                 "column": {
                                     "description": "The column that contains this fact (if not specified, will use each record)",
-                                    "type":  ["string", "null"]
+                                    "type": ["string", "null"]
                                 },
                                 "description": {
                                     "description": "User-friendly description of the fact",
@@ -97,6 +97,10 @@
                                 "description": {
                                     "description": "User-friendly description of the fact property",
                                     "type": "string"
+                                },
+                                "include_experiment_computation": {
+                                    "description": "If true, this fact property will be used in experiment result computation",
+                                    "type": "boolean"
                                 }
                             }
                         }

eppo_metrics_sync/validation.py

@@ -69,6 +69,15 @@ def valid_fact_references(payload):
             str(', '.join(fact_references.difference(fact_names)))
         )
 
+def valid_experiment_computation(payload):
+    for fact_source in payload.fact_sources:
+        if 'properties' in fact_source:
+            for property in fact_source['properties']:
+                if 'include_experiment_computation' in property:
+                    if not isinstance(property['include_experiment_computation'], bool):
+                        payload.validation_errors.append(
+                            f"Invalid include_experiment_computation value. It must be a boolean value for property: {property['name']}"
+                        )
 
 def metric_aggregation_is_valid(payload):
     for m in payload.metrics:

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "eppo_metrics_sync"
-version = "0.1.5"
+version = "0.1.6"
 description = "Sync metrics to Eppo"
 readme = "README.md"
 requires-python = ">=3.7"

tests/test_validation.py

@@ -50,11 +50,9 @@ def test_valid_guardrail_cutoff_signs():
 
 
 """def test_unique_fact_property_names():
-
     eppo_metrics_sync = EppoMetricsSync(directory = None)
-    eppo_metrics_sync.load_yaml(
+    eppo_metrics_sync.load_eppo_yaml(
         path = test_yaml_dir + "/duplicated_fact_property_names.yaml")
-
     with pytest.raises(ValueError, match = "Fact property names are not unique: device"):
         eppo_metrics_sync.validate()"""
 
@@ -66,6 +64,12 @@ def test_invalid_fact_reference():
     with pytest.raises(ValueError, match=re.escape("Invalid fact reference(s): revenue")):
         eppo_metrics_sync.validate()
 
+def test_invalid_experiment_computation():
+    eppo_metrics_sync = EppoMetricsSync(directory=None)
+    eppo_metrics_sync.load_eppo_yaml(
+        path=test_yaml_dir + "/invalid_experiment_computation.yaml")
+    with pytest.raises(ValueError, match=re.escape("Invalid include_experiment_computation value. It must be a boolean value for property: Traffic Source")):
+        eppo_metrics_sync.validate()
 
 def test_invalid_winsorization_operation():
     test_agg = {

tests/yaml/invalid/invalid_experiment_computation.yaml

@@ -0,0 +1,33 @@
+fact_sources:
+- name: Purchase
+  sql: |
+    SELECT
+      events.*
+    FROM customer_db.onboarding.events as events
+    WHERE event_type = 'Revenue'
+  timestamp_column: TS
+  entities:
+  - entity_name: User
+    column: USER_ID
+  facts:
+  - name: Purchase # facts without column specified reflect "EACH RECORD"
+  - name: Purchase Revenue
+    column: EVENT_VALUE
+  properties:
+  - name: Traffic Source
+    column: TRAFFIC_SOURCE
+    include_experiment_computation: 0
+  - name: Country
+    column: COUNTRY
+    include_experiment_computation: true
+  - name: Browser
+    column: BROWSER
+    include_experiment_computation: true
+  reference_url: https://github.com/Eppo-exp/eppo-metrics-sync
+metrics:
+- name: Unique Purchase by User
+  entity: User # it would be nice if this was optional if there is exactly 1 entity defined above
+  is_guardrail: true
+  numerator:
+    fact_name: Purchase
+    operation: distinct_entity