CDD-2983 reject validation with missing is public with override option

ingestion/data_transfer_models/headline.py

@@ -7,10 +7,9 @@
 
 import validation
 from ingestion.utils import type_hints
-from metrics.api.settings.auth import AUTH_ENABLED
+from metrics.api.settings.auth import ALLOW_MISSING_IS_PUBLIC_FIELD
 from validation.data_transfer_models.base import (
     IncomingBaseDataModel,
-    NonPublicDataSentToPublicIngestionError,
 )
 
 
@@ -134,14 +133,6 @@ def validate_confidence_intervals(self) -> Self:
         )
         return self
 
-    @model_validator(mode="after")
-    def invalidate_non_public_data_for_public_ingestion(self) -> Self:
-        """Checks that if this is a public instance of the product then `is_public=False` data is invalidated."""
-        if not AUTH_ENABLED and not self.is_public:
-            raise NonPublicDataSentToPublicIngestionError
-
-        return self
-
 
 class HeadlineDTO(IncomingBaseDataModel):
     data: list[InboundHeadlineSpecificFields]
@@ -183,7 +174,11 @@ def _build_enriched_headline_specific_fields(
             upper_confidence=individual_time_series.get("upper_confidence", None),
             metric_value=individual_time_series["metric_value"],
             lower_confidence=individual_time_series.get("lower_confidence", None),
-            is_public=individual_time_series.get("is_public", True),
+            is_public=(
+                individual_time_series["is_public"]
+                if not ALLOW_MISSING_IS_PUBLIC_FIELD
+                else individual_time_series.get("is_public", True)
+            ),
         )
         for individual_time_series in source_data["data"]
         if individual_time_series["metric_value"] is not None

ingestion/data_transfer_models/time_series.py

@@ -1,15 +1,13 @@
 import datetime
-from typing import Self
 
-from pydantic import BaseModel, field_validator, model_validator
+from pydantic import BaseModel, field_validator
 from pydantic.fields import Field
 
 import validation
 from ingestion.utils import type_hints
-from metrics.api.settings.auth import AUTH_ENABLED
+from metrics.api.settings.auth import ALLOW_MISSING_IS_PUBLIC_FIELD
 from validation.data_transfer_models.base import (
     IncomingBaseDataModel,
-    NonPublicDataSentToPublicIngestionError,
 )
 
 
@@ -43,14 +41,6 @@ def cast_embargo_to_uk_timezone(
         """
         return validation.cast_date_to_uk_timezone(date_value=embargo)
 
-    @model_validator(mode="after")
-    def invalidate_non_public_data_for_public_ingestion(self) -> Self:
-        """Checks that if this is a public instance of the product then `is_public=False` data is invalidated."""
-        if not AUTH_ENABLED and not self.is_public:
-            raise NonPublicDataSentToPublicIngestionError
-
-        return self
-
 
 class TimeSeriesDTO(IncomingBaseDataModel):
     metric_frequency: str
@@ -128,7 +118,11 @@ def _build_enriched_time_series_specific_fields(
                 "in_reporting_delay_period", False
             ),
             force_write=individual_time_series.get("force_write", False),
-            is_public=individual_time_series.get("is_public", True),
+            is_public=(
+                individual_time_series["is_public"]
+                if not ALLOW_MISSING_IS_PUBLIC_FIELD
+                else individual_time_series.get("is_public", True)
+            ),
         )
         for individual_time_series in source_data["time_series"]
         if individual_time_series["metric_value"] is not None

metrics/api/settings/auth.py

@@ -5,4 +5,12 @@ def is_auth_enabled() -> bool:
     return str(os.environ.get("AUTH_ENABLED", "")).lower() in {"true", "1"}
 
 
+def is_allow_missing_is_public_field() -> bool:
+    return str(os.environ.get("ALLOW_MISSING_IS_PUBLIC_FIELD", "")).lower() in {
+        "true",
+        "1",
+    }
+
+
 AUTH_ENABLED = is_auth_enabled()
+ALLOW_MISSING_IS_PUBLIC_FIELD = is_allow_missing_is_public_field()

source_data/6-in-1_coverage_coverageByYear_CTRY_E92000001_all_all_12m.json

@@ -1 +1 @@
-{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"England","geography_code":"E92000001","age":"all","sex":"all","stratum":"12m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":93.61,"embargo":null},{"epiweek":13,"date":"2011-03-31","metric_value":94.15,"embargo":null},{"epiweek":13,"date":"2012-03-31","metric_value":94.67,"embargo":null},{"epiweek":13,"date":"2013-03-31","metric_value":94.74,"embargo":null},{"epiweek":14,"date":"2014-03-31","metric_value":94.34,"embargo":null},{"epiweek":14,"date":"2015-03-31","metric_value":94.23,"embargo":null},{"epiweek":13,"date":"2016-03-31","metric_value":93.56,"embargo":null},{"epiweek":13,"date":"2017-03-31","metric_value":93.41,"embargo":null},{"epiweek":13,"date":"2018-03-31","metric_value":93.12,"embargo":null},{"epiweek":13,"date":"2019-03-31","metric_value":92.09,"embargo":null},{"epiweek":14,"date":"2020-03-31","metric_value":92.57,"embargo":null},{"epiweek":13,"date":"2021-03-31","metric_value":92.04,"embargo":null},{"epiweek":13,"date":"2022-03-31","metric_value":91.84,"embargo":null},{"epiweek":13,"date":"2023-03-31","metric_value":91.8,"embargo":null},{"epiweek":13,"date":"2024-03-31","metric_value":91.22,"embargo":null}],"refresh_date":"2025-08-01 11:00:53"}
+{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"England","geography_code":"E92000001","age":"all","sex":"all","stratum":"12m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":93.61,"embargo":null,"is_public":true},{"epiweek":13,"date":"2011-03-31","metric_value":94.15,"embargo":null,"is_public":true},{"epiweek":13,"date":"2012-03-31","metric_value":94.67,"embargo":null,"is_public":true},{"epiweek":13,"date":"2013-03-31","metric_value":94.74,"embargo":null,"is_public":true},{"epiweek":14,"date":"2014-03-31","metric_value":94.34,"embargo":null,"is_public":true},{"epiweek":14,"date":"2015-03-31","metric_value":94.23,"embargo":null,"is_public":true},{"epiweek":13,"date":"2016-03-31","metric_value":93.56,"embargo":null,"is_public":true},{"epiweek":13,"date":"2017-03-31","metric_value":93.41,"embargo":null,"is_public":true},{"epiweek":13,"date":"2018-03-31","metric_value":93.12,"embargo":null,"is_public":true},{"epiweek":13,"date":"2019-03-31","metric_value":92.09,"embargo":null,"is_public":true},{"epiweek":14,"date":"2020-03-31","metric_value":92.57,"embargo":null,"is_public":true},{"epiweek":13,"date":"2021-03-31","metric_value":92.04,"embargo":null,"is_public":true},{"epiweek":13,"date":"2022-03-31","metric_value":91.84,"embargo":null,"is_public":true},{"epiweek":13,"date":"2023-03-31","metric_value":91.8,"embargo":null,"is_public":true},{"epiweek":13,"date":"2024-03-31","metric_value":91.22,"embargo":null,"is_public":true}],"refresh_date":"2025-08-01 11:00:53"}

source_data/6-in-1_coverage_coverageByYear_CTRY_E92000001_all_all_24m.json

@@ -1 +1 @@
-{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"England","geography_code":"E92000001","age":"all","sex":"all","stratum":"24m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":95.29,"embargo":null},{"epiweek":13,"date":"2011-03-31","metric_value":95.98,"embargo":null},{"epiweek":13,"date":"2012-03-31","metric_value":96.14,"embargo":null},{"epiweek":13,"date":"2013-03-31","metric_value":96.3,"embargo":null},{"epiweek":14,"date":"2014-03-31","metric_value":96.14,"embargo":null},{"epiweek":14,"date":"2015-03-31","metric_value":95.74,"embargo":null},{"epiweek":13,"date":"2016-03-31","metric_value":95.24,"embargo":null},{"epiweek":13,"date":"2017-03-31","metric_value":95.15,"embargo":null},{"epiweek":13,"date":"2018-03-31","metric_value":95.1,"embargo":null},{"epiweek":13,"date":"2019-03-31","metric_value":94.17,"embargo":null},{"epiweek":14,"date":"2020-03-31","metric_value":93.85,"embargo":null},{"epiweek":13,"date":"2021-03-31","metric_value":93.85,"embargo":null},{"epiweek":13,"date":"2022-03-31","metric_value":93.04,"embargo":null},{"epiweek":13,"date":"2023-03-31","metric_value":92.62,"embargo":null},{"epiweek":13,"date":"2024-03-31","metric_value":92.44,"embargo":null}],"refresh_date":"2025-08-01 11:00:53"}
+{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"England","geography_code":"E92000001","age":"all","sex":"all","stratum":"24m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":95.29,"embargo":null,"is_public":true},{"epiweek":13,"date":"2011-03-31","metric_value":95.98,"embargo":null,"is_public":true},{"epiweek":13,"date":"2012-03-31","metric_value":96.14,"embargo":null,"is_public":true},{"epiweek":13,"date":"2013-03-31","metric_value":96.3,"embargo":null,"is_public":true},{"epiweek":14,"date":"2014-03-31","metric_value":96.14,"embargo":null,"is_public":true},{"epiweek":14,"date":"2015-03-31","metric_value":95.74,"embargo":null,"is_public":true},{"epiweek":13,"date":"2016-03-31","metric_value":95.24,"embargo":null,"is_public":true},{"epiweek":13,"date":"2017-03-31","metric_value":95.15,"embargo":null,"is_public":true},{"epiweek":13,"date":"2018-03-31","metric_value":95.1,"embargo":null,"is_public":true},{"epiweek":13,"date":"2019-03-31","metric_value":94.17,"embargo":null,"is_public":true},{"epiweek":14,"date":"2020-03-31","metric_value":93.85,"embargo":null,"is_public":true},{"epiweek":13,"date":"2021-03-31","metric_value":93.85,"embargo":null,"is_public":true},{"epiweek":13,"date":"2022-03-31","metric_value":93.04,"embargo":null,"is_public":true},{"epiweek":13,"date":"2023-03-31","metric_value":92.62,"embargo":null,"is_public":true},{"epiweek":13,"date":"2024-03-31","metric_value":92.44,"embargo":null,"is_public":true}],"refresh_date":"2025-08-01 11:00:53"}

source_data/6-in-1_coverage_coverageByYear_CTRY_E92000001_all_all_5y.json

@@ -1 +1 @@
-{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"England","geography_code":"E92000001","age":"all","sex":"all","stratum":"5y","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2016-03-31","metric_value":95.57,"embargo":null},{"epiweek":13,"date":"2017-03-31","metric_value":95.59,"embargo":null},{"epiweek":13,"date":"2018-03-31","metric_value":95.56,"embargo":null},{"epiweek":13,"date":"2019-03-31","metric_value":95.04,"embargo":null},{"epiweek":14,"date":"2020-03-31","metric_value":95.24,"embargo":null},{"epiweek":13,"date":"2021-03-31","metric_value":95.24,"embargo":null},{"epiweek":13,"date":"2022-03-31","metric_value":94.39,"embargo":null},{"epiweek":13,"date":"2023-03-31","metric_value":93.2,"embargo":null},{"epiweek":13,"date":"2024-03-31","metric_value":92.62,"embargo":null},{"epiweek":14,"date":"2014-03-31","metric_value":95.6,"embargo":null},{"epiweek":14,"date":"2015-03-31","metric_value":95.56,"embargo":null}],"refresh_date":"2025-08-01 11:00:53"}
+{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"England","geography_code":"E92000001","age":"all","sex":"all","stratum":"5y","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2016-03-31","metric_value":95.57,"embargo":null,"is_public":true},{"epiweek":13,"date":"2017-03-31","metric_value":95.59,"embargo":null,"is_public":true},{"epiweek":13,"date":"2018-03-31","metric_value":95.56,"embargo":null,"is_public":true},{"epiweek":13,"date":"2019-03-31","metric_value":95.04,"embargo":null,"is_public":true},{"epiweek":14,"date":"2020-03-31","metric_value":95.24,"embargo":null,"is_public":true},{"epiweek":13,"date":"2021-03-31","metric_value":95.24,"embargo":null,"is_public":true},{"epiweek":13,"date":"2022-03-31","metric_value":94.39,"embargo":null,"is_public":true},{"epiweek":13,"date":"2023-03-31","metric_value":93.2,"embargo":null,"is_public":true},{"epiweek":13,"date":"2024-03-31","metric_value":92.62,"embargo":null,"is_public":true},{"epiweek":14,"date":"2014-03-31","metric_value":95.6,"embargo":null,"is_public":true},{"epiweek":14,"date":"2015-03-31","metric_value":95.56,"embargo":null,"is_public":true}],"refresh_date":"2025-08-01 11:00:53"}

source_data/6-in-1_coverage_coverageByYear_CTRY_N92000002_all_all_12m.json

@@ -1 +1 @@
-{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"Northern Ireland","geography_code":"N92000002","age":"all","sex":"all","stratum":"12m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":97.77,"embargo":null},{"epiweek":13,"date":"2011-03-31","metric_value":97.51,"embargo":null},{"epiweek":13,"date":"2012-03-31","metric_value":97.57,"embargo":null},{"epiweek":13,"date":"2013-03-31","metric_value":97.56,"embargo":null},{"epiweek":14,"date":"2014-03-31","metric_value":97.48,"embargo":null},{"epiweek":14,"date":"2015-03-31","metric_value":96.84,"embargo":null},{"epiweek":13,"date":"2016-03-31","metric_value":97.22,"embargo":null},{"epiweek":13,"date":"2017-03-31","metric_value":97.04,"embargo":null},{"epiweek":13,"date":"2018-03-31","metric_value":96.2,"embargo":null},{"epiweek":13,"date":"2019-03-31","metric_value":94.52,"embargo":null},{"epiweek":14,"date":"2020-03-31","metric_value":94.45,"embargo":null},{"epiweek":13,"date":"2021-03-31","metric_value":94.53,"embargo":null},{"epiweek":13,"date":"2022-03-31","metric_value":93.52,"embargo":null},{"epiweek":13,"date":"2023-03-31","metric_value":93.1,"embargo":null},{"epiweek":13,"date":"2024-03-31","metric_value":91.8,"embargo":null}],"refresh_date":"2025-08-01 11:00:53"}
+{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"Northern Ireland","geography_code":"N92000002","age":"all","sex":"all","stratum":"12m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":97.77,"embargo":null,"is_public":true},{"epiweek":13,"date":"2011-03-31","metric_value":97.51,"embargo":null,"is_public":true},{"epiweek":13,"date":"2012-03-31","metric_value":97.57,"embargo":null,"is_public":true},{"epiweek":13,"date":"2013-03-31","metric_value":97.56,"embargo":null,"is_public":true},{"epiweek":14,"date":"2014-03-31","metric_value":97.48,"embargo":null,"is_public":true},{"epiweek":14,"date":"2015-03-31","metric_value":96.84,"embargo":null,"is_public":true},{"epiweek":13,"date":"2016-03-31","metric_value":97.22,"embargo":null,"is_public":true},{"epiweek":13,"date":"2017-03-31","metric_value":97.04,"embargo":null,"is_public":true},{"epiweek":13,"date":"2018-03-31","metric_value":96.2,"embargo":null,"is_public":true},{"epiweek":13,"date":"2019-03-31","metric_value":94.52,"embargo":null,"is_public":true},{"epiweek":14,"date":"2020-03-31","metric_value":94.45,"embargo":null,"is_public":true},{"epiweek":13,"date":"2021-03-31","metric_value":94.53,"embargo":null,"is_public":true},{"epiweek":13,"date":"2022-03-31","metric_value":93.52,"embargo":null,"is_public":true},{"epiweek":13,"date":"2023-03-31","metric_value":93.1,"embargo":null,"is_public":true},{"epiweek":13,"date":"2024-03-31","metric_value":91.8,"embargo":null,"is_public":true}],"refresh_date":"2025-08-01 11:00:53"}

source_data/6-in-1_coverage_coverageByYear_CTRY_N92000002_all_all_24m.json

@@ -1 +1 @@
-{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"Northern Ireland","geography_code":"N92000002","age":"all","sex":"all","stratum":"24m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":98.53,"embargo":null},{"epiweek":13,"date":"2011-03-31","metric_value":98.83,"embargo":null},{"epiweek":13,"date":"2012-03-31","metric_value":98.43,"embargo":null},{"epiweek":13,"date":"2013-03-31","metric_value":98.58,"embargo":null},{"epiweek":14,"date":"2014-03-31","metric_value":98.65,"embargo":null},{"epiweek":14,"date":"2015-03-31","metric_value":97.54,"embargo":null},{"epiweek":13,"date":"2016-03-31","metric_value":98.22,"embargo":null},{"epiweek":13,"date":"2017-03-31","metric_value":97.89,"embargo":null},{"epiweek":13,"date":"2018-03-31","metric_value":97.77,"embargo":null},{"epiweek":13,"date":"2019-03-31","metric_value":96.97,"embargo":null},{"epiweek":14,"date":"2020-03-31","metric_value":95.83,"embargo":null},{"epiweek":13,"date":"2021-03-31","metric_value":95.85,"embargo":null},{"epiweek":13,"date":"2022-03-31","metric_value":95.33,"embargo":null},{"epiweek":13,"date":"2023-03-31","metric_value":94.86,"embargo":null},{"epiweek":13,"date":"2024-03-31","metric_value":93.6,"embargo":null}],"refresh_date":"2025-08-01 11:00:53"}
+{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"Northern Ireland","geography_code":"N92000002","age":"all","sex":"all","stratum":"24m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":98.53,"embargo":null,"is_public":true},{"epiweek":13,"date":"2011-03-31","metric_value":98.83,"embargo":null,"is_public":true},{"epiweek":13,"date":"2012-03-31","metric_value":98.43,"embargo":null,"is_public":true},{"epiweek":13,"date":"2013-03-31","metric_value":98.58,"embargo":null,"is_public":true},{"epiweek":14,"date":"2014-03-31","metric_value":98.65,"embargo":null,"is_public":true},{"epiweek":14,"date":"2015-03-31","metric_value":97.54,"embargo":null,"is_public":true},{"epiweek":13,"date":"2016-03-31","metric_value":98.22,"embargo":null,"is_public":true},{"epiweek":13,"date":"2017-03-31","metric_value":97.89,"embargo":null,"is_public":true},{"epiweek":13,"date":"2018-03-31","metric_value":97.77,"embargo":null,"is_public":true},{"epiweek":13,"date":"2019-03-31","metric_value":96.97,"embargo":null,"is_public":true},{"epiweek":14,"date":"2020-03-31","metric_value":95.83,"embargo":null,"is_public":true},{"epiweek":13,"date":"2021-03-31","metric_value":95.85,"embargo":null,"is_public":true},{"epiweek":13,"date":"2022-03-31","metric_value":95.33,"embargo":null,"is_public":true},{"epiweek":13,"date":"2023-03-31","metric_value":94.86,"embargo":null,"is_public":true},{"epiweek":13,"date":"2024-03-31","metric_value":93.6,"embargo":null,"is_public":true}],"refresh_date":"2025-08-01 11:00:53"}