Task/cdd 3206 be filter db query for only is public true data

ingestion/data_transfer_models/headline.py

@@ -7,10 +7,9 @@
 
 import validation
 from ingestion.utils import type_hints
-from metrics.api.settings.auth import AUTH_ENABLED
+from metrics.api.settings.auth import ALLOW_MISSING_IS_PUBLIC_FIELD
 from validation.data_transfer_models.base import (
     IncomingBaseDataModel,
-    NonPublicDataSentToPublicIngestionError,
 )
 
 
@@ -134,14 +133,6 @@ def validate_confidence_intervals(self) -> Self:
         )
         return self
 
-    @model_validator(mode="after")
-    def invalidate_non_public_data_for_public_ingestion(self) -> Self:
-        """Checks that if this is a public instance of the product then `is_public=False` data is invalidated."""
-        if not AUTH_ENABLED and not self.is_public:
-            raise NonPublicDataSentToPublicIngestionError
-
-        return self
-
 
 class HeadlineDTO(IncomingBaseDataModel):
     data: list[InboundHeadlineSpecificFields]
@@ -183,7 +174,11 @@ def _build_enriched_headline_specific_fields(
             upper_confidence=individual_time_series.get("upper_confidence", None),
             metric_value=individual_time_series["metric_value"],
             lower_confidence=individual_time_series.get("lower_confidence", None),
-            is_public=individual_time_series.get("is_public", True),
+            is_public=(
+                individual_time_series["is_public"]
+                if not ALLOW_MISSING_IS_PUBLIC_FIELD
+                else individual_time_series.get("is_public", True)
+            ),
         )
         for individual_time_series in source_data["data"]
         if individual_time_series["metric_value"] is not None

ingestion/data_transfer_models/time_series.py

@@ -1,15 +1,13 @@
 import datetime
-from typing import Self
 
-from pydantic import BaseModel, field_validator, model_validator
+from pydantic import BaseModel, field_validator
 from pydantic.fields import Field
 
 import validation
 from ingestion.utils import type_hints
-from metrics.api.settings.auth import AUTH_ENABLED
+from metrics.api.settings.auth import ALLOW_MISSING_IS_PUBLIC_FIELD
 from validation.data_transfer_models.base import (
     IncomingBaseDataModel,
-    NonPublicDataSentToPublicIngestionError,
 )
 
 
@@ -43,14 +41,6 @@ def cast_embargo_to_uk_timezone(
         """
         return validation.cast_date_to_uk_timezone(date_value=embargo)
 
-    @model_validator(mode="after")
-    def invalidate_non_public_data_for_public_ingestion(self) -> Self:
-        """Checks that if this is a public instance of the product then `is_public=False` data is invalidated."""
-        if not AUTH_ENABLED and not self.is_public:
-            raise NonPublicDataSentToPublicIngestionError
-
-        return self
-
 
 class TimeSeriesDTO(IncomingBaseDataModel):
     metric_frequency: str
@@ -128,7 +118,11 @@ def _build_enriched_time_series_specific_fields(
                 "in_reporting_delay_period", False
             ),
             force_write=individual_time_series.get("force_write", False),
-            is_public=individual_time_series.get("is_public", True),
+            is_public=(
+                individual_time_series["is_public"]
+                if not ALLOW_MISSING_IS_PUBLIC_FIELD
+                else individual_time_series.get("is_public", True)
+            ),
         )
         for individual_time_series in source_data["time_series"]
         if individual_time_series["metric_value"] is not None

metrics/api/permissions/fluent_permissions.py

@@ -1,9 +1,15 @@
 from collections.abc import Iterable
 from dataclasses import dataclass
 
+from metrics.api.settings import auth
 from metrics.data.models.rbac_models import RBACPermission
 
 
+def is_public_data_only_enforced() -> bool:
+    """Whether public-only data is currently enforced."""
+    return auth.ENFORCE_PUBLIC_DATA_ONLY
+
+
 def validate_permissions_for_non_public(
     *,
     theme: str,
@@ -22,6 +28,9 @@ def validate_permissions_for_non_public(
         False otherwise.
 
     """
+    if is_public_data_only_enforced():
+        return False
+
     requested_data_parameters = RequestedDataParameters(
         theme=theme,
         sub_theme=sub_theme,

metrics/api/settings/auth.py

@@ -5,4 +5,32 @@ def is_auth_enabled() -> bool:
     return str(os.environ.get("AUTH_ENABLED", "")).lower() in {"true", "1"}
 
 
+def is_allow_missing_is_public_field() -> bool:
+    return str(os.environ.get("ALLOW_MISSING_IS_PUBLIC_FIELD", "")).lower() in {
+        "true",
+        "1",
+    }
+
+
+def is_enforce_public_data_only() -> bool:
+    return str(os.environ.get("ENFORCE_PUBLIC_DATA_ONLY", "true")).lower() in {
+        "true",
+        "1",
+    }
+
+
 AUTH_ENABLED = is_auth_enabled()
+ALLOW_MISSING_IS_PUBLIC_FIELD = is_allow_missing_is_public_field()
+
+# If False, non-authorized RBAC paths return only public headline/time-series rows,
+#           while authorized RBAC paths can include non-public rows when permitted.
+# If True, RBAC non-public row access is suppressed and only public dashboard API
+#          row data is returned, except:
+#          a) `/api/audit/*` endpoints can still return public and non-public
+#              diagnostics.
+#          b) Metadata/master-table endpoints (not headline/time-series rows)
+#             remain unfiltered.
+#
+# Note: This flag is only used by the private API, because we are not allowing
+#       non-public data through the public API anyway.
+ENFORCE_PUBLIC_DATA_ONLY = is_enforce_public_data_only()

metrics/data/managers/core_models/time_series.py

@@ -14,6 +14,7 @@
 from django.utils import timezone
 
 from metrics.api.permissions.fluent_permissions import (
+    is_public_data_only_enforced,
     validate_permissions_for_non_public,
 )
 from metrics.data.models import RBACPermission
@@ -466,9 +467,14 @@ def get_available_geographies(self, *, topic: str) -> models.QuerySet:
                         [Row(geography__name='England', geography__geography_type__name='Nation')]>`
 
         """
+        queryset = self.filter(metric__topic__name=topic)
+
+        if is_public_data_only_enforced():
+            queryset = queryset.filter(is_public=True)
+            queryset = self._exclude_data_under_embargo(queryset=queryset)
+
         return (
-            self.filter(metric__topic__name=topic)
-            .values_list(
+            queryset.values_list(
                 "geography__name",
                 "geography__geography_type__name",
                 "geography__geography_code",

public_api/serializers/api_time_series_request_serializer.py

@@ -78,7 +78,9 @@ def get_queryset(self) -> "APITimeSeriesQuerySet":
         """
         kwargs: dict[str, str] = self.get_kwargs_from_request()
         return self.api_time_series_manager.get_distinct_column_values_with_filters(
-            lookup_field=self.lookup_field, restrict_to_public=True, **kwargs
+            lookup_field=self.lookup_field,
+            restrict_to_public=True,  # because we are not allowing non-public data through the public API
+            **kwargs
         )
 
     def build_timeseries_dto_slice(self) -> list[APITimeSeriesDTO]:

public_api/version_02/serializers/api_time_series_request_serializer.py

@@ -84,15 +84,16 @@ def get_queryset(self) -> "APITimeSeriesQuerySet":
         """
         kwargs: dict[str, str] = self.get_formatted_kwargs_from_request()
         return self.api_time_series_manager.get_distinct_column_values_with_filters(
-            lookup_field=self.lookup_field, restrict_to_public=True, **kwargs
+            lookup_field=self.lookup_field,
+            restrict_to_public=True,  # because we are not allowing non-public data through the public API
+            **kwargs
         )
 
     def build_timeseries_dto_slice(self) -> list[APITimeSeriesDTO]:
         """Builds a list of simple `APITimeSeriesDTO` from the kwargs of the request and the given `value`
 
         Returns:
             List[APITimeSeriesDTO]: List of created data transfer objects
-
         """
         queryset = self.get_queryset()
         return [

public_api/version_02/views/timeseries_viewset.py

@@ -99,5 +99,5 @@ def get_queryset(self):
             geography_type=self.kwargs["geography_type"],
             geography=self.kwargs["geography"],
             metric=self.kwargs["metric"],
-            restrict_to_public=True,
+            restrict_to_public=True,  # because we are not allowing non-public data through the public API
         )

public_api/views/timeseries_viewset.py

@@ -97,5 +97,5 @@ def get_queryset(self):
             geography_type=self.kwargs["geography_type"],
             geography=self.kwargs["geography"],
             metric=self.kwargs["metric"],
-            restrict_to_public=True,
+            restrict_to_public=True,  # because we are not allowing non-public data through the public API
         )

source_data/6-in-1_coverage_coverageByYear_CTRY_E92000001_all_all_12m.json

@@ -1 +1 @@
-{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"England","geography_code":"E92000001","age":"all","sex":"all","stratum":"12m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":93.61,"embargo":null},{"epiweek":13,"date":"2011-03-31","metric_value":94.15,"embargo":null},{"epiweek":13,"date":"2012-03-31","metric_value":94.67,"embargo":null},{"epiweek":13,"date":"2013-03-31","metric_value":94.74,"embargo":null},{"epiweek":14,"date":"2014-03-31","metric_value":94.34,"embargo":null},{"epiweek":14,"date":"2015-03-31","metric_value":94.23,"embargo":null},{"epiweek":13,"date":"2016-03-31","metric_value":93.56,"embargo":null},{"epiweek":13,"date":"2017-03-31","metric_value":93.41,"embargo":null},{"epiweek":13,"date":"2018-03-31","metric_value":93.12,"embargo":null},{"epiweek":13,"date":"2019-03-31","metric_value":92.09,"embargo":null},{"epiweek":14,"date":"2020-03-31","metric_value":92.57,"embargo":null},{"epiweek":13,"date":"2021-03-31","metric_value":92.04,"embargo":null},{"epiweek":13,"date":"2022-03-31","metric_value":91.84,"embargo":null},{"epiweek":13,"date":"2023-03-31","metric_value":91.8,"embargo":null},{"epiweek":13,"date":"2024-03-31","metric_value":91.22,"embargo":null}],"refresh_date":"2025-08-01 11:00:53"}
+{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"England","geography_code":"E92000001","age":"all","sex":"all","stratum":"12m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":93.61,"embargo":null,"is_public":true},{"epiweek":13,"date":"2011-03-31","metric_value":94.15,"embargo":null,"is_public":true},{"epiweek":13,"date":"2012-03-31","metric_value":94.67,"embargo":null,"is_public":true},{"epiweek":13,"date":"2013-03-31","metric_value":94.74,"embargo":null,"is_public":true},{"epiweek":14,"date":"2014-03-31","metric_value":94.34,"embargo":null,"is_public":true},{"epiweek":14,"date":"2015-03-31","metric_value":94.23,"embargo":null,"is_public":true},{"epiweek":13,"date":"2016-03-31","metric_value":93.56,"embargo":null,"is_public":true},{"epiweek":13,"date":"2017-03-31","metric_value":93.41,"embargo":null,"is_public":true},{"epiweek":13,"date":"2018-03-31","metric_value":93.12,"embargo":null,"is_public":true},{"epiweek":13,"date":"2019-03-31","metric_value":92.09,"embargo":null,"is_public":true},{"epiweek":14,"date":"2020-03-31","metric_value":92.57,"embargo":null,"is_public":true},{"epiweek":13,"date":"2021-03-31","metric_value":92.04,"embargo":null,"is_public":true},{"epiweek":13,"date":"2022-03-31","metric_value":91.84,"embargo":null,"is_public":true},{"epiweek":13,"date":"2023-03-31","metric_value":91.8,"embargo":null,"is_public":true},{"epiweek":13,"date":"2024-03-31","metric_value":91.22,"embargo":null,"is_public":true}],"refresh_date":"2025-08-01 11:00:53"}

source_data/6-in-1_coverage_coverageByYear_CTRY_E92000001_all_all_24m.json

@@ -1 +1 @@
-{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"England","geography_code":"E92000001","age":"all","sex":"all","stratum":"24m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":95.29,"embargo":null},{"epiweek":13,"date":"2011-03-31","metric_value":95.98,"embargo":null},{"epiweek":13,"date":"2012-03-31","metric_value":96.14,"embargo":null},{"epiweek":13,"date":"2013-03-31","metric_value":96.3,"embargo":null},{"epiweek":14,"date":"2014-03-31","metric_value":96.14,"embargo":null},{"epiweek":14,"date":"2015-03-31","metric_value":95.74,"embargo":null},{"epiweek":13,"date":"2016-03-31","metric_value":95.24,"embargo":null},{"epiweek":13,"date":"2017-03-31","metric_value":95.15,"embargo":null},{"epiweek":13,"date":"2018-03-31","metric_value":95.1,"embargo":null},{"epiweek":13,"date":"2019-03-31","metric_value":94.17,"embargo":null},{"epiweek":14,"date":"2020-03-31","metric_value":93.85,"embargo":null},{"epiweek":13,"date":"2021-03-31","metric_value":93.85,"embargo":null},{"epiweek":13,"date":"2022-03-31","metric_value":93.04,"embargo":null},{"epiweek":13,"date":"2023-03-31","metric_value":92.62,"embargo":null},{"epiweek":13,"date":"2024-03-31","metric_value":92.44,"embargo":null}],"refresh_date":"2025-08-01 11:00:53"}
+{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"England","geography_code":"E92000001","age":"all","sex":"all","stratum":"24m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":95.29,"embargo":null,"is_public":true},{"epiweek":13,"date":"2011-03-31","metric_value":95.98,"embargo":null,"is_public":true},{"epiweek":13,"date":"2012-03-31","metric_value":96.14,"embargo":null,"is_public":true},{"epiweek":13,"date":"2013-03-31","metric_value":96.3,"embargo":null,"is_public":true},{"epiweek":14,"date":"2014-03-31","metric_value":96.14,"embargo":null,"is_public":true},{"epiweek":14,"date":"2015-03-31","metric_value":95.74,"embargo":null,"is_public":true},{"epiweek":13,"date":"2016-03-31","metric_value":95.24,"embargo":null,"is_public":true},{"epiweek":13,"date":"2017-03-31","metric_value":95.15,"embargo":null,"is_public":true},{"epiweek":13,"date":"2018-03-31","metric_value":95.1,"embargo":null,"is_public":true},{"epiweek":13,"date":"2019-03-31","metric_value":94.17,"embargo":null,"is_public":true},{"epiweek":14,"date":"2020-03-31","metric_value":93.85,"embargo":null,"is_public":true},{"epiweek":13,"date":"2021-03-31","metric_value":93.85,"embargo":null,"is_public":true},{"epiweek":13,"date":"2022-03-31","metric_value":93.04,"embargo":null,"is_public":true},{"epiweek":13,"date":"2023-03-31","metric_value":92.62,"embargo":null,"is_public":true},{"epiweek":13,"date":"2024-03-31","metric_value":92.44,"embargo":null,"is_public":true}],"refresh_date":"2025-08-01 11:00:53"}

source_data/6-in-1_coverage_coverageByYear_CTRY_E92000001_all_all_5y.json

@@ -1 +1 @@
-{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"England","geography_code":"E92000001","age":"all","sex":"all","stratum":"5y","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2016-03-31","metric_value":95.57,"embargo":null},{"epiweek":13,"date":"2017-03-31","metric_value":95.59,"embargo":null},{"epiweek":13,"date":"2018-03-31","metric_value":95.56,"embargo":null},{"epiweek":13,"date":"2019-03-31","metric_value":95.04,"embargo":null},{"epiweek":14,"date":"2020-03-31","metric_value":95.24,"embargo":null},{"epiweek":13,"date":"2021-03-31","metric_value":95.24,"embargo":null},{"epiweek":13,"date":"2022-03-31","metric_value":94.39,"embargo":null},{"epiweek":13,"date":"2023-03-31","metric_value":93.2,"embargo":null},{"epiweek":13,"date":"2024-03-31","metric_value":92.62,"embargo":null},{"epiweek":14,"date":"2014-03-31","metric_value":95.6,"embargo":null},{"epiweek":14,"date":"2015-03-31","metric_value":95.56,"embargo":null}],"refresh_date":"2025-08-01 11:00:53"}
+{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"England","geography_code":"E92000001","age":"all","sex":"all","stratum":"5y","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2016-03-31","metric_value":95.57,"embargo":null,"is_public":true},{"epiweek":13,"date":"2017-03-31","metric_value":95.59,"embargo":null,"is_public":true},{"epiweek":13,"date":"2018-03-31","metric_value":95.56,"embargo":null,"is_public":true},{"epiweek":13,"date":"2019-03-31","metric_value":95.04,"embargo":null,"is_public":true},{"epiweek":14,"date":"2020-03-31","metric_value":95.24,"embargo":null,"is_public":true},{"epiweek":13,"date":"2021-03-31","metric_value":95.24,"embargo":null,"is_public":true},{"epiweek":13,"date":"2022-03-31","metric_value":94.39,"embargo":null,"is_public":true},{"epiweek":13,"date":"2023-03-31","metric_value":93.2,"embargo":null,"is_public":true},{"epiweek":13,"date":"2024-03-31","metric_value":92.62,"embargo":null,"is_public":true},{"epiweek":14,"date":"2014-03-31","metric_value":95.6,"embargo":null,"is_public":true},{"epiweek":14,"date":"2015-03-31","metric_value":95.56,"embargo":null,"is_public":true}],"refresh_date":"2025-08-01 11:00:53"}

source_data/6-in-1_coverage_coverageByYear_CTRY_N92000002_all_all_12m.json

@@ -1 +1 @@
-{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"Northern Ireland","geography_code":"N92000002","age":"all","sex":"all","stratum":"12m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":97.77,"embargo":null},{"epiweek":13,"date":"2011-03-31","metric_value":97.51,"embargo":null},{"epiweek":13,"date":"2012-03-31","metric_value":97.57,"embargo":null},{"epiweek":13,"date":"2013-03-31","metric_value":97.56,"embargo":null},{"epiweek":14,"date":"2014-03-31","metric_value":97.48,"embargo":null},{"epiweek":14,"date":"2015-03-31","metric_value":96.84,"embargo":null},{"epiweek":13,"date":"2016-03-31","metric_value":97.22,"embargo":null},{"epiweek":13,"date":"2017-03-31","metric_value":97.04,"embargo":null},{"epiweek":13,"date":"2018-03-31","metric_value":96.2,"embargo":null},{"epiweek":13,"date":"2019-03-31","metric_value":94.52,"embargo":null},{"epiweek":14,"date":"2020-03-31","metric_value":94.45,"embargo":null},{"epiweek":13,"date":"2021-03-31","metric_value":94.53,"embargo":null},{"epiweek":13,"date":"2022-03-31","metric_value":93.52,"embargo":null},{"epiweek":13,"date":"2023-03-31","metric_value":93.1,"embargo":null},{"epiweek":13,"date":"2024-03-31","metric_value":91.8,"embargo":null}],"refresh_date":"2025-08-01 11:00:53"}
+{"parent_theme":"immunisation","child_theme":"childhood-vaccines","topic":"6-in-1","metric_group":"coverage","metric":"6-in-1_coverage_coverageByYear","geography_type":"Nation","geography":"Northern Ireland","geography_code":"N92000002","age":"all","sex":"all","stratum":"12m","metric_frequency":"annual","time_series":[{"epiweek":13,"date":"2010-03-31","metric_value":97.77,"embargo":null,"is_public":true},{"epiweek":13,"date":"2011-03-31","metric_value":97.51,"embargo":null,"is_public":true},{"epiweek":13,"date":"2012-03-31","metric_value":97.57,"embargo":null,"is_public":true},{"epiweek":13,"date":"2013-03-31","metric_value":97.56,"embargo":null,"is_public":true},{"epiweek":14,"date":"2014-03-31","metric_value":97.48,"embargo":null,"is_public":true},{"epiweek":14,"date":"2015-03-31","metric_value":96.84,"embargo":null,"is_public":true},{"epiweek":13,"date":"2016-03-31","metric_value":97.22,"embargo":null,"is_public":true},{"epiweek":13,"date":"2017-03-31","metric_value":97.04,"embargo":null,"is_public":true},{"epiweek":13,"date":"2018-03-31","metric_value":96.2,"embargo":null,"is_public":true},{"epiweek":13,"date":"2019-03-31","metric_value":94.52,"embargo":null,"is_public":true},{"epiweek":14,"date":"2020-03-31","metric_value":94.45,"embargo":null,"is_public":true},{"epiweek":13,"date":"2021-03-31","metric_value":94.53,"embargo":null,"is_public":true},{"epiweek":13,"date":"2022-03-31","metric_value":93.52,"embargo":null,"is_public":true},{"epiweek":13,"date":"2023-03-31","metric_value":93.1,"embargo":null,"is_public":true},{"epiweek":13,"date":"2024-03-31","metric_value":91.8,"embargo":null,"is_public":true}],"refresh_date":"2025-08-01 11:00:53"}