FTRS-2587 Data migration toggles

application/packages/python/ftrs_common/feature_flags/__init__.py

@@ -22,11 +22,10 @@
 from ftrs_common.feature_flags.feature_flags_client import (
     FeatureFlagError,
     FeatureFlagsClient,
-    get_feature_flags,
     is_enabled,
 )
 
-__all__ = ["FeatureFlagError", "FeatureFlagsClient", "get_feature_flags", "is_enabled"]
+__all__ = ["FeatureFlagError", "FeatureFlagsClient", "is_enabled"]
 
 from ftrs_common.feature_flags.feature_flag_config import (
     FeatureFlag,

application/packages/python/ftrs_common/feature_flags/feature_flags_client.py

@@ -1,9 +1,12 @@
+import os
 from functools import lru_cache
+from typing import Protocol
 
 from aws_lambda_powertools.utilities.feature_flags import AppConfigStore, FeatureFlags
 from aws_lambda_powertools.utilities.feature_flags.exceptions import (
     ConfigurationStoreError,
 )
+from ftrs_common.feature_flags.feature_flag_config import FeatureFlag
 from ftrs_common.logbase import FeatureFlagLogBase
 from ftrs_common.logger import Logger
 from ftrs_common.utils.config import Settings
@@ -15,6 +18,29 @@
 CACHE_TTL_SECONDS = 45
 
 
+class FeatureFlagsClientProtocol(Protocol):
+    """Protocol defining the interface for feature flag clients."""
+
+    def is_enabled(self, flag_name: str, default: bool = False) -> bool:
+        """Check if a feature flag is enabled."""
+        ...
+
+
+class LocalFlagsClient:
+    """Local feature flags client for development and testing."""
+
+    def __init__(self) -> None:
+        self.flags: dict[str, bool] = {}
+        # Automatically load all feature flags from the enum
+        for flag in FeatureFlag:
+            env_var_name = flag.value.upper()
+            env_value = os.getenv(env_var_name, "true")
+            self.flags[flag.value] = env_value.lower() == "true"
+
+    def is_enabled(self, flag_name: str, default: bool = False) -> bool:
+        return self.flags.get(flag_name, default)
+
+
 class FeatureFlagError(Exception):
     """Exception raised when feature flag evaluation fails."""
 
@@ -144,14 +170,16 @@ def is_enabled(
 
 
 @lru_cache(maxsize=1)
-def _get_client() -> FeatureFlagsClient:
-    """Get a cached FeatureFlagsClient instance."""
+def _get_client() -> FeatureFlagsClientProtocol:
+    """Get a cached feature flags client instance."""
+    settings = Settings()
+    if settings.env == "local" or (
+        settings.env == "dev" and settings.workspace is not None
+    ):
+        return LocalFlagsClient()
     return FeatureFlagsClient()
 
 
-def get_feature_flags() -> FeatureFlags:
-    return _get_client().get_feature_flags()
-
-
 def is_enabled(flag_name: str, default: bool = False) -> bool:
+    """Check if a feature flag is enabled."""
     return _get_client().is_enabled(flag_name, default)

application/packages/python/ftrs_common/tests/feature_flags/test_feature_flags_client.py

@@ -9,8 +9,8 @@
     CACHE_TTL_SECONDS,
     FeatureFlagError,
     FeatureFlagsClient,
+    LocalFlagsClient,
     _get_client,
-    get_feature_flags,
     is_enabled,
 )
 from ftrs_common.logbase import FeatureFlagLogBase
@@ -57,6 +57,31 @@ def mock_logger(mocker: MockerFixture) -> MagicMock:
     return mocker.patch("ftrs_common.feature_flags.feature_flags_client.logger")
 
 
+class TestLocalFlagsClient:
+    def test_init_sets_default_flags(self, mocker: MockerFixture) -> None:
+        mocker.patch.dict("os.environ", {}, clear=False)
+        client = LocalFlagsClient()
+        assert "data_migration_search_triage_code_enabled" in client.flags
+        assert client.flags["data_migration_search_triage_code_enabled"] is True
+
+    def test_is_enabled_returns_false_when_env_var_is_false(
+        self, mocker: MockerFixture
+    ) -> None:
+        mocker.patch.dict(
+            "os.environ", {"DATA_MIGRATION_SEARCH_TRIAGE_CODE_ENABLED": "FALSE"}
+        )
+        client = LocalFlagsClient()
+        assert client.is_enabled("data_migration_search_triage_code_enabled") is False
+
+    def test_is_enabled_returns_default_for_missing_flag(
+        self, mocker: MockerFixture
+    ) -> None:
+        mocker.patch.dict("os.environ", {}, clear=False)
+        client = LocalFlagsClient()
+        assert client.is_enabled("nonexistent_flag", default=True) is True
+        assert client.is_enabled("nonexistent_flag", default=False) is False
+
+
 class TestFeatureFlagError:
     def test_error_contains_flag_name_and_message(self) -> None:
         error = FeatureFlagError(
@@ -363,21 +388,47 @@ def test_logs_warning_when_flag_not_found(
 
 
 class TestModuleFunctions:
+    def test_get_client_returns_local_client_for_local_env(
+        self, mocker: MockerFixture
+    ) -> None:
+        mock = mocker.patch("ftrs_common.feature_flags.feature_flags_client.Settings")
+        mock.return_value.env = "local"
+        mock.return_value.workspace = None
+        _get_client.cache_clear()
+
+        client = _get_client()
+        assert isinstance(client, LocalFlagsClient)
+
+    def test_get_client_returns_local_client_for_dev_with_workspace(
+        self, mocker: MockerFixture
+    ) -> None:
+        mock = mocker.patch("ftrs_common.feature_flags.feature_flags_client.Settings")
+        mock.return_value.env = "dev"
+        mock.return_value.workspace = "test-workspace"
+        _get_client.cache_clear()
+
+        client = _get_client()
+        assert isinstance(client, LocalFlagsClient)
+
+    def test_get_client_returns_feature_flags_client_for_dev_without_workspace(
+        self, mocker: MockerFixture
+    ) -> None:
+        mock = mocker.patch("ftrs_common.feature_flags.feature_flags_client.Settings")
+        mock.return_value.env = "dev"
+        mock.return_value.workspace = None
+        mock.return_value.appconfig_application_id = "test-app"
+        mock.return_value.appconfig_environment_id = "test-env"
+        mock.return_value.appconfig_configuration_profile_id = "test-profile"
+        _get_client.cache_clear()
+
+        client = _get_client()
+        assert isinstance(client, FeatureFlagsClient)
+
     def test_get_client_returns_cached_instance(self, mock_settings: MagicMock) -> None:
         client1 = _get_client()
         client2 = _get_client()
         assert client1 is client2
 
-    def test_get_feature_flags_function_returns_feature_flags(
-        self,
-        mock_settings: MagicMock,
-        mock_appconfig_store: MagicMock,
-        mock_feature_flags_class: MagicMock,
-        mock_logger: MagicMock,
-    ) -> None:
-        result = get_feature_flags()
-        assert result is mock_feature_flags_class.return_value
-
     def test_is_enabled_function_checks_flag(
         self,
         mock_settings: MagicMock,

infrastructure/stacks/data_migration/lambda.tf

@@ -109,12 +109,11 @@ module "queue_populator_lambda" {
   subnet_ids         = [for subnet in data.aws_subnet.private_subnets_details : subnet.id]
   security_group_ids = [try(aws_security_group.rds_accessor_lambda_security_group[0].id, data.aws_security_group.rds_accessor_lambda_security_group[0].id)]
 
-  number_of_policy_jsons = "4"
+  number_of_policy_jsons = "3"
   policy_jsons = [
     data.aws_iam_policy_document.secrets_access_policy.json,
     data.aws_iam_policy_document.sqs_access_policy.json,
-    data.aws_iam_policy_document.lambda_kms_access.json,
-    data.aws_iam_policy.appconfig_access_policy.policy
+    data.aws_iam_policy_document.lambda_kms_access.json
   ]
 
   layers = concat(
@@ -124,13 +123,10 @@ module "queue_populator_lambda" {
   )
 
   environment_variables = {
-    "ENVIRONMENT"                        = var.environment
-    "WORKSPACE"                          = terraform.workspace == "default" ? "" : terraform.workspace
-    "SQS_QUEUE_URL"                      = aws_sqs_queue.dms_event_queue.url
-    "PROJECT_NAME"                       = var.project
-    "APPCONFIG_APPLICATION_ID"           = data.aws_ssm_parameter.appconfig_application_id.value
-    "APPCONFIG_ENVIRONMENT_ID"           = local.appconfig_environment_id
-    "APPCONFIG_CONFIGURATION_PROFILE_ID" = local.appconfig_configuration_profile_id
+    "ENVIRONMENT"   = var.environment
+    "WORKSPACE"     = terraform.workspace == "default" ? "" : terraform.workspace
+    "SQS_QUEUE_URL" = aws_sqs_queue.dms_event_queue.url
+    "PROJECT_NAME"  = var.project
   }
   account_id     = data.aws_caller_identity.current.account_id
   account_prefix = local.account_prefix

services/data-migration/.env.local

@@ -0,0 +1,5 @@
+# Local development (e.g. use environment variables instead of AppConfig toggles)
+ENVIRONMENT = "local"
+
+## AppConfig feature toggles
+DATA_MIGRATION_SEARCH_TRIAGE_CODE_ENABLED=true

services/data-migration/src/cli/main.py

@@ -134,3 +134,8 @@ def restore_from_s3_handler(
     Handler for restoring data from S3 to all DynamoDB tables.
     """
     asyncio.run(run_s3_restore(env, workspace))
+
+
+# PyCharm local debugging
+if __name__ == "__main__":
+    typer_app()

services/data-migration/src/queue_populator/lambda_handler.py

@@ -4,7 +4,6 @@
 
 import boto3
 from aws_lambda_powertools.utilities.typing import LambdaContext
-from ftrs_common.feature_flags import FeatureFlag, FeatureFlagsClient
 from ftrs_common.logger import Logger
 from ftrs_data_layer.domain.legacy import Service
 from ftrs_data_layer.logbase import DataMigrationLogBase
@@ -20,10 +19,6 @@
 SQS_CLIENT = boto3.client("sqs")
 
 
-# Initialize outside handler - reused across invocations
-FEATURE_FLAGS_CLIENT: FeatureFlagsClient = FeatureFlagsClient()
-
-
 class QueuePopulatorEvent(BaseModel):
     table_name: str = "services"
     service_id: Optional[int] = None
@@ -159,13 +154,6 @@ def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> None:
     """
     AWS Lambda entrypoint for populating the queue with legacy services.
     """
-    if FEATURE_FLAGS_CLIENT.is_enabled(
-        FeatureFlag.DATA_MIGRATION_SEARCH_TRIAGE_CODE_ENABLED
-    ):
-        LOGGER.info("Healthcare service feature flag is enabled")
-    else:
-        LOGGER.info("Healthcare service feature flag is disabled")
-
     parsed_event = QueuePopulatorEvent(**event)
     config = QueuePopulatorConfig(
         db_config=DatabaseConfig.from_secretsmanager(),

services/data-migration/src/reference_data_load/application.py

@@ -1,3 +1,5 @@
+from ftrs_common.feature_flags import FeatureFlag
+from ftrs_common.feature_flags.feature_flags_client import FeatureFlagsClientProtocol
 from ftrs_common.logger import Logger
 from sqlmodel import create_engine
 
@@ -7,14 +9,26 @@
 
 
 class ReferenceDataLoadApplication:
-    def __init__(self, config: ReferenceDataLoadConfig | None = None) -> None:
+    def __init__(
+        self,
+        config: ReferenceDataLoadConfig | None = None,
+        feature_flags_client: FeatureFlagsClientProtocol | None = None,
+    ) -> None:
         self.logger = Logger.get(service="reference-data-load")
         self.config = config or ReferenceDataLoadConfig()
         self.engine = create_engine(self.config.db_config.connection_string, echo=False)
+        self.feature_flags_client = feature_flags_client
 
     def handle(self, event: ReferenceDataLoadEvent) -> None:
         match event.type:
             case "triagecode":
+                if not self.feature_flags_client.is_enabled(
+                    FeatureFlag.DATA_MIGRATION_SEARCH_TRIAGE_CODE_ENABLED
+                ):
+                    self.logger.info(
+                        "Triage code loading is disabled by feature flag, skipping"
+                    )
+                    return None
                 return self._load_triage_codes()
 
         raise ValueError(f"Unknown event type: {event.type}")

services/data-migration/src/reference_data_load/lambda_handler.py

@@ -1,4 +1,5 @@
 from aws_lambda_powertools.utilities.typing import LambdaContext
+from ftrs_common.feature_flags import FeatureFlagsClient
 from ftrs_common.logger import Logger
 
 from common.events import ReferenceDataLoadEvent
@@ -7,6 +8,8 @@
 APP: ReferenceDataLoadApplication | None = None
 LOGGER = Logger.get(service="reference-data-load")
 
+FEATURE_FLAGS_CLIENT: FeatureFlagsClient = FeatureFlagsClient()
+
 
 @LOGGER.inject_lambda_context
 def lambda_handler(event: dict, context: LambdaContext) -> None:
@@ -16,6 +19,6 @@ def lambda_handler(event: dict, context: LambdaContext) -> None:
     """
     global APP  # noqa: PLW0603
     if APP is None:
-        APP = ReferenceDataLoadApplication()
+        APP = ReferenceDataLoadApplication(feature_flags_client=FEATURE_FLAGS_CLIENT)
 
     APP.handle(ReferenceDataLoadEvent(**event))

services/data-migration/src/service_migration/transformer/gp_enhanced_access.py

@@ -1,5 +1,6 @@
 import re
 
+from ftrs_common.feature_flags import FeatureFlag, is_enabled
 from ftrs_data_layer.domain import HealthcareServiceCategory, HealthcareServiceType
 from ftrs_data_layer.domain import legacy as legacy_model
 
@@ -63,7 +64,15 @@ def is_service_supported(
     ) -> tuple[bool, str | None]:
         """
         Check if the service is a GP Enhanced Access service.
+
+        When the feature flag is disabled, GP Enhanced Access services are not supported.
         """
+        if not is_enabled(FeatureFlag.DATA_MIGRATION_SEARCH_TRIAGE_CODE_ENABLED):
+            return (
+                False,
+                "GP Enhanced Access service selection is disabled by feature flag",
+            )
+
         if service.typeid not in [
             cls.GP_ACCESS_HUB_TYPE_ID,
             cls.PCN_ENHANCED_SERVICE_TYPE_ID,

services/data-migration/src/service_migration/transformer/gp_practice.py

@@ -1,5 +1,6 @@
 import re
 
+from ftrs_common.feature_flags import FeatureFlag, is_enabled
 from ftrs_data_layer.domain import HealthcareServiceCategory, HealthcareServiceType
 from ftrs_data_layer.domain import legacy as legacy_model
 
@@ -27,13 +28,29 @@ class GPPracticeTransformer(ServiceTransformer):
 
     Filter criteria:
     - The service must be active
+
+    Feature flag behavior:
+    - When data_migration_search_triage_code_enabled is disabled:
+      Only organisation is created (no location or healthcare_service)
+    - When enabled: All resources (organisation, location, healthcare_service) are created
     """
 
     def transform(self, service: legacy_model.Service) -> ServiceTransformOutput:
         """
         Transform the given GP practice service into the new data model format.
+
+        When the feature flag is disabled, only the organisation is created.
+        When enabled, organisation, location and healthcare_service are all created.
         """
         organisation = self.build_organisation(service)
+
+        if not is_enabled(FeatureFlag.DATA_MIGRATION_SEARCH_TRIAGE_CODE_ENABLED):
+            return ServiceTransformOutput(
+                organisation=[organisation],
+                healthcare_service=[],
+                location=[],
+            )
+
         location = self.build_location(service, organisation.id)
         healthcare_service = self.build_healthcare_service(
             service,