Build new ECS cluster in stage; move celery; launch flower (#617)

* Get up to date with main, start in on new configs

* Build accounts-stage megacluster

* Move Redis cluster def after Fargate clusters, use SGs

* Add new AFC in stage, get Celery working there

* Add Flower service to accounts-stage

* Tidy up

* Update stage image

* Turn off the old celery cluster; update CI accordingly

* Use variablized images

* Update celery scale

* Flower should never need to scale past 1 container

* Try to dry out the stage config

* Remove bastion

* Tidy up

* Tidy up

* Tidy up

Author: ryanjjung

.github/workflows/merge.yml

@@ -162,17 +162,7 @@ jobs:
           # Create a YAML config stump containing only the nested tree leading to the image tag update
           cat << EOF > newimage.yaml
           resources:
-            tb:fargate:FargateClusterWithLogging:
-              accounts:
-                task_definition:
-                  container_definitions:
-                    accounts:
-                      image: "$ECR_TAG"
-              accounts-celery:
-                task_definition:
-                  container_definitions:
-                    accounts:
-                      image: "$ECR_TAG"
+            .accounts_image: &ACCOUNTS_IMAGE $ECR_TAG
           EOF
 
           # Use yq to merge the stump into the main config
@@ -183,7 +173,8 @@ jobs:
           pulumi stack select thunderbird/stage
           pulumi up -y --diff \
             --target 'urn:pulumi:stage::accounts::tb:fargate:FargateClusterWithLogging$aws:ecs/taskDefinition:TaskDefinition::accounts-stage-fargate-accounts-taskdef' \
-            --target 'urn:pulumi:stage::accounts::tb:fargate:FargateClusterWithLogging$aws:ecs/taskDefinition:TaskDefinition::accounts-stage-fargate-accounts-celery-taskdef' \
+            --target 'urn:pulumi:stage::accounts::aws:ecs/taskDefinition:TaskDefinition::accounts-stage-afc-accounts-taskdef-celery' \
+            --target 'urn:pulumi:stage::accounts::aws:ecs/taskDefinition:TaskDefinition::accounts-stage-afc-accounts-taskdef-flower' \
             --target-dependents
 
   # When accounts changes are merged in, build and push a new Docker image
@@ -296,12 +287,7 @@ jobs:
           # Create a YAML config stump containing only the nested tree leading to the image tag update
           cat << EOF > newimage.yaml
           resources:
-            tb:fargate:FargateClusterWithLogging:
-              keycloak:
-                task_definition:
-                  container_definitions:
-                    keycloak:
-                      image: "$ECR_TAG"
+            keycloak_image: &KEYCLOAK_IMAGE $ECR_TAG
           EOF
 
           # Use yq to merge the stump into the main config

.github/workflows/validate.yml

@@ -104,7 +104,6 @@ jobs:
 
   run-e2e-tests-local:
     needs: detect-changes
-    # if: needs.detect-changes.outputs.iac-changed
     if: false
     runs-on: ubuntu-latest
     environment: staging

pulumi/__main__.py

@@ -77,16 +77,17 @@
         **instance_opts,
     )
 
-# Build an ElastiCache Redis cluster allowing access from the Accounts containers
-redis_opts = resources['tb:elasticache:ElastiCacheReplicaGroup']['accounts']
-redis = tb_pulumi.elasticache.ElastiCacheReplicationGroup(
-    name=f'{project.name_prefix}-redis',
-    project=project,
-    source_sgids=[container_sgs['accounts'].resources['sg'].id, container_sgs['accounts-celery'].resources['sg'].id],
-    subnets=vpc.resources['subnets'],
-    opts=pulumi.ResourceOptions(depends_on=[vpc, container_sgs['accounts']]),
-    **redis_opts,
-)
+# Build a single Fargate cluster to run and scale all our accounts-related services
+autoscaling_fargate_clusters = {
+    cluster_name: tb_pulumi.fargate.AutoscalingFargateCluster(
+        f'{project.name_prefix}-afc-{cluster_name}',
+        project=project,
+        subnets=vpc.resources['subnets'],
+        **cluster_config,
+    )
+    for cluster_name, cluster_config in resources.get('tb:fargate:AutoscalingFargateCluster', {}).items()
+}
+
 
 # Build Fargate clusters to run our containers
 autoscalers = {}
@@ -125,6 +126,24 @@
             **autoscaler_opts,
         )
 
+# Build an ElastiCache Redis cluster allowing access from the Accounts containers
+redis_opts = resources['tb:elasticache:ElastiCacheReplicaGroup']['accounts']
+redis_source_sgids = [
+    container_sgs['accounts'].resources['sg'].id,
+    container_sgs['accounts-celery'].resources['sg'].id,
+]
+for afc_name, afc in autoscaling_fargate_clusters.items():
+    for container_name, lbs in afc.resources['container_security_groups'].items():
+        redis_source_sgids.extend([sg.resources['sg'].id for sg in lbs.values()])
+redis = tb_pulumi.elasticache.ElastiCacheReplicationGroup(
+    name=f'{project.name_prefix}-redis',
+    project=project,
+    source_sgids=redis_source_sgids,
+    subnets=vpc.resources['subnets'],
+    opts=pulumi.ResourceOptions(depends_on=[vpc, container_sgs['accounts']]),
+    **redis_opts,
+)
+
 
 cloudflare_backend_record = cloudflare.Record(
     f'{project.name_prefix}-dns-backend',