safety(pulumi): deploy cold (desired_count 0), suspend autoscaling, disable schedules

Author: Jonathan Alvarez Delgado

.gitignore

@@ -50,6 +50,6 @@ venv*
 
 # Pulumi local artefacts (outputs, notes, analysis)
 infra/pulumi/pulumi-*.txt
-infra/pulumi/preview-output-*.txt
+infra/pulumi/preview-*.txt
 infra/pulumi/analysis.md
 infra/pulumi/infrastructure-inventory.md

infra/pulumi/__main__.py

@@ -780,7 +780,9 @@ def main():
                             "help",
                         ],  # Default; again overridden per schedule
                         "environment": [
-                            {"name": "DJANGO_SETTINGS_MODULE", "value": "settings"}
+                            {"name": "DJANGO_SETTINGS_MODULE", "value": "settings_local_stage"},
+                            {"name": "BOOTSTRAP_SAFE", "value": "true"},
+                            {"name": "NETAPP_STORAGE_ROOT", "value": "/tmp/storage"},
                         ],
                         "logConfiguration": {
                             "logDriver": "awslogs",
@@ -936,7 +938,7 @@ def main():
                         {"containerOverrides": [{"name": "cron", "command": command}]}
                     ),
                 ),
-                state="ENABLED",
+                state=task_config.get("state", "DISABLED"),
                 opts=pulumi.ResourceOptions(
                     parent=schedule_group,
                     depends_on=[cron_task_definition, scheduler_role],

infra/pulumi/config.stage.yaml

@@ -20,17 +20,17 @@ resources:
     addons-server:
       name: atn-stage-addons-server
       image_tag_mutability: MUTABLE
-      force_delete: true  # Stage only: allows pulumi destroy with images present
+      force_delete: false  # Repo now holds CI-built images; protect from accidental destroy
       scan_on_push: true
       encryption_type: AES256
-      # Lifecycle policy keep last 50 tagged images (here any tag), expire untagged after 7 days
-      # This catches SHA tags, stage-latest, and any future tag patterns
+      # Lifecycle policy keep last 50 tagged images (stage-/sha- prefixes), expire untagged after 7 days
+      # Matches tags prefixed with stage- (e.g., stage-latest) and sha- (commit SHAs)
       lifecycle_policy: |
         {
           "rules": [
             {
               "rulePriority": 1,
-              "description": "Keep last 50 tagged images (any tag)",
+              "description": "Keep last 50 tagged images (stage-/sha- prefixes)",
               "selection": {
                 "tagStatus": "tagged",
                 "tagPrefixList": ["stage-", "sha-"],
@@ -150,10 +150,7 @@ resources:
   # Image: 768512802988.dkr.ecr.us-west-2.amazonaws.com/atn-stage-addons-server:stage-latest
   tb:fargate:FargateClusterWithLogging:
     web:
-      # desired_count intentionally omitted: autoscaling owns the count.
-      # tb_pulumi sets ignore_changes on desired_count when not specified,
-      # preventing Pulumi from fighting the autoscaler. min_capacity in
-      # the autoscaling config acts as the effective baseline
+      desired_count: 0        # Start cold; scale up manually after validation
       assign_public_ip: false
       internal: false  # Public-facing ALB
       enable_container_insights: true
@@ -196,6 +193,8 @@ resources:
             environment:
             - name: DJANGO_SETTINGS_MODULE
               value: settings_local_stage
+            - name: BOOTSTRAP_SAFE
+              value: 'true'
             - name: UWSGI_PROCESSES
               value: '4'
             - name: UWSGI_THREADS
@@ -211,7 +210,7 @@ resources:
     # t3a.large has 8GB RAM - needed for addons-linter memory requirements
     # Multiple queue groups for different workloads
     worker:
-      # desired_count omitted: autoscaling owns the count (see web comment above)
+      desired_count: 0        # Start cold; scale up manually after validation
       assign_public_ip: false
       internal: true
       build_load_balancer: false  # Workers don't need ALB
@@ -234,6 +233,8 @@ resources:
             environment:
             - name: DJANGO_SETTINGS_MODULE
               value: settings_local_stage
+            - name: BOOTSTRAP_SAFE
+              value: 'true'
             - name: CELERY_CONCURRENCY
               value: '4'
             - name: CELERY_QUEUES
@@ -249,7 +250,7 @@ resources:
     # Separate ALB endpoint for version checking API (versioncheck.addons.thunderbird.net)
     # Lightweight service - c7a.medium equivalent
     versioncheck:
-      # desired_count omitted: autoscaling owns the count (see web comment)
+      desired_count: 0        # Start cold; scale up manually after validation
       assign_public_ip: false
       internal: false
       enable_container_insights: true
@@ -292,6 +293,8 @@ resources:
             environment:
             - name: DJANGO_SETTINGS_MODULE
               value: settings_local_stage
+            - name: BOOTSTRAP_SAFE
+              value: 'true'
             - name: UWSGI_PROCESSES
               value: '4'
             - name: UWSGI_THREADS
@@ -326,23 +329,26 @@ resources:
     web:
       cpu_threshold: 70
       ram_threshold: 70
-      min_capacity: 2
+      min_capacity: 0        # Start at 0; scale up manually after validation
       max_capacity: 8
       cooldown: 300
+      suspend: true           # Suspended until services are validated
 
     worker:
       cpu_threshold: 70
       ram_threshold: 80      # Higher: addons-linter memory spikes are normal
-      min_capacity: 2
+      min_capacity: 0
       max_capacity: 6
       cooldown: 300
+      suspend: true
 
     versioncheck:
       cpu_threshold: 70
       ram_threshold: 70
-      min_capacity: 1
+      min_capacity: 0
       max_capacity: 4
       cooldown: 300
+      suspend: true
 
   # =============================================================================
   # ElastiCache - Memcached (intended to replace current Memcached setup)

settings_local_stage.py

@@ -34,9 +34,19 @@ def get_secret(secret_name, region_name="us-west-2"):
         raise Exception(f"Failed to retrieve secret {secret_name}: {e}")
 
 
+# -----------------------------------------------------------------------------
+# Bootstrap safety toggle
+# -----------------------------------------------------------------------------
+# When BOOTSTRAP_SAFE is true we deliberately use RO database credentials
+# (if present) so that even if something accidentally starts, MySQL itself
+# enforces read-only access
+BOOTSTRAP_SAFE = env.bool("BOOTSTRAP_SAFE", default=False)
+MYSQL_SECRET_NAME = "atn/stage/mysql_ro" if BOOTSTRAP_SAFE else "atn/stage/mysql"
+
+
 # Retrieve secrets from AWS Secrets Manager
 _email_url_secret = get_secret('atn/stage/email_url')
-_mysql_secret = get_secret('atn/stage/mysql')
+_mysql_secret = get_secret(MYSQL_SECRET_NAME)
 _inbound_email_secret = get_secret('atn/stage/inbound_email')
 _django_secret = get_secret('atn/stage/django_secret_key')
 _celery_broker_secret = get_secret('atn/stage/celery_broker')
@@ -288,7 +298,7 @@ def get_secret(secret_name, region_name="us-west-2"):
 
 ES_DEFAULT_NUM_SHARDS = 10
 
-READ_ONLY = env.bool('READ_ONLY', default=False)
+READ_ONLY = env.bool("READ_ONLY", default=BOOTSTRAP_SAFE)
 
 # TODO: Github user ?
 GITHUB_API_USER = ''