Merge pull request #1666 from GSA/main

04/24/2025 Production Deploy

Author: ccostino

Makefile

@@ -64,7 +64,7 @@ too-complex:
 	poetry run radon cc ./app -a -nc
 
 .PHONY: run-flask
-run-flask: ## Run flask
+run-flask:
 	poetry run newrelic-admin run-program flask run -p 6011 --host=0.0.0.0
 
 .PHONY: run-celery

README.md

@@ -507,7 +507,7 @@ instructions above for more details.
   - [Deploying to Production](./docs/all.md#-deploying-to-production)
   - [Smoke-testing the App](./docs/all.md#-smoke-testing-the-app)
   - [Configuration Management](./docs/all.md#-configuration-management)
-  - [DNS Changes](./docs/all.md#-dns-changes)
+  - [DNS and Domain Changes](./docs/all.md#-dns-and-domain-changes)
   - [Exporting test results for compliance monitoring](./docs/all.md#exporting-test-results-for-compliance-monitoring)
   - [Known Gotchas](./docs/all.md#-known-gotchas)
   - [User Account Management](./docs/all.md#-user-account-management)

app/__init__.py

@@ -13,6 +13,7 @@
 from flask.ctx import has_app_context
 from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
+from flask_socketio import SocketIO
 from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
 from sqlalchemy import event
 from werkzeug.exceptions import HTTPException as WerkzeugHTTPException
@@ -94,6 +95,14 @@ def apply_driver_hacks(self, app, info, options):
 redis_store = RedisClient()
 document_download_client = DocumentDownloadClient()
 
+socketio = SocketIO(
+    cors_allowed_origins=[
+        config.Config.ADMIN_BASE_URL,
+    ],
+    message_queue=config.Config.REDIS_URL,
+    logger=True,
+    engineio_logger=True,
+)
 
 notification_provider_clients = NotificationProviderClients()
 
@@ -111,6 +120,11 @@ def create_app(application):
     application.config["NOTIFY_APP_NAME"] = application.name
     init_app(application)
 
+    socketio.init_app(application)
+
+    from app.socket_handlers import register_socket_handlers
+
+    register_socket_handlers(socketio)
     request_helper.init_app(application)
     db.init_app(application)
     migrate.init_app(application, db=db)

app/config.py

@@ -179,6 +179,7 @@ class Config(object):
     S3_RESOURCE = session.resource("s3", config=AWS_CLIENT_CONFIG)
 
     CELERY = {
+        "broker_connection_retry_on_startup": True,
         "worker_max_tasks_per_child": 500,
         "task_ignore_result": True,
         "result_persistent": False,

app/dao/notifications_dao.py

@@ -26,9 +26,11 @@
 from app import create_uuid, db
 from app.dao.dao_utils import autocommit
 from app.dao.inbound_sms_dao import Pagination
+from app.dao.jobs_dao import dao_get_job_by_id
 from app.enums import KeyType, NotificationStatus, NotificationType
 from app.models import FactNotificationStatus, Notification, NotificationHistory
 from app.utils import (
+    emit_job_update_summary,
     escape_special_characters,
     get_midnight_in_utc,
     midnight_n_days_ago,
@@ -895,6 +897,19 @@ def dao_update_delivery_receipts(receipts, delivered):
         f"#loadtestperformance batch update query time: \
         updated {len(receipts)} notification in {elapsed_time} ms"
     )
+    job_ids = (
+        db.session.execute(
+            select(Notification.job_id).where(
+                Notification.message_id.in_(id_to_carrier.keys())
+            )
+        )
+        .scalars()
+        .all()
+    )
+
+    for job_id in set(job_ids):
+        job = dao_get_job_by_id(job_id)
+        emit_job_update_summary(job)
 
 
 def dao_close_out_delivery_receipts():

app/socket_handlers.py

@@ -0,0 +1,16 @@
+from flask import current_app, request
+from flask_socketio import join_room, leave_room
+
+
+def register_socket_handlers(socketio):
+    @socketio.on("join")
+    def on_join(data):  # noqa: F401
+        room = data.get("room")
+        join_room(room)
+        current_app.logger.info(f"Socket {request.sid} joined room {room}")
+
+    @socketio.on("leave")
+    def on_leave(data):  # noqa: F401
+        room = data.get("room")
+        leave_room(room)
+        current_app.logger.info(f"Socket {request.sid} left room {room}")

app/utils.py

@@ -131,3 +131,18 @@ def utc_now():
 def debug_not_production(msg):
     if os.getenv("NOTIFY_ENVIRONMENT") not in ["production"]:
         current_app.logger.info(msg)
+
+
+def emit_job_update_summary(job):
+    from app import socketio
+
+    current_app.logger.info(f"Emitting summary for job {job.id}")
+    socketio.emit(
+        "job_updated",
+        {
+            "job_id": str(job.id),
+            "job_status": job.job_status,
+            "notification_count": job.notification_count,
+        },
+        room=f"job-{job.id}",
+    )

docs/all.md

@@ -53,7 +53,7 @@
   - [Smoke-testing the App](#-smoke-testing-the-app)
   - [Simulated bulk send testing](#-simulated-bulk-send-testing)
   - [Configuration Management](#-configuration-management)
-  - [DNS Changes](#-dns-changes)
+  - [DNS and Domain Changes](#-dns-and-domain-changes)
   - [Exporting test results for compliance monitoring](#exporting-test-results-for-compliance-monitoring)
   - [Known Gotchas](#-known-gotchas)
   - [User Account Management](#-user-account-management)
@@ -1068,7 +1068,7 @@ that the security of the system is maintained.
 1. [Smoke-testing the App](#smoke-testing)
 1. [Simulated bulk send testing](#simulated-bulk-send-testing)
 1. [Configuration Management](#cm)
-1. [DNS Changes](#dns)
+1. [DNS and Domain Changes](#dns)
 1. [Known Gotchas](#gotcha)
 1. [User Account Management](#ac)
 1. [SMS Phone Number Management](#phone-numbers)
@@ -1239,15 +1239,41 @@ US_Notify Administrators are responsible for ensuring that remediations for vuln
 - Low - 180 days
 - Informational - 365 days (depending on the analysis of the issue)
 
-## <a name="dns"></a> DNS Changes
+## <a name="dns"></a> DNS and Domain Changes
 
-Notify.gov DNS records are maintained within [the 18f/dns repository](https://github.com/18F/dns/blob/main/terraform/notify.gov.tf). To create new DNS records for notify.gov or any subdomains:
+Notify.gov DNS records are maintained within [the GSA-TTS/dns repository](https://github.com/GSA-TTS/dns/blob/main/terraform/notify.gov.tf), and the domains and routes are managed directly in our Cloud.gov production space.
 
-1. Update the `notify.gov.tf` terraform to update oƒr create the new records within Route53 and push the branch to the 18f/dns repository.
-1. Open a PR.
-1. Verify that the plan output within circleci creates the records that you expect.
-1. Request a PR review from the 18F/tts-tech-portfolio team
-1. Once the PR is approved and merged, verify that the apply step happened correctly within [CircleCI](https://app.circleci.com/pipelines/github/18F/dns)
+**Step 1:  Make changes to the DNS records**
+
+1. If you haven't already, clone a local copy of [the GSA-TTS/dns repository](https://github.com/GSA-TTS/dns).
+1. Create a new branch and update the [`notify.gov.tf`]((https://github.com/GSA-TTS/dns/blob/main/terraform/notify.gov.tf)) Terraform file to update, create, or remove DNS records within AWS Route 53.
+1. Open a PR in the repository and verify that the plan output within CircleCI makes the changes that you expect.
+1. Request a PR review from the `@tts-tech-operations` team within the GSA-TTS GitHub org.
+1. Once the PR is approved and merged, verify that the apply step happened correctly within [CircleCI](https://app.circleci.com/pipelines/github/GSA-TTS/dns).
+
+**Step 2:  Make changes to the domains and routes in Cloud.gov**
+
+The domains and routes are managed via the [external domain service](https://www.cloud.gov/docs/services/external-domain-service/) within Cloud.gov.
+
+If you're creating new domains:
+
+1. Sign in to the `cf` CLI in your terminal and target the `notify-production` space.
+1. Create the new domain(s) with [`cf create-private-domain`](https://docs.cloudfoundry.org/devguide/deploy-apps/routes-domains.html#private-domains).
+1. Map the routes needed to the new domain(s) with [`cf map-route`](https://docs.cloudfoundry.org/devguide/deploy-apps/routes-domains.html#map-route).
+1. Update the service to account for the new domain(s): `cf update-service notify-admin-domain-production -c '{"domains": "example.gov,www.example.gov,..."}'` (make sure to list *all* domains that need to be accounted for, including any existing ones that you want to keep!).
+
+If you're removing existing domains:
+
+1. Sign in to the `cf` CLI in your terminal and target the `notify-production` space.
+1. Unmap the routes to the existing domain(s) with [`cf unmap-route`](https://docs.cloudfoundry.org/devguide/deploy-apps/routes-domains.html#unmap-route).
+1. Delete the existing domain(s) with [`cf delete-private-domain`](https://docs.cloudfoundry.org/devguide/deploy-apps/routes-domains.html#private-domains).
+1. Update the service to account for the deleted domain(s): `cf update-service notify-admin-domain-production -c '{"domains": "example.gov,www.example.gov,..."}'` (make sure to list *all* domains that need to be accounted for, including any existing ones that you want to keep!).
+
+**Step 3:  Redeploy or restage the Admin app:**
+
+Restage or redeploy the `notify-admin-production` app.  To restage, you can trigger the action in GitHub or run the command directly: `cf restage notify-admin-production --strategy rolling`.
+
+Test that the changes took effect properly by going to the domain(s) that were adjusted and seeing if they resolve correctly and/or no longer resolve as expected. Note that this may take up to 72 hours, depending on how long it takes for the DNS changes to propogate.
 
 ## Exporting test results for compliance monitoring
 
@@ -1507,3 +1533,19 @@ Note: better to search on space 'notify-production' rather than specifically for
 #notify-admin-1505 (general login issues)
 #notify-admin-1701 (wrong sender phone number)
 #notify-admin-1859 (job is created with created_at being the wrong time)
+
+### refreshing the login.gov certificate
+
+1. generate certificate:   `openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.crt -nodes`
+2. update the github secrets for staging, demo, production (contents of key.pem go in LOGIN_PEM and contents of cert.crt in LOGIN_PUB).
+DO NOT RESTAGE YET.
+3. use the same certificate for staging, demo, and production
+4. login to the login.gov partner app (https://portal.int.identitysandbox.gov)
+5. add the new certificate to the production version of Notify in the partner app (our partner app account has sandbox and production)
+6. Make a Zendesk support request for login.gov to push the new version of Notify (https://zendesk.login.gov)
+7. Do not delete the old certificate, because you need things to keep working until you complete the transition.
+8. When you receive an email from login.gov that the app has been pushed successfully, restage notify on the staging tier
+9. If staging works, you can restage demo and production
+10. Delete the old certificate in the partner app, send another zendesk request to push again.  This is best practice but a lower
+priority, because certificates eventually expire anyway and we have changed the certificate in github secrets, so the old cert is
+no longer relevant.

package-lock.json

@@ -0,0 +1,5 @@
+{
+  "dependencies": {
+    "socket.io-client": "^4.8.1"
+  }
+}