Merge pull request #1853 from GSA/main

7-22-2025 Production Deploy

Author: ccostino

.coveragerc

@@ -0,0 +1,5 @@
+[report]
+exclude_lines =
+    pragma: no cover
+    ^\s*logger\.
+    ^\s*current_app\.logger\.

.ds.baseline

@@ -229,7 +229,7 @@
         "filename": "tests/app/dao/test_services_dao.py",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 289,
+        "line_number": 290,
         "is_secret": false
       }
     ],
@@ -267,15 +267,15 @@
         "filename": "tests/app/notifications/test_receive_notification.py",
         "hashed_secret": "913a73b565c8e2c8ed94497580f619397709b8b6",
         "is_verified": false,
-        "line_number": 27,
+        "line_number": 28,
         "is_secret": false
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "tests/app/notifications/test_receive_notification.py",
         "hashed_secret": "d70eab08607a4d05faa2d0d6647206599e9abc65",
         "is_verified": false,
-        "line_number": 57,
+        "line_number": 58,
         "is_secret": false
       }
     ],
@@ -295,7 +295,7 @@
         "filename": "tests/app/service/test_rest.py",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 1282,
+        "line_number": 1288,
         "is_secret": false
       }
     ],
@@ -339,7 +339,7 @@
         "filename": "tests/app/user/test_rest.py",
         "hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
         "is_verified": false,
-        "line_number": 864,
+        "line_number": 874,
         "is_secret": false
       }
     ],
@@ -374,5 +374,5 @@
       }
     ]
   },
-  "generated_at": "2025-06-09T16:07:54Z"
+  "generated_at": "2025-07-02T18:56:01Z"
 }

.github/dependabot.yml

@@ -13,5 +13,3 @@ updates:
       - "dependabot" # Custom label to identify Dependabot PRs
     assignees:
       - "alexjanousekGSA"
-    reviewers:
-      - "alexjanousekGSA"

Makefile

@@ -70,8 +70,8 @@ run-celery: ## Run celery, TODO remove purge for staging/prod
 		-A run_celery.notify_celery worker \
 		--pidfile="/tmp/celery.pid" \
 		--loglevel=INFO \
-		--pool=threads
-		--concurrency=10
+		--pool=eventlet
+		--concurrency=20
 
 
 .PHONY: dead-code
@@ -104,8 +104,8 @@ test: ## Run tests and create coverage report
 	poetry run isort ./app ./tests
 	poetry run coverage run --omit=*/migrations/*,*/tests/* -m pytest --maxfail=10
 
-    ## TODO set this back to 95 asap
-	poetry run coverage report -m --fail-under=93
+
+	poetry run coverage report -m --fail-under=95
 	poetry run coverage html -d .coverage_cache
 
 .PHONY: test-debug

app/aws/s3.py

@@ -13,6 +13,7 @@
 from app.clients import AWS_CLIENT_CONFIG
 
 # from app.service.rest import get_service_by_id
+from app.utils import hilite
 from notifications_utils import aware_utcnow
 
 FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv"
@@ -22,11 +23,6 @@
 ttl = 60 * 60 * 24 * 7
 
 
-# Global variable
-s3_client = None
-s3_resource = None
-
-
 def get_service_id_from_key(key):
     key = key.replace("service-", "")
     key = key.split("/")
@@ -69,32 +65,31 @@ def clean_cache():
 
 
 def get_s3_client():
-    global s3_client
-    if s3_client is None:
-        access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"]
-        secret_key = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"]
-        region = current_app.config["CSV_UPLOAD_BUCKET"]["region"]
-        session = Session(
-            aws_access_key_id=access_key,
-            aws_secret_access_key=secret_key,
-            region_name=region,
-        )
-        s3_client = session.client("s3", config=AWS_CLIENT_CONFIG)
+
+    access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"]
+    secret_key = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"]
+    region = current_app.config["CSV_UPLOAD_BUCKET"]["region"]
+    session = Session(
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        region_name=region,
+    )
+    current_app.logger.info(hilite("About to call session.client"))
+    s3_client = session.client("s3", config=AWS_CLIENT_CONFIG)
+    current_app.logger.info(hilite("SESSION CALLED"))
     return s3_client
 
 
 def get_s3_resource():
-    global s3_resource
-    if s3_resource is None:
-        access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"]
-        secret_key = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"]
-        region = current_app.config["CSV_UPLOAD_BUCKET"]["region"]
-        session = Session(
-            aws_access_key_id=access_key,
-            aws_secret_access_key=secret_key,
-            region_name=region,
-        )
-        s3_resource = session.resource("s3", config=AWS_CLIENT_CONFIG)
+    access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"]
+    secret_key = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"]
+    region = current_app.config["CSV_UPLOAD_BUCKET"]["region"]
+    session = Session(
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        region_name=region,
+    )
+    s3_resource = session.resource("s3", config=AWS_CLIENT_CONFIG)
     return s3_resource
 
 

app/celery/nightly_tasks.py

@@ -8,7 +8,6 @@
 from app.aws.s3 import remove_csv_object
 from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
 from app.config import QueueNames
-from app.cronitor import cronitor
 from app.dao.fact_processing_time_dao import insert_update_processing_time
 from app.dao.inbound_sms_dao import delete_inbound_sms_older_than_retention
 from app.dao.jobs_dao import (
@@ -30,8 +29,7 @@
 from app.utils import get_midnight_in_utc, utc_now
 
 
-@notify_celery.task(name="remove_sms_email_jobs")
-@cronitor("remove_sms_email_jobs")
+@notify_celery.task(name="remove-sms-email-jobs")
 def remove_sms_email_csv_files():
     _remove_csv_files([NotificationType.EMAIL, NotificationType.SMS])
 
@@ -51,6 +49,7 @@ def cleanup_unfinished_jobs():
     for job in jobs:
         # The query already checks that the processing_finished time is null, so here we are saying
         # if it started more than 4 hours ago, that's too long
+        acceptable_finish_time = None
         try:
             if job.processing_started is not None:
                 acceptable_finish_time = job.processing_started + timedelta(minutes=5)
@@ -59,7 +58,7 @@ def cleanup_unfinished_jobs():
                 f"Job ID {job.id} processing_started is {job.processing_started}.",
             )
             raise
-        if now > acceptable_finish_time:
+        if acceptable_finish_time and now > acceptable_finish_time:
             remove_csv_object(job.original_file_name)
             dao_archive_job(job)
 
@@ -74,14 +73,12 @@ def delete_notifications_older_than_retention():
     )
 
 
-@notify_celery.task(name="delete-sms-notifications")
-@cronitor("delete-sms-notifications")
+@notify_celery.task(name="delete-sms-notifications-older-than-retention")
 def delete_sms_notifications_older_than_retention():
     _delete_notifications_older_than_retention_by_type(NotificationType.SMS)
 
 
-@notify_celery.task(name="delete-email-notifications")
-@cronitor("delete-email-notifications")
+@notify_celery.task(name="delete-email-notifications-older-than-retention")
 def delete_email_notifications_older_than_retention():
     _delete_notifications_older_than_retention_by_type(NotificationType.EMAIL)
 
@@ -161,7 +158,6 @@ def delete_notifications_for_service_and_type(
 
 
 @notify_celery.task(name="timeout-sending-notifications")
-@cronitor("timeout-sending-notifications")
 def timeout_notifications():
     notifications = ["dummy value so len() > 0"]
 
@@ -183,7 +179,6 @@ def timeout_notifications():
 
 
 @notify_celery.task(name="delete-inbound-sms")
-@cronitor("delete-inbound-sms")
 def delete_inbound_sms():
     try:
         start = utc_now()
@@ -199,7 +194,6 @@ def delete_inbound_sms():
 
 
 @notify_celery.task(name="save-daily-notification-processing-time")
-@cronitor("save-daily-notification-processing-time")
 def save_daily_notification_processing_time(local_date=None):
     # local_date is a string in the format of "YYYY-MM-DD"
     if local_date is None:

app/celery/reporting_tasks.py

@@ -4,7 +4,6 @@
 
 from app import notify_celery
 from app.config import QueueNames
-from app.cronitor import cronitor
 from app.dao.fact_billing_dao import fetch_billing_data_for_day, update_fact_billing
 from app.dao.fact_notification_status_dao import update_fact_notification_status
 from app.dao.notifications_dao import get_service_ids_with_notifications_on_date
@@ -13,7 +12,6 @@
 
 
 @notify_celery.task(name="create-nightly-billing")
-@cronitor("create-nightly-billing")
 def create_nightly_billing(day_start=None):
     # day_start is a datetime.date() object. e.g.
     # up to 4 days of data counting back from day_start is consolidated
@@ -58,7 +56,6 @@ def create_nightly_billing_for_day(process_day):
 
 
 @notify_celery.task(name="create-nightly-notification-status")
-@cronitor("create-nightly-notification-status")
 def create_nightly_notification_status():
     """
     Aggregate notification statuses into rows in ft_notification_status.

app/celery/tasks.py

@@ -167,7 +167,7 @@ def process_row(row, template, job, service, sender_id=None):
 # Assuming the limit is annual, is it calendar year, fiscal year, MOU year?
 # Do we need a command to run to clear the redis value, or should it happen automatically?
 def __total_sending_limits_for_job_exceeded(service, job, job_id):
-    print(hilite("ENTER __total_sending_limits_for_job_exceeded"))
+
     try:
         total_sent = check_service_over_total_message_limit(KeyType.NORMAL, service)
         if total_sent + job.notification_count > service.total_message_limit:

app/clients/pinpoint/aws_pinpoint.py

@@ -37,6 +37,7 @@ def validate_phone_number(self, country_code, phone_number):
             # TODO right now this will only print with AWS simulated numbers,
             # but remove this when that changes
             current_app.logger.info(hilite(response))
+            return response
         except ClientError:
             current_app.logger.exception(
                 "#notify-debug-validate-phone-number Could not validate with pinpoint"

app/commands.py

@@ -752,12 +752,13 @@ def create_admin_jwt():
 @notify_command(name="create-user-jwt")
 @click.option("-t", "--token", required=True, prompt=False)
 def create_user_jwt(token):
-    if getenv("NOTIFY_ENVIRONMENT", "") != "development":
+    if getenv("NOTIFY_ENVIRONMENT", "") not in ["development", "test"]:
         current_app.logger.error("Can only be run in development")
         return
     service_id = token[-73:-37]
     api_key = token[-36:]
-    current_app.logger.info(create_jwt_token(api_key, service_id))
+    token = create_jwt_token(api_key, service_id)
+    current_app.logger.info(token)
 
 
 def _update_template(id, name, template_type, content, subject):