Merge pull request #1776 from GSA/main

06/12/25 Production Deploy

Author: ccostino

.ds.baseline

@@ -127,16 +127,6 @@
     }
   ],
   "results": {
-    ".github/actions/deploy-proxy/action.yml": [
-      {
-        "type": "Hex High Entropy String",
-        "filename": ".github/actions/deploy-proxy/action.yml",
-        "hashed_secret": "a6c13f5da3788e8d654cd24001dc79a238723248",
-        "is_verified": false,
-        "line_number": 18,
-        "is_secret": false
-      }
-    ],
     ".github/workflows/checks.yml": [
       {
         "type": "Secret Keyword",
@@ -161,15 +151,15 @@
         "filename": ".github/workflows/daily_checks.yml",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 71,
+        "line_number": 63,
         "is_secret": false
       },
       {
         "type": "Basic Auth Credentials",
         "filename": ".github/workflows/daily_checks.yml",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 87,
+        "line_number": 79,
         "is_secret": false
       }
     ],
@@ -305,7 +295,7 @@
         "filename": "tests/app/service/test_rest.py",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 1285,
+        "line_number": 1282,
         "is_secret": false
       }
     ],
@@ -384,5 +374,5 @@
       }
     ]
   },
-  "generated_at": "2025-06-02T13:22:36Z"
+  "generated_at": "2025-06-09T16:07:54Z"
 }

.github/actions/deploy-proxy/action.yml

@@ -15,7 +15,7 @@ inputs:
     default: https://github.com/GSA-TTS/cg-egress-proxy.git
   proxy_version:
     description: git ref to be deployed
-    default: 1500c67157c1a7a6fbbda7a2de172b3d0a67e703
+    default: main
 runs:
   using: composite
   steps:

.github/actions/setup-project/action.yml

@@ -9,10 +9,10 @@ runs:
         sudo apt-get update \
         && sudo apt-get install -y --no-install-recommends \
         libcurl4-openssl-dev
-    - name: Set up Python 3.12.3
+    - name: Set up Python 3.12.2
       uses: actions/setup-python@v4
       with:
-        python-version: "3.12.3"
+        python-version: "3.12.2"
     - name: Install poetry
       shell: bash
       run: pip install poetry==2.1.3

.github/workflows/checks.yml

@@ -87,15 +87,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-project
       - name: Create requirements.txt
-        run: poetry export --output requirements_tmp.txt --without-hashes
-      - name: Filter requirements.txt
-        run: grep -v "oscrypto@ git" requirements_tmp.txt > requirements.txt
-      - name: Verify requirements.txt
-        run: ls -l requirements.txt
-      - name: Print requirements.txt
-        run: |
-          echo "Contents of requirements.txt:"
-          cat requirements.txt
+        run: poetry export --output requirements.txt
       - uses: pypa/gh-action-pip-audit@v1.1.0
         with:
           inputs: requirements.txt

.github/workflows/daily_checks.yml

@@ -26,15 +26,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-project
       - name: Create requirements.txt
-        run: poetry export --output requirements_tmp.txt --without-hashes
-      - name: Filter requirements.txt
-        run: grep -v "oscrypto@ git" requirements_tmp.txt > requirements.txt
-      - name: Verify requirements.txt
-        run: ls -l requirements.txt
-      - name: Print requirements.txt
-        run: |
-          echo "Contents of requirements.txt:"
-          cat requirements.txt
+        run: poetry export --output requirements.txt
       - uses: pypa/gh-action-pip-audit@v1.1.0
         with:
           inputs: requirements.txt

Makefile

@@ -108,6 +108,10 @@ test: ## Run tests and create coverage report
 	poetry run coverage report -m --fail-under=93
 	poetry run coverage html -d .coverage_cache
 
+.PHONY: test-debug
+test-debug:
+	poetry run pytest --pdb -x
+
 .PHONY: py-lock
 py-lock: ## Syncs dependencies and updates lock file without performing recursive internal updates
 	poetry sync --no-root

README.md

@@ -40,7 +40,7 @@ You will need the following items:
 This project currently works with these major versions of the following main
 components:
 
-- Python 3.12.x
+- Python 3.13.x
 - PostgreSQL 15.x (version 12.x is used in the hosted environments)
 
 These instructions will walk you through how to set your machine up with all of
@@ -173,12 +173,12 @@ session to make the changes take effect.
 Now we're ready to install the Python version we need with `pyenv`, like so:
 
 ```sh
-pyenv install 3.12
+pyenv install 3.13
 ```
 
-This will install the latest version of Python 3.12.
+This will install the latest version of Python 3.13.
 
-_NOTE: This project currently runs on Python 3.12.x._
+_NOTE: This project currently runs on Python 3.13.x._
 
 #### Python Dependency Installation
 
@@ -311,10 +311,10 @@ If you're upgrading an existing project to a newer version of Python, you can
 follow these steps to get yourself up-to-date.
 
 First, use `pyenv` to install the newer version of Python you'd like to use;
-we'll use `3.12` in our example here since we recently upgraded to this version:
+we'll use `3.13` in our example here since we recently upgraded to this version:
 
 ```sh
-pyenv install 3.12
+pyenv install 3.13
 ```
 
 Next, delete the virtual environment you previously had set up.  If you followed

app/__init__.py

@@ -11,7 +11,6 @@
 from celery import Celery, Task, current_task
 from flask import current_app, g, has_request_context, jsonify, make_response, request
 from flask.ctx import has_app_context
-from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
 from flask_socketio import SocketIO
 from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
@@ -86,7 +85,6 @@ def apply_driver_hacks(self, app, info, options):
     }
 )
 migrate = Migrate()
-ma = Marshmallow()
 notify_celery = NotifyCelery()
 aws_ses_client = AwsSesClient()
 aws_ses_stub_client = AwsSesStubClient()
@@ -131,7 +129,6 @@ def create_app(application):
     request_helper.init_app(application)
     db.init_app(application)
     migrate.init_app(application, db=db)
-    ma.init_app(application)
     zendesk_client.init_app(application)
     logging.init_app(application)
     aws_sns_client.init_app(application)

app/aws/s3.py

@@ -11,6 +11,8 @@
 
 from app import job_cache, job_cache_lock
 from app.clients import AWS_CLIENT_CONFIG
+
+# from app.service.rest import get_service_by_id
 from notifications_utils import aware_utcnow
 
 FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv"
@@ -162,6 +164,34 @@ def cleanup_old_s3_objects():
         current_app.logger.exception(
             "#delete-old-s3-objects An error occurred while cleaning up old s3 objects",
         )
+    try:
+        response = s3_client.list_objects_v2(Bucket=bucket_name)
+
+        service_ids = set()
+        while True:
+            for obj in response.get("Contents", []):
+                # Get the service id out of the upload key
+                key = obj["Key"]
+                object_arr = key.split("/")
+                service_id = object_arr[0]
+                service_id = service_id.replace("-service-notify", "")
+                service_ids.add(service_id)
+            if "NextContinuationToken" in response:
+                response = s3_client.list_objects_v2(
+                    Bucket=bucket_name,
+                    ContinuationToken=response["NextContinuationToken"],
+                )
+            else:
+                break
+            retained_services = []
+            for service_id in service_ids:
+                retained_services.append(service_id)
+
+        return service_ids
+    except Exception as error:
+        current_app.logger.exception(
+            f"#delete-old-s3-objects An error occurred while cleaning up old s3 objects: {str(error)}"
+        )
 
 
 def get_job_id_from_s3_object_key(key):

app/celery/tasks.py

@@ -19,6 +19,7 @@
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
+from app.dao.services_dao import dao_fetch_service_by_id
 from app.dao.templates_dao import dao_get_template_by_id
 from app.enums import JobStatus, KeyType, NotificationType
 from app.errors import TotalRequestsError
@@ -496,7 +497,15 @@ def clean_job_cache():
 
 @notify_celery.task(name="delete-old-s3-objects")
 def delete_old_s3_objects():
-    s3.cleanup_old_s3_objects()
+
+    existing_service_ids = s3.cleanup_old_s3_objects()
+    service_names = []
+    for service_id in existing_service_ids:
+        service = dao_fetch_service_by_id(service_id)
+        service_names.append(service.name)
+    current_app.logger.info(
+        f"#delete-old-s3-objects Services with retained csvs: {service_names}"
+    )
 
 
 @notify_celery.task(name="process-incomplete-jobs")