Merge pull request #2946 from fedspendingtransparency/warmfix/transaction-search-deletes-qat

[Warmfix] Follows pattern for deleting stale fabs/fpds to also delete transaction_search

Author: tony-sappe

usaspending_api/broker/helpers/delete_stale_fabs.py

@@ -32,11 +32,13 @@ def delete_stale_fabs(ids_to_delete):
     if delete_transaction_ids:
         fabs = 'DELETE FROM "transaction_fabs" tf WHERE tf."transaction_id" IN ({});'
         tn = 'DELETE FROM "transaction_normalized" tn WHERE tn."id" IN ({});'
+        ts = 'DELETE FROM "transaction_search" ts WHERE ts."transaction_id" IN ({});'
         td = "DELETE FROM transaction_delta td WHERE td.transaction_id in ({});"
         queries.extend(
             [
                 fabs.format(delete_transaction_str_ids),
                 tn.format(delete_transaction_str_ids),
+                ts.format(delete_transaction_str_ids),
                 td.format(delete_transaction_str_ids),
             ]
         )

usaspending_api/conftest.py

@@ -6,10 +6,11 @@
 
 from django.conf import settings
 from django.core.management import call_command
-from django.db import connections, connection
+from django.db import connections
 from django.test import override_settings
 from pathlib import Path
 
+from usaspending_api.common.helpers.sql_helpers import execute_sql_simple
 from usaspending_api.common.elasticsearch.elasticsearch_sql_helpers import (
     ensure_view_exists,
     ensure_business_categories_functions_exist,
@@ -46,14 +47,24 @@ def pytest_addoption(parser):
 def delete_tables_for_tests():
     """
     Outside of testing, the transaction_search table is created by using a series of chunked matviews that are combined
-    into a Django managed table. When unit testing transaction_search is created as a single matview. To prevent a
+    into a Django managed table. When unit testing transaction_search is created as a single view. To prevent a
     naming conflict, the unused Django managed table is deleted while testing.
     """
-    with connection.cursor() as cursor:
-        try:
-            cursor.execute("DROP TABLE IF EXISTS transaction_search;")
-        except Exception:
-            pass
+    try:
+        execute_sql_simple("DROP TABLE IF EXISTS transaction_search;")
+    except Exception:
+        pass
+
+
+def add_view_protection():
+    """
+    When unit testing transaction_search is created as a single view. Views can't be deleted from, so a custom rule
+    is added to transaction_search to prevent sql errors when deletes are attempted
+    """
+    try:
+        execute_sql_simple("CREATE RULE ts_del_protect AS ON DELETE TO transaction_search DO INSTEAD NOTHING;")
+    except Exception:
+        pass
 
 
 @pytest.fixture(scope="session")
@@ -108,6 +119,7 @@ def django_db_setup(
             generate_matviews(materialized_views_as_traditional_views=True)
             ensure_view_exists(settings.ES_TRANSACTIONS_ETL_VIEW_NAME)
             ensure_view_exists(settings.ES_AWARDS_ETL_VIEW_NAME)
+            add_view_protection()
             ensure_business_categories_functions_exist()
             call_command("load_broker_static_data")
 

usaspending_api/etl/transaction_loaders/fpds_loader.py

@@ -68,14 +68,23 @@ def delete_stale_fpds(detached_award_procurement_ids):
         cursor.execute(f"delete from transaction_fpds where transaction_id in ({txn_id_str}) returning transaction_id")
         deleted_fpds = set(cursor.fetchall())
 
+        cursor.execute(
+            f"delete from transaction_search where transaction_id in ({txn_id_str}) returning transaction_id"
+        )
+        deleted_ts = set(cursor.fetchall())
+
         cursor.execute(f"delete from transaction_normalized where id in ({txn_id_str}) returning id")
         deleted_transactions = set(cursor.fetchall())
 
         if deleted_transactions != deleted_fpds:
             msg = "Delete Mismatch! Counts of transaction_normalized ({}) and transaction_fpds ({}) deletes"
             raise RuntimeError(msg.format(len(deleted_transactions), len(deleted_fpds)))
 
-        logger.info(f"{len(deleted_fpds):,} transactions deleted")
+        if deleted_transactions != deleted_ts:
+            msg = "Delete Mismatch! Counts of transaction_normalized ({}) and transaction_search ({}) deletes"
+            raise RuntimeError(msg.format(len(deleted_transactions), len(deleted_ts)))
+
+        logger.info(f"{len(deleted_transactions):,} transactions deleted")
 
         return awards_touched