Merge pull request #2849 from fedspendingtransparency/fix/dev-6456-es-field-mismatch

[DEV-6456] fixed columns in ES transactions ETL and renamed one in code

Author: tony-sappe

usaspending_api/awards/v2/lookups/elasticsearch_lookups.py

@@ -8,7 +8,7 @@
 TRANSACTIONS_LOOKUP = {
     "Recipient Name": "recipient_name.keyword",
     "Action Date": "action_date",
-    "Transaction Amount": "transaction_amount",
+    "Transaction Amount": "federal_action_obligation",
     "Award Type": "type_description.keyword",
     "Awarding Agency": "awarding_toptier_agency_name.keyword",
     "Awarding Sub Agency": "awarding_subtier_agency_name.keyword",

usaspending_api/database_scripts/etl/transaction_delta_view.sql

@@ -12,6 +12,16 @@ SELECT
     "fain",
     "uri",
     "piid",
+    CASE
+        WHEN "detached_award_proc_unique" IS NOT NULL THEN 'CONT_TX_' || "detached_award_proc_unique"
+        WHEN "afa_generated_unique" IS NOT NULL THEN 'ASST_TX_' || "afa_generated_unique"
+        ELSE NULL
+    END AS generated_unique_transaction_id,
+    CASE
+        WHEN "type" IN ('02', '03', '04', '05', '06', '10', '07', '08', '09', '11') AND "fain" IS NOT NULL THEN "fain"
+        WHEN "piid" IS NOT NULL THEN "piid"  -- contracts. Did it this way to easily handle IDV contracts
+        ELSE "uri"
+    END AS display_award_id,
     "action_date",
     "fiscal_action_date",
     "last_modified_date",
@@ -43,7 +53,6 @@ SELECT
     "ordering_period_end_date",
     "cfda_number",
     "cfda_title",
-    "cfda_id",
     "pop_country_name",
     "pop_country_code",
     "pop_state_code",
@@ -68,8 +77,6 @@ SELECT
     "parent_recipient_unique_id",
     "awarding_agency_id",
     "funding_agency_id",
-    "awarding_toptier_agency_id",
-    "funding_toptier_agency_id",
     "awarding_toptier_agency_name",
     "funding_toptier_agency_name",
     "awarding_subtier_agency_name",
@@ -78,7 +85,6 @@ SELECT
     "funding_toptier_agency_abbreviation",
     "awarding_subtier_agency_abbreviation",
     "funding_subtier_agency_abbreviation",
-    "treasury_account_identifiers",
     "tas_paths",
     "tas_components",
     "federal_accounts",

usaspending_api/etl/es_transaction_template.json

@@ -6,7 +6,7 @@
     "index": {
       "number_of_shards" : 5,
       "number_of_replicas" : 0,
-      "sort.field" : ["transaction_amount.keyword", "action_date"],
+      "sort.field" : ["federal_action_obligation.keyword", "action_date"],
       "sort.order" : ["desc", "desc"],
       "sort.missing": ["_last", "_last"]
     },
@@ -218,7 +218,7 @@
         "type": "scaled_float",
         "scaling_factor": 100
       },
-      "transaction_amount": {
+      "federal_action_obligation": {
         "type": "scaled_float",
         "scaling_factor": 100,
         "fields": {

usaspending_api/etl/management/commands/es_rapidloader.py

@@ -388,7 +388,7 @@ def complete_process(self) -> None:
     "fiscal_year",
     "award_fiscal_year",
     "award_amount",
-    "transaction_amount",
+    "federal_action_obligation",
     "face_value_loan_guarantee",
     "original_loan_subsidy_cost",
     "generated_pragmatic_obligation",

usaspending_api/search/tests/unit/test_elasticsearch_helpers.py

@@ -130,13 +130,13 @@ def test_swap_keys():
     test = {
         "Recipient Name": "recipient_name",
         "Action Date": "action_date",
-        "Transaction Amount": "transaction_amount",
+        "Transaction Amount": "federal_action_obligation",
     }
 
     results = swap_keys(test)
 
     assert results == {
         "recipient_name": "recipient_name",
         "action_date": "action_date",
-        "transaction_amount": "transaction_amount",
+        "federal_action_obligation": "federal_action_obligation",
     }

usaspending_api/search/v2/elasticsearch_helper.py

@@ -69,7 +69,7 @@ def spending_by_transaction_count(request_data):
     return response
 
 
-def get_sum_aggregation_results(keyword, field="transaction_amount"):
+def get_sum_aggregation_results(keyword, field="federal_action_obligation"):
     group_by_agg_key_values = {"field": field}
     aggs = A("sum", **group_by_agg_key_values)
     filter_query = QueryWithFilters.generate_transactions_elasticsearch_query(
@@ -133,7 +133,7 @@ def get_sum_and_count_aggregation_results(keyword):
         {"keyword_search": [es_minimal_sanitize(keyword)]}
     )
     search = TransactionSearch().filter(filter_query)
-    search.aggs.bucket("prime_awards_obligation_amount", {"sum": {"field": "transaction_amount"}})
+    search.aggs.bucket("prime_awards_obligation_amount", {"sum": {"field": "federal_action_obligation"}})
     search.aggs.bucket("prime_awards_count", {"value_count": {"field": "transaction_id"}})
     response = search.handle_execute()