Sprint 206 Prod Deploy

.github/workflows/pull-request-checks.yaml

@@ -44,95 +44,64 @@ jobs:
       needs.Cancel-If-Draft-Pull-Request.result == 'skipped'
     uses: ./.github/workflows/build-broker-docker-image-for-test.yaml
 
-  # Prepare Code Climate to receive coverage report
-  Code-Climate-Before-Build:
-    name: Code Climate Before Build
-    needs: Run-Code-Style-Checks
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/code-climate-before-build.yaml
-    with:
-      source-branch: ${{ github.head_ref }}
-      source-commit-sha: ${{ github.event.pull_request.head.sha }}
-    secrets:
-      cc_test_reporter_id: ${{ secrets.CC_TEST_REPORTER_ID }}
-
   # The test suite is broken into multiple groups based on a combination of what is under test
   # and the time that it takes to run tests for that group.
   Run-Spark-Integration-Load-Transaction-FABS-and-FPDS-Tests:
     name: Run Spark Integration Load Transactions FABS and FPDS Tests
     needs:
-      - Code-Climate-Before-Build
+      - Run-Code-Style-Checks
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
 
   Run-Spark-Integration-Load-Transactions-Lookup-Tests:
     name: Run Spark Integration Load Transactions Lookup Tests
     needs:
-      - Code-Climate-Before-Build
+      - Run-Code-Style-Checks
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-load-transactions-lookup.yaml
 
   Run-Spark-Integration-Load-To-From-Delta-Tests:
     name: Run Spark Integration Load To From Delta Tests
     needs:
-      - Code-Climate-Before-Build
+      - Run-Code-Style-Checks
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-load-to-from-delta.yaml
 
   Run-Spark-Integration-Other-Tests:
     name: Run Spark Integration Other Tests
     needs:
-      - Code-Climate-Before-Build
+      - Run-Code-Style-Checks
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-other.yaml
 
   Run-Non-Spark-Integration-Tests:
     name: Run Non-Spark Integration Tests
     needs:
-      - Code-Climate-Before-Build
+      - Run-Code-Style-Checks
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-non-spark-integration.yaml
 
   Run-Non-Spark-Integration-Tests-With-Signal-Handling:
     name: Run Non-Spark Integration Tests With Signal Handling
     needs:
-      - Code-Climate-Before-Build
+      - Run-Code-Style-Checks
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-non-spark-integration-signal-handling.yaml
 
   Run-Unit-Tests:
     name: Run Unit Tests
-    needs: Code-Climate-Before-Build
+    needs: Run-Code-Style-Checks
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-unit.yaml
 
   Run-Unit-Tests-With-Signal-Handling:
     name: Run Unit Tests With Signal Handling
-    needs: Code-Climate-Before-Build
+    needs: Run-Code-Style-Checks
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-unit-signal-handling.yaml
-
-  Code-Climate-Report-Coverage:
-    name: Code Climate Report Coverage
-    needs:
-      - Run-Spark-Integration-Load-Transaction-FABS-and-FPDS-Tests
-      - Run-Spark-Integration-Load-Transactions-Lookup-Tests
-      - Run-Spark-Integration-Load-To-From-Delta-Tests
-      - Run-Spark-Integration-Other-Tests
-      - Run-Non-Spark-Integration-Tests
-      - Run-Non-Spark-Integration-Tests-With-Signal-Handling
-      - Run-Unit-Tests
-      - Run-Unit-Tests-With-Signal-Handling
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/code-climate-report-coverage.yaml
-    with:
-      source-branch: ${{ github.head_ref }}
-      source-commit-sha: ${{ github.event.pull_request.head.sha }}
-    secrets:
-      cc_test_reporter_id: ${{ secrets.CC_TEST_REPORTER_ID }}

setup.cfg

@@ -1,5 +1,5 @@
 [flake8]
 select=C,E,F,W,B,B950
-ignore=E501,W503,E203,F541
+ignore=E501,W503,E203,F541,E704
 exclude=.venv,build,usaspending_api.egg-info,usaspending_api/*/migrations/*
 max-line-length=120

usaspending_api/api_contracts/contracts/v2/search/spending_by_transaction.md

@@ -396,6 +396,7 @@ List of column names to request
 - `NAICS`
 - `PSC`
 - `Primary Place of Performance`
+- `recipient_id`
 - `Recipient Location`
 - `Recipient Name`
 - `Recipient UEI`
@@ -415,6 +416,7 @@ List of column names to request
 + `internal_id`: `68856340` (required, string, nullable)
 + `generated_internal_id`: `CONT_AWD_00013U_7090_KJ88_4735` (required, string, nullable)
 + `Mod`: `P00206` (required, string, nullable)
++ `recipient_id`: `1e5032cf-11df-a3bf-4240-6dda5f6d45ff-C` (optional, string, nullable)
 + `Recipient Name`: `LEIDOS INNOVATIONS CORPORATION` (required, string, nullable)
 + `Transaction Amount`: `40000000.00` (required, string, nullable)
 
@@ -433,6 +435,7 @@ List of column names to request
 + `Last Date to Order` (required, string, nullable)
 + `Loan Value` (required, string, nullable)
 + `Mod` (required, string, nullable)
++ `recipient_id` (optional, string, nullable)
 + `Recipient Name` (required, string, nullable)
 + `Subsidy Cost` (required, string, nullable)
-+ `Transaction Amount` (required, string, nullable)
++ `Transaction Amount` (required, string, nullable)

usaspending_api/awards/v2/lookups/elasticsearch_lookups.py

@@ -1,51 +1,90 @@
 """
-Look ups for elasticsearch fields to be displayed for the front end
+Lookups for elasticsearch fields to be displayed for the front end
 """
 
 from copy import deepcopy
+from dataclasses import dataclass
+from enum import Enum
 
 from usaspending_api.awards.v2.lookups.lookups import all_award_types_mappings
 
-TRANSACTIONS_LOOKUP = {
-    "Recipient Name": "recipient_name.keyword",
-    "Action Date": "action_date",
-    "Transaction Amount": "federal_action_obligation",
-    "Award Type": "type_description.keyword",
-    "Awarding Agency": "awarding_toptier_agency_name.keyword",
-    "Awarding Sub Agency": "awarding_subtier_agency_name.keyword",
-    "Funding Agency": "funding_toptier_agency_name",
-    "Funding Sub Agency": "funding_subtier_agency_name",
-    "Issued Date": "period_of_performance_start_date",
-    "Loan Value": "face_value_loan_guarantee",
-    "Subsidy Cost": "original_loan_subsidy_cost",
-    "Mod": "modification_number.keyword",
-    "Award ID": "display_award_id",
-    "awarding_agency_id": "awarding_agency_id",
-    "internal_id": "award_id",
-    "generated_internal_id": "generated_unique_award_id",
-    "Last Date to Order": "ordering_period_end_date",
-    "def_codes": "disaster_emergency_fund_codes",
-    "Transaction Description": "transaction_description.keyword",
-    "Action Type": "action_type",
-    "Recipient UEI": "recipient_uei.keyword",
-    "awarding_agency_slug": "awarding_toptier_agency_name.keyword",
-    "funding_agency_slug": "funding_toptier_agency_name.keyword",
-    "recipient_location_city_name": "recipient_location_city_name.keyword",
-    "recipient_location_state_code": "recipient_location_state_code",
-    "recipient_location_country_name": "recipient_location_country_name.keyword",
-    "recipient_location_address_line1": "recipient_location_address_line1.keyword",
-    "recipient_location_address_line2": "recipient_location_address_line2.keyword",
-    "recipient_location_address_line3": "recipient_location_address_line3.keyword",
-    "pop_city_name": "pop_city_name.keyword",
-    "pop_state_code": "pop_state_code",
-    "pop_country_name": "pop_country_name.keyword",
-    "naics_code": "naics_code.keyword",
-    "naics_description": "naics_description.keyword",
-    "product_or_service_code": "product_or_service_code.keyword",
-    "product_or_service_description": "product_or_service_description.keyword",
-    "cfda_number": "cfda_number.keyword",
-    "cfda_title": "cfda_title.keyword",
-}
+
+@dataclass
+class ElasticsearchField:
+    """
+    Represents a field that is searchable by an API endpoint and pairs it with the corresponding elasticsearch field.
+
+    Args:
+        field_name: The name of the field provided by the user when selecting fields and returned by the API
+        full_path: A complete path that may include additional field types such as ".keyword"
+        short_path: The full_path with any additional field types removed; may be 1:1 with full_path
+    """
+
+    field_name: str
+    full_path: str
+    short_path: str
+
+
+class TransactionField(str, Enum):
+    ACTION_DATE = ("Action Date", "action_date")
+    ACTION_TYPE = ("Action Type", "action_type")
+    AWARD_ID = ("Award ID", "display_award_id")
+    AWARD_TYPE = ("Award Type", "type_description.keyword")
+    AWARDING_AGENCY = ("Awarding Agency", "awarding_toptier_agency_name.keyword")
+    AWARDING_AGENCY_ID = ("awarding_agency_id", "awarding_agency_id")
+    AWARDING_AGENCY_SLUG = ("awarding_agency_slug", "awarding_toptier_agency_name.keyword")
+    AWARDING_SUB_AGENCY = ("Awarding Sub Agency", "awarding_subtier_agency_name.keyword")
+    CFDA_NUMBER = ("cfda_number", "cfda_number.keyword")
+    CFDA_TITLE = ("cfda_title", "cfda_title.keyword")
+    DEF_CODES = ("def_codes", "disaster_emergency_fund_codes")
+    FUNDING_AGENCY = ("Funding Agency", "funding_toptier_agency_name.keyword")
+    FUNDING_AGENCY_SLUG = ("funding_agency_slug", "funding_toptier_agency_name.keyword")
+    FUNDING_SUB_AGENCY = ("Funding Sub Agency", "funding_subtier_agency_name.keyword")
+    GENERATED_INTERNAL_ID = ("generated_internal_id", "generated_unique_award_id")
+    INTERNAL_ID = ("internal_id", "award_id")
+    ISSUED_DATE = ("Issued Date", "period_of_performance_start_date")
+    LAST_DATE_TO_ORDER = ("Last Date to Order", "ordering_period_end_date")
+    LOAN_VALUE = ("Loan Value", "face_value_loan_guarantee")
+    MOD = ("Mod", "modification_number.keyword")
+    NAICS_CODE = ("naics_code", "naics_code.keyword")
+    NAICS_DESCRIPTION = ("naics_description", "naics_description.keyword")
+    POP_CITY_NAME = ("pop_city_name", "pop_city_name.keyword")
+    POP_COUNTRY_NAME = ("pop_country_name", "pop_country_name.keyword")
+    POP_STATE_CODE = ("pop_state_code", "pop_state_code")
+    PSC_CODE = ("product_or_service_code", "product_or_service_code.keyword")
+    PSC_DESCRIPTION = ("product_or_service_description", "product_or_service_description.keyword")
+    RECIPIENT_ID = ("recipient_id", "recipient_agg_key")
+    RECIPIENT_LOCATION_ADDRESS_LINE_1 = ("recipient_location_address_line1", "recipient_location_address_line1.keyword")
+    RECIPIENT_LOCATION_ADDRESS_LINE_2 = ("recipient_location_address_line2", "recipient_location_address_line2.keyword")
+    RECIPIENT_LOCATION_ADDRESS_LINE_3 = ("recipient_location_address_line3", "recipient_location_address_line3.keyword")
+    RECIPIENT_LOCATION_CITY_NAME = ("recipient_location_city_name", "recipient_location_city_name.keyword")
+    RECIPIENT_LOCATION_COUNTRY_NAME = ("recipient_location_country_name", "recipient_location_country_name.keyword")
+    RECIPIENT_LOCATION_STATE_CODE = ("recipient_location_state_code", "recipient_location_state_code")
+    RECIPIENT_NAME = ("Recipient Name", "recipient_name.keyword")
+    RECIPIENT_UEI = ("Recipient UEI", "recipient_uei.keyword")
+    SUBSIDY_COST = ("Subsidy Cost", "original_loan_subsidy_cost")
+    TRANSACTION_AMOUNT = ("Transaction Amount", "federal_action_obligation")
+    TRANSACTION_DESCRIPTION = ("Transaction Description", "transaction_description.keyword")
+
+    def __new__(cls, field_name: str, full_path: str) -> "str":
+        obj = str.__new__(cls, field_name)
+        obj._value_ = field_name
+        short_path = full_path.split(".")[0]
+        obj._es_field = ElasticsearchField(field_name, full_path, short_path)
+        return obj
+
+    @property
+    def field_name(self) -> str:
+        return self._es_field.field_name
+
+    @property
+    def full_path(self) -> str:
+        return self._es_field.full_path
+
+    @property
+    def short_path(self) -> str:
+        return self._es_field.short_path
+
 
 base_mapping = {
     "Award ID": "display_award_id",
@@ -145,8 +184,6 @@
     },
 }
 
-TRANSACTIONS_SOURCE_LOOKUP = {key: value.replace(".keyword", "") for key, value in TRANSACTIONS_LOOKUP.items()}
-
 CONTRACT_SOURCE_LOOKUP = {key: value.replace(".keyword", "") for key, value in contracts_mapping.items()}
 IDV_SOURCE_LOOKUP = {key: value.replace(".keyword", "") for key, value in idv_mapping.items()}
 NON_LOAN_ASST_SOURCE_LOOKUP = {key: value.replace(".keyword", "") for key, value in non_loan_assist_mapping.items()}

usaspending_api/common/helpers/download_csv_strategies.py

@@ -115,7 +115,7 @@ def download_to_csv(
             raise e
         finally:
             Path(temp_file_path).unlink()
-        return CSVDownloadMetadata([destination_path], row_count)
+        return CSVDownloadMetadata([str(destination_path)], row_count)
 
 
 class SparkToCSVStrategy(AbstractToCSVStrategy):