Skip to content

Commit 63fcaf7

Browse files
authored
Merge pull request #4446 from fedspendingtransparency/staging
Sprint 206 Prod Deploy
2 parents a587eec + 81ec59e commit 63fcaf7

33 files changed

+1340
-986
lines changed

.github/workflows/code-climate-before-build.yaml

Lines changed: 0 additions & 33 deletions
This file was deleted.

.github/workflows/code-climate-report-coverage.yaml

Lines changed: 0 additions & 52 deletions
This file was deleted.

.github/workflows/pull-request-checks.yaml

Lines changed: 8 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -44,95 +44,64 @@ jobs:
4444
needs.Cancel-If-Draft-Pull-Request.result == 'skipped'
4545
uses: ./.github/workflows/build-broker-docker-image-for-test.yaml
4646

47-
# Prepare Code Climate to receive coverage report
48-
Code-Climate-Before-Build:
49-
name: Code Climate Before Build
50-
needs: Run-Code-Style-Checks
51-
if: ${{ !failure() && !cancelled() }}
52-
uses: ./.github/workflows/code-climate-before-build.yaml
53-
with:
54-
source-branch: ${{ github.head_ref }}
55-
source-commit-sha: ${{ github.event.pull_request.head.sha }}
56-
secrets:
57-
cc_test_reporter_id: ${{ secrets.CC_TEST_REPORTER_ID }}
58-
5947
# The test suite is broken into multiple groups based on a combination of what is under test
6048
# and the time that it takes to run tests for that group.
6149
Run-Spark-Integration-Load-Transaction-FABS-and-FPDS-Tests:
6250
name: Run Spark Integration Load Transactions FABS and FPDS Tests
6351
needs:
64-
- Code-Climate-Before-Build
52+
- Run-Code-Style-Checks
6553
- Build-Broker-Docker-Image
6654
if: ${{ !failure() && !cancelled() }}
6755
uses: ./.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
6856

6957
Run-Spark-Integration-Load-Transactions-Lookup-Tests:
7058
name: Run Spark Integration Load Transactions Lookup Tests
7159
needs:
72-
- Code-Climate-Before-Build
60+
- Run-Code-Style-Checks
7361
- Build-Broker-Docker-Image
7462
if: ${{ !failure() && !cancelled() }}
7563
uses: ./.github/workflows/test-spark-integration-load-transactions-lookup.yaml
7664

7765
Run-Spark-Integration-Load-To-From-Delta-Tests:
7866
name: Run Spark Integration Load To From Delta Tests
7967
needs:
80-
- Code-Climate-Before-Build
68+
- Run-Code-Style-Checks
8169
- Build-Broker-Docker-Image
8270
if: ${{ !failure() && !cancelled() }}
8371
uses: ./.github/workflows/test-spark-integration-load-to-from-delta.yaml
8472

8573
Run-Spark-Integration-Other-Tests:
8674
name: Run Spark Integration Other Tests
8775
needs:
88-
- Code-Climate-Before-Build
76+
- Run-Code-Style-Checks
8977
- Build-Broker-Docker-Image
9078
if: ${{ !failure() && !cancelled() }}
9179
uses: ./.github/workflows/test-spark-integration-other.yaml
9280

9381
Run-Non-Spark-Integration-Tests:
9482
name: Run Non-Spark Integration Tests
9583
needs:
96-
- Code-Climate-Before-Build
84+
- Run-Code-Style-Checks
9785
- Build-Broker-Docker-Image
9886
if: ${{ !failure() && !cancelled() }}
9987
uses: ./.github/workflows/test-non-spark-integration.yaml
10088

10189
Run-Non-Spark-Integration-Tests-With-Signal-Handling:
10290
name: Run Non-Spark Integration Tests With Signal Handling
10391
needs:
104-
- Code-Climate-Before-Build
92+
- Run-Code-Style-Checks
10593
- Build-Broker-Docker-Image
10694
if: ${{ !failure() && !cancelled() }}
10795
uses: ./.github/workflows/test-non-spark-integration-signal-handling.yaml
10896

10997
Run-Unit-Tests:
11098
name: Run Unit Tests
111-
needs: Code-Climate-Before-Build
99+
needs: Run-Code-Style-Checks
112100
if: ${{ !failure() && !cancelled() }}
113101
uses: ./.github/workflows/test-unit.yaml
114102

115103
Run-Unit-Tests-With-Signal-Handling:
116104
name: Run Unit Tests With Signal Handling
117-
needs: Code-Climate-Before-Build
105+
needs: Run-Code-Style-Checks
118106
if: ${{ !failure() && !cancelled() }}
119107
uses: ./.github/workflows/test-unit-signal-handling.yaml
120-
121-
Code-Climate-Report-Coverage:
122-
name: Code Climate Report Coverage
123-
needs:
124-
- Run-Spark-Integration-Load-Transaction-FABS-and-FPDS-Tests
125-
- Run-Spark-Integration-Load-Transactions-Lookup-Tests
126-
- Run-Spark-Integration-Load-To-From-Delta-Tests
127-
- Run-Spark-Integration-Other-Tests
128-
- Run-Non-Spark-Integration-Tests
129-
- Run-Non-Spark-Integration-Tests-With-Signal-Handling
130-
- Run-Unit-Tests
131-
- Run-Unit-Tests-With-Signal-Handling
132-
if: ${{ !failure() && !cancelled() }}
133-
uses: ./.github/workflows/code-climate-report-coverage.yaml
134-
with:
135-
source-branch: ${{ github.head_ref }}
136-
source-commit-sha: ${{ github.event.pull_request.head.sha }}
137-
secrets:
138-
cc_test_reporter_id: ${{ secrets.CC_TEST_REPORTER_ID }}

setup.cfg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[flake8]
22
select=C,E,F,W,B,B950
3-
ignore=E501,W503,E203,F541
3+
ignore=E501,W503,E203,F541,E704
44
exclude=.venv,build,usaspending_api.egg-info,usaspending_api/*/migrations/*
55
max-line-length=120

usaspending_api/api_contracts/contracts/v2/search/spending_by_transaction.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -396,6 +396,7 @@ List of column names to request
396396
- `NAICS`
397397
- `PSC`
398398
- `Primary Place of Performance`
399+
- `recipient_id`
399400
- `Recipient Location`
400401
- `Recipient Name`
401402
- `Recipient UEI`
@@ -415,6 +416,7 @@ List of column names to request
415416
+ `internal_id`: `68856340` (required, string, nullable)
416417
+ `generated_internal_id`: `CONT_AWD_00013U_7090_KJ88_4735` (required, string, nullable)
417418
+ `Mod`: `P00206` (required, string, nullable)
419+
+ `recipient_id`: `1e5032cf-11df-a3bf-4240-6dda5f6d45ff-C` (optional, string, nullable)
418420
+ `Recipient Name`: `LEIDOS INNOVATIONS CORPORATION` (required, string, nullable)
419421
+ `Transaction Amount`: `40000000.00` (required, string, nullable)
420422

@@ -433,6 +435,7 @@ List of column names to request
433435
+ `Last Date to Order` (required, string, nullable)
434436
+ `Loan Value` (required, string, nullable)
435437
+ `Mod` (required, string, nullable)
438+
+ `recipient_id` (optional, string, nullable)
436439
+ `Recipient Name` (required, string, nullable)
437440
+ `Subsidy Cost` (required, string, nullable)
438-
+ `Transaction Amount` (required, string, nullable)
441+
+ `Transaction Amount` (required, string, nullable)

usaspending_api/awards/v2/lookups/elasticsearch_lookups.py

Lines changed: 80 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,90 @@
11
"""
2-
Look ups for elasticsearch fields to be displayed for the front end
2+
Lookups for elasticsearch fields to be displayed for the front end
33
"""
44

55
from copy import deepcopy
6+
from dataclasses import dataclass
7+
from enum import Enum
68

79
from usaspending_api.awards.v2.lookups.lookups import all_award_types_mappings
810

9-
TRANSACTIONS_LOOKUP = {
10-
"Recipient Name": "recipient_name.keyword",
11-
"Action Date": "action_date",
12-
"Transaction Amount": "federal_action_obligation",
13-
"Award Type": "type_description.keyword",
14-
"Awarding Agency": "awarding_toptier_agency_name.keyword",
15-
"Awarding Sub Agency": "awarding_subtier_agency_name.keyword",
16-
"Funding Agency": "funding_toptier_agency_name",
17-
"Funding Sub Agency": "funding_subtier_agency_name",
18-
"Issued Date": "period_of_performance_start_date",
19-
"Loan Value": "face_value_loan_guarantee",
20-
"Subsidy Cost": "original_loan_subsidy_cost",
21-
"Mod": "modification_number.keyword",
22-
"Award ID": "display_award_id",
23-
"awarding_agency_id": "awarding_agency_id",
24-
"internal_id": "award_id",
25-
"generated_internal_id": "generated_unique_award_id",
26-
"Last Date to Order": "ordering_period_end_date",
27-
"def_codes": "disaster_emergency_fund_codes",
28-
"Transaction Description": "transaction_description.keyword",
29-
"Action Type": "action_type",
30-
"Recipient UEI": "recipient_uei.keyword",
31-
"awarding_agency_slug": "awarding_toptier_agency_name.keyword",
32-
"funding_agency_slug": "funding_toptier_agency_name.keyword",
33-
"recipient_location_city_name": "recipient_location_city_name.keyword",
34-
"recipient_location_state_code": "recipient_location_state_code",
35-
"recipient_location_country_name": "recipient_location_country_name.keyword",
36-
"recipient_location_address_line1": "recipient_location_address_line1.keyword",
37-
"recipient_location_address_line2": "recipient_location_address_line2.keyword",
38-
"recipient_location_address_line3": "recipient_location_address_line3.keyword",
39-
"pop_city_name": "pop_city_name.keyword",
40-
"pop_state_code": "pop_state_code",
41-
"pop_country_name": "pop_country_name.keyword",
42-
"naics_code": "naics_code.keyword",
43-
"naics_description": "naics_description.keyword",
44-
"product_or_service_code": "product_or_service_code.keyword",
45-
"product_or_service_description": "product_or_service_description.keyword",
46-
"cfda_number": "cfda_number.keyword",
47-
"cfda_title": "cfda_title.keyword",
48-
}
11+
12+
@dataclass
13+
class ElasticsearchField:
14+
"""
15+
Represents a field that is searchable by an API endpoint and pairs it with the corresponding elasticsearch field.
16+
17+
Args:
18+
field_name: The name of the field provided by the user when selecting fields and returned by the API
19+
full_path: A complete path that may include additional field types such as ".keyword"
20+
short_path: The full_path with any additional field types removed; may be 1:1 with full_path
21+
"""
22+
23+
field_name: str
24+
full_path: str
25+
short_path: str
26+
27+
28+
class TransactionField(str, Enum):
29+
ACTION_DATE = ("Action Date", "action_date")
30+
ACTION_TYPE = ("Action Type", "action_type")
31+
AWARD_ID = ("Award ID", "display_award_id")
32+
AWARD_TYPE = ("Award Type", "type_description.keyword")
33+
AWARDING_AGENCY = ("Awarding Agency", "awarding_toptier_agency_name.keyword")
34+
AWARDING_AGENCY_ID = ("awarding_agency_id", "awarding_agency_id")
35+
AWARDING_AGENCY_SLUG = ("awarding_agency_slug", "awarding_toptier_agency_name.keyword")
36+
AWARDING_SUB_AGENCY = ("Awarding Sub Agency", "awarding_subtier_agency_name.keyword")
37+
CFDA_NUMBER = ("cfda_number", "cfda_number.keyword")
38+
CFDA_TITLE = ("cfda_title", "cfda_title.keyword")
39+
DEF_CODES = ("def_codes", "disaster_emergency_fund_codes")
40+
FUNDING_AGENCY = ("Funding Agency", "funding_toptier_agency_name.keyword")
41+
FUNDING_AGENCY_SLUG = ("funding_agency_slug", "funding_toptier_agency_name.keyword")
42+
FUNDING_SUB_AGENCY = ("Funding Sub Agency", "funding_subtier_agency_name.keyword")
43+
GENERATED_INTERNAL_ID = ("generated_internal_id", "generated_unique_award_id")
44+
INTERNAL_ID = ("internal_id", "award_id")
45+
ISSUED_DATE = ("Issued Date", "period_of_performance_start_date")
46+
LAST_DATE_TO_ORDER = ("Last Date to Order", "ordering_period_end_date")
47+
LOAN_VALUE = ("Loan Value", "face_value_loan_guarantee")
48+
MOD = ("Mod", "modification_number.keyword")
49+
NAICS_CODE = ("naics_code", "naics_code.keyword")
50+
NAICS_DESCRIPTION = ("naics_description", "naics_description.keyword")
51+
POP_CITY_NAME = ("pop_city_name", "pop_city_name.keyword")
52+
POP_COUNTRY_NAME = ("pop_country_name", "pop_country_name.keyword")
53+
POP_STATE_CODE = ("pop_state_code", "pop_state_code")
54+
PSC_CODE = ("product_or_service_code", "product_or_service_code.keyword")
55+
PSC_DESCRIPTION = ("product_or_service_description", "product_or_service_description.keyword")
56+
RECIPIENT_ID = ("recipient_id", "recipient_agg_key")
57+
RECIPIENT_LOCATION_ADDRESS_LINE_1 = ("recipient_location_address_line1", "recipient_location_address_line1.keyword")
58+
RECIPIENT_LOCATION_ADDRESS_LINE_2 = ("recipient_location_address_line2", "recipient_location_address_line2.keyword")
59+
RECIPIENT_LOCATION_ADDRESS_LINE_3 = ("recipient_location_address_line3", "recipient_location_address_line3.keyword")
60+
RECIPIENT_LOCATION_CITY_NAME = ("recipient_location_city_name", "recipient_location_city_name.keyword")
61+
RECIPIENT_LOCATION_COUNTRY_NAME = ("recipient_location_country_name", "recipient_location_country_name.keyword")
62+
RECIPIENT_LOCATION_STATE_CODE = ("recipient_location_state_code", "recipient_location_state_code")
63+
RECIPIENT_NAME = ("Recipient Name", "recipient_name.keyword")
64+
RECIPIENT_UEI = ("Recipient UEI", "recipient_uei.keyword")
65+
SUBSIDY_COST = ("Subsidy Cost", "original_loan_subsidy_cost")
66+
TRANSACTION_AMOUNT = ("Transaction Amount", "federal_action_obligation")
67+
TRANSACTION_DESCRIPTION = ("Transaction Description", "transaction_description.keyword")
68+
69+
def __new__(cls, field_name: str, full_path: str) -> "str":
70+
obj = str.__new__(cls, field_name)
71+
obj._value_ = field_name
72+
short_path = full_path.split(".")[0]
73+
obj._es_field = ElasticsearchField(field_name, full_path, short_path)
74+
return obj
75+
76+
@property
77+
def field_name(self) -> str:
78+
return self._es_field.field_name
79+
80+
@property
81+
def full_path(self) -> str:
82+
return self._es_field.full_path
83+
84+
@property
85+
def short_path(self) -> str:
86+
return self._es_field.short_path
87+
4988

5089
base_mapping = {
5190
"Award ID": "display_award_id",
@@ -145,8 +184,6 @@
145184
},
146185
}
147186

148-
TRANSACTIONS_SOURCE_LOOKUP = {key: value.replace(".keyword", "") for key, value in TRANSACTIONS_LOOKUP.items()}
149-
150187
CONTRACT_SOURCE_LOOKUP = {key: value.replace(".keyword", "") for key, value in contracts_mapping.items()}
151188
IDV_SOURCE_LOOKUP = {key: value.replace(".keyword", "") for key, value in idv_mapping.items()}
152189
NON_LOAN_ASST_SOURCE_LOOKUP = {key: value.replace(".keyword", "") for key, value in non_loan_assist_mapping.items()}

usaspending_api/common/helpers/download_csv_strategies.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ def download_to_csv(
115115
raise e
116116
finally:
117117
Path(temp_file_path).unlink()
118-
return CSVDownloadMetadata([destination_path], row_count)
118+
return CSVDownloadMetadata([str(destination_path)], row_count)
119119

120120

121121
class SparkToCSVStrategy(AbstractToCSVStrategy):

0 commit comments

Comments
 (0)