Merge pull request #2947 from fedspendingtransparency/qat

Sprint 125 Production Deploy

Author: tony-sappe

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
     -   id: debug-statements
     -   id: flake8
 -   repo: https://github.com/psf/black
-    rev: 19.10b0
+    rev: 20.8b1
     hooks:
     -   id: black
         language_version: python3.7

requirements/requirements-app.txt

@@ -1,62 +1,39 @@
-appdirs==1.4.3
-asyncpg==0.18.3
+asyncpg==0.21.*
 atomicwrites==1.3.0
-attrs==19.1.0
-boto3==1.9.224
-botocore==1.12.224
-certifi==2019.6.16
-chardet==3.0.4
+attrs==20.*
+boto3==1.16.*
+certifi==2020.12.5
 dataclasses-json==0.5.*
-django_cte==1.1.*
 ddtrace==0.37.1
 dj-database-url==0.5.0
 django-cors-headers==2.5.3
-django-debug-toolbar==2.1
+django-debug-toolbar==3.2
 django-extensions==2.2.5
 django-filter==0.14.0
 django-partial-index==0.6.0
-django-queryset-csv==1.0.1
 django-simple-history==2.7.3
-django-spaghetti-and-meatballs==0.2.2
+django-spaghetti-and-meatballs==0.4.0
 Django==2.2.*
-djangorestframework==3.11.0
+django_cte==1.1.*
+djangorestframework==3.12.2
 docutils==0.15.2
-drf-extensions==0.3.1
-drf-tracking==1.4.0
-elasticsearch==7.1.0
+drf-api-tracking==1.7.8
+drf-extensions==0.6.*
 elasticsearch-dsl==7.1.0
-entrypoints==0.3
+elasticsearch==7.1.0
 et-xmlfile==1.0.1
-filechunkio==1.8
 filelock==3.0.12
-fiscalyear==0.1.0
-idna==2.8
-importlib-metadata==0.20
-jdcal==1.4.1
-jmespath==0.9.4
-jsonpickle==1.2
+fiscalyear==0.2.0
 Markdown<3.0
-more-itertools==7.2.0
-numpy==1.17.2
+numpy==1.19.*
 openpyxl==2.4.7
-pandas==0.25.1
+pandas==1.1.*
 pip==20.*
-pluggy==0.12.0
 psutil==5.6.*
-psycopg2-binary==2.7.5
+psycopg2-binary==2.8.*
 py-gfm==0.1.4
-py==1.8.0
-pycodestyle==2.5.0
-pyflakes==2.1.1
-python-dateutil==2.8.0
 python-json-logger==0.1.9
-pytz==2019.2
-requests==2.22.0
+requests==2.25.*
 retrying==1.3.3
-s3transfer==0.2.1
-six==1.12.0
-sqlparse==0.3.0
-unicodecsv==0.14.1
-urllib3==1.25.*
-xlrd==1.0.0
-zipp==0.6.0
+urllib3==1.26.*
+xlrd3==1.0.0

requirements/requirements-dev.txt

@@ -1,14 +1,10 @@
-black==19.10b0
-coverage==4.5.4
-docker==4.1.0
+black==20.8b1
+docker==4.4.1
 dredd-hooks==0.2.0
-flake8==3.7.8
-mccabe==0.6.1
+flake8==3.8.4
 mock==3.0.5
-model-mommy==1.6.0
+model-mommy==2.0.0
 pre-commit==1.20.0
-pycodestyle==2.5.0
-pyflakes==2.1.1
-pytest-cov==2.8.1
-pytest-django==3.5.1
-pytest==4.3.1
+pytest-cov==2.10.1
+pytest-django==3.10.0
+pytest==6.2.1

requirements/requirements-server.txt

@@ -1,7 +1,7 @@
-django-elasticache==1.0.2
-django-redis==4.8.0
+django-elasticache==1.0.*
+django-redis==4.12.*
 packaging==16.8
 pylibmc==1.5.2
 pyparsing==2.2.0
 supervisor==4.0.2
-uwsgi==2.0.17.1
+uwsgi==2.0.*

setup.cfg

@@ -4,7 +4,7 @@ addopts=--cov=usaspending_api
 
 [flake8]
 select=C,E,F,W,B,B950
-ignore=E501,W503,E203
+ignore=E501,W503,E203,F541
 exclude=.venv,usaspending_api/*/migrations/*
 max-line-length=120
 

usaspending_api/accounts/tests/integration/test_federal_account.py

@@ -18,8 +18,8 @@
 @pytest.mark.django_db
 def test_federal_account_spending_by_category_unique_program_activity_names(client):
     """
-        Test the spending by category endpoint for the federal account profile page to ensure a unique set of
-        program activity names are returned
+    Test the spending by category endpoint for the federal account profile page to ensure a unique set of
+    program activity names are returned
     """
 
     models_to_mock = [

usaspending_api/awards/tests/test_transactions.py

@@ -26,7 +26,7 @@ def test_transaction_endpoint_v2_award_fk(client):
     )
     mommy.make("awards.TransactionNormalized", description="this should match", _fill_optional=True, award=awd)
 
-    resp = client.post("/api/v2/transactions/", {"award_id": "10"})
+    resp = client.post("/api/v2/transactions/", {"award_id": 10})
     assert resp.status_code == status.HTTP_200_OK
     assert json.loads(resp.content.decode("utf-8"))["results"][0]["description"] == "this should match"
 

usaspending_api/awards/v2/data_layer/orm_utils.py

@@ -26,21 +26,21 @@ def delete_keys_from_dict(dictionary):
 
 def split_mapper_into_qs(mapper):
     """
-        Django ORM has trouble using .annotate() when the destination field conflicts with an
-        existing model field, even if it's the same source field (no renaming occurring)
+    Django ORM has trouble using .annotate() when the destination field conflicts with an
+    existing model field, even if it's the same source field (no renaming occurring)
 
-        Assuming there is a dictionary with model field names as keys and the target field as the value,
-        Split that into two objects:
-            values_list: a list of fields which you wish to retrieve without renaming
-                aka when `key` == `value`
-            annotate_dict: a dictionary/OrderedDict of target and source fields to rename
+    Assuming there is a dictionary with model field names as keys and the target field as the value,
+    Split that into two objects:
+        values_list: a list of fields which you wish to retrieve without renaming
+            aka when `key` == `value`
+        annotate_dict: a dictionary/OrderedDict of target and source fields to rename
 
-        parameters
-            - mapper:  dictionary/OrderedDict
+    parameters
+        - mapper:  dictionary/OrderedDict
 
-        return:
-            - values_list: list
-            -annotate_dict:  OrderedDict
+    return:
+        - values_list: list
+        -annotate_dict:  OrderedDict
     """
     values_list = [k for k, v in mapper.items() if k == v]
     annotate_dict = OrderedDict([(v, F(k)) for k, v in mapper.items() if k != v])

usaspending_api/awards/v2/filters/filter_helpers.py

@@ -22,12 +22,12 @@
 
 def merge_date_ranges(date_range_list):
     """
-        Given a list of date ranges (using the defined namedtuple "Range"), combine overlapping date ranges
-        While adjacent fiscal years do not overlap the desired behavior is to combine them
-            FY2010 ends on 2010-09-30, FY2011 start on 2010-10-01.
-        To address this, when comparing ranges 1 day is removed from the start date and 1 day is added to the end date
-        Then the overlapping ranges must be > 1 instead of > 0
-        Inspired by Raymond Hettinger [https://stackoverflow.com/a/9044111]
+    Given a list of date ranges (using the defined namedtuple "Range"), combine overlapping date ranges
+    While adjacent fiscal years do not overlap the desired behavior is to combine them
+        FY2010 ends on 2010-09-30, FY2011 start on 2010-10-01.
+    To address this, when comparing ranges 1 day is removed from the start date and 1 day is added to the end date
+    Then the overlapping ranges must be > 1 instead of > 0
+    Inspired by Raymond Hettinger [https://stackoverflow.com/a/9044111]
     """
     ordered_list = sorted([sorted(t) for t in date_range_list])
     saved_range = Range(start=ordered_list[0][0], end=ordered_list[0][1])
@@ -142,7 +142,7 @@ def total_obligation_queryset(amount_obj, model, filters):
 
 def can_use_month_aggregation(time_period):
     """
-        time_period is the list of action_date ranges from API
+    time_period is the list of action_date ranges from API
     """
     try:
         for v in time_period:
@@ -173,7 +173,7 @@ def can_use_total_obligation_enum(amount_obj):
 
 def only_action_date_type(time_period):
     """
-        if a date_type is last_modified_date, don't use the matview this applies to
+    if a date_type is last_modified_date, don't use the matview this applies to
     """
     try:
         for v in time_period:

…api/awards/v2/filters/matview_filters.py …spending_api/awards/v2/filters/search.pyusaspending_api/awards/v2/filters/matview_filters.py renamed to usaspending_api/awards/v2/filters/search.py usaspending_api/awards/v2/filters/matview_filters.py renamed to usaspending_api/awards/v2/filters/search.py

@@ -6,7 +6,7 @@
 from usaspending_api.accounts.views.federal_accounts_v2 import filter_on
 from usaspending_api.awards.models import FinancialAccountsByAwards
 from usaspending_api.search.filters.postgres.psc import PSCCodes
-from usaspending_api.search.models import AwardSearchView, UniversalTransactionView
+from usaspending_api.search.models import AwardSearchView, TransactionSearch
 from usaspending_api.awards.v2.filters.filter_helpers import combine_date_range_queryset, total_obligation_queryset
 from usaspending_api.awards.v2.filters.location_filter_geocode import geocode_filter_locations
 from usaspending_api.common.exceptions import InvalidParameterException, NotImplementedException
@@ -26,8 +26,8 @@ def universal_award_matview_filter(filters):
     return matview_search_filter(filters, AwardSearchView, for_downloads=True)
 
 
-def universal_transaction_matview_filter(filters):
-    return matview_search_filter(filters, UniversalTransactionView, for_downloads=True)
+def transaction_search_filter(filters):
+    return matview_search_filter(filters, TransactionSearch, for_downloads=True)
 
 
 def matview_search_filter_determine_award_matview_model(filters):