Merge pull request #2982 from fedspendingtransparency/qat

Merge QAT -> Staging

Author: tony-sappe

usaspending_api/etl/management/commands/load_submission.py

@@ -167,19 +167,20 @@ def get_broker_submission(self):
                             )
                         ) AS history
                     from
-                        (select distinct
-                            submission_id,
-                            publish_history_id,
-                            certify_history_id
-                        from published_files_history
-                        where submission_id = %s) as distinct_pairings
-                    left join
-                        publish_history as ph
-                            on distinct_pairings.publish_history_id = ph.publish_history_id
-                    left join
-                        certify_history as ch
-                            on distinct_pairings.certify_history_id = ch.certify_history_id
-                    group by distinct_pairings.submission_id)
+                        (
+                            select distinct
+                                submission_id,
+                                publish_history_id,
+                                certify_history_id
+                            from published_files_history
+                            where submission_id = %s
+                        ) as distinct_pairings
+                    left outer join
+                        publish_history as ph using (publish_history_id)
+                    left outer join
+                        certify_history as ch using (certify_history_id)
+                    group by distinct_pairings.submission_id
+                )
                 select
                     s.submission_id,
                     (
@@ -203,9 +204,8 @@ def get_broker_submission(self):
                     pch.history
                 from
                     submission as s
-                join
-                    publish_certify_history as pch
-                        on pch.submission_id = s.submission_id
+                inner join
+                    publish_certify_history as pch using (submission_id)
             """,
             [self.submission_id],
         )

usaspending_api/reporting/tests/integration/test_agencies_submission_history.py

@@ -1,9 +1,11 @@
+import json
 import pytest
+
 from model_mommy import mommy
 from rest_framework import status
 
 
-url = "/api/v2/reporting/agencies/{agency_data}/submission_history/"
+url = "/api/v2/reporting/agencies/{agency}/{fy}/{period}/submission_history/"
 
 
 @pytest.fixture
@@ -17,6 +19,9 @@ def setup_test_data(db):
         published_date="2019-07-16 16:09:52.125837-04",
         certified_date="2019-07-16 16:09:52.125837-04",
         toptier_code="020",
+        history=json.loads(
+            '[{"certified_date": "2019-07-16T16:09:52.125837Z", "published_date": "2019-07-16T16:09:52.125837Z"}]'
+        ),
     )
     mommy.make(
         "submissions.SubmissionAttributes",
@@ -26,6 +31,10 @@ def setup_test_data(db):
         published_date="2020-08-17 18:37:21.605023-04",
         certified_date="2020-08-17 18:37:21.605023-04",
         toptier_code="020",
+        history=json.loads(
+            '[{"certified_date": "2020-08-17T18:37:21.605023Z", "published_date": "2020-08-17T18:37:21.605023Z"},'
+            + '{"certified_date": "2017-08-14T14:17:00.729315Z", "published_date": "2017-08-14T14:17:00.729315Z"}]'
+        ),
     )
     mommy.make(
         "submissions.SubmissionAttributes",
@@ -34,42 +43,90 @@ def setup_test_data(db):
         reporting_fiscal_period=7,
         published_date="2017-08-14 14:17:00.729315-04",
         certified_date="2017-08-14 14:17:00.729315-04",
-        toptier_code="020",
+        toptier_code="075",
+        history=json.loads(
+            '[{"certified_date": "2017-08-14T14:17:00.729315Z", "published_date": "2017-08-14T14:17:00.729315Z"}]'
+        ),
     )
     mommy.make(
         "submissions.SubmissionAttributes",
-        submission_id=4,
-        reporting_fiscal_year=2019,
-        reporting_fiscal_period=7,
-        published_date="2017-08-14 14:17:00.729315-04",
-        certified_date="2017-08-14 14:17:00.729315-04",
-        toptier_code="075",
+        submission_id=10,
+        reporting_fiscal_year=2020,
+        reporting_fiscal_period=12,
+        published_date="2021-02-16 18:20:00.729315-04",
+        certified_date="2021-02-16 18:17:00.729315-04",
+        toptier_code="222",
+        history=json.loads(
+            '[{"certified_date": null, "published_date": "2020-01-17T18:37:21.605023Z"},'
+            + '{"certified_date": null, "published_date": "2020-01-14T14:17:00.729315Z"},'
+            + '{"certified_date": "2021-02-14T14:17:00.729315Z", "published_date": "2021-02-14T14:16:00.729315Z"},'
+            + '{"certified_date": "2021-02-16T14:17:00.729315Z", "published_date": "2021-02-16T14:16:00.729315Z"}]'
+        ),
     )
 
 
 def test_basic_success(client, setup_test_data):
-    resp = client.get(url.format(agency_data="020/2019/6"))
+    resp = client.get(url.format(agency="020", fy=2019, period=6))
     assert resp.status_code == status.HTTP_200_OK
     response = resp.json()
     assert len(response["results"]) == 1
     assert response["results"] == [
-        {"publication_date": "2019-07-16T20:09:52.125837Z", "certification_date": "2019-07-16T20:09:52.125837Z"}
+        {"publication_date": "2019-07-16T16:09:52.125837Z", "certification_date": "2019-07-16T16:09:52.125837Z"}
+    ]
+
+    resp = client.get(url.format(agency="075", fy=2019, period=7))
+    assert resp.status_code == status.HTTP_200_OK
+    response = resp.json()
+    assert len(response["results"]) == 1
+    assert response["results"] == [
+        {"publication_date": "2017-08-14T14:17:00.729315Z", "certification_date": "2017-08-14T14:17:00.729315Z"}
     ]
 
 
 def test_multiple_submissions(client, setup_test_data):
-    resp = client.get(url.format(agency_data="020/2019/7"))
+    resp = client.get(url.format(agency="020", fy=2019, period=7))
+    assert resp.status_code == status.HTTP_200_OK
+    response = resp.json()
+    assert len(response["results"]) == 2
+    assert response["results"] == [
+        {"publication_date": "2020-08-17T18:37:21.605023Z", "certification_date": "2020-08-17T18:37:21.605023Z"},
+        {"publication_date": "2017-08-14T14:17:00.729315Z", "certification_date": "2017-08-14T14:17:00.729315Z"},
+    ]
+
+    resp = client.get(url.format(agency="020", fy=2019, period=7) + "?sort=publication_date&order=asc")
     assert resp.status_code == status.HTTP_200_OK
     response = resp.json()
     assert len(response["results"]) == 2
     assert response["results"] == [
-        {"publication_date": "2020-08-17T22:37:21.605023Z", "certification_date": "2020-08-17T22:37:21.605023Z"},
-        {"publication_date": "2017-08-14T18:17:00.729315Z", "certification_date": "2017-08-14T18:17:00.729315Z"},
+        {"publication_date": "2017-08-14T14:17:00.729315Z", "certification_date": "2017-08-14T14:17:00.729315Z"},
+        {"publication_date": "2020-08-17T18:37:21.605023Z", "certification_date": "2020-08-17T18:37:21.605023Z"},
     ]
-    resp = client.get(url.format(agency_data="075/2019/7"))
+
+
+def test_no_data(client, setup_test_data):
+    resp = client.get(url.format(agency="222", fy=2021, period=12))
+    assert resp.status_code == status.HTTP_204_NO_CONTENT
+
+
+def test_certification_nulls(client, setup_test_data):
+    resp = client.get(url.format(agency="222", fy=2020, period=12) + "?sort=certification_date&order=desc")
     assert resp.status_code == status.HTTP_200_OK
     response = resp.json()
-    assert len(response["results"]) == 1
+    assert len(response["results"]) == 4
+    assert response["results"] == [
+        {"certification_date": "2021-02-16T14:17:00.729315Z", "publication_date": "2021-02-16T14:16:00.729315Z"},
+        {"certification_date": "2021-02-14T14:17:00.729315Z", "publication_date": "2021-02-14T14:16:00.729315Z"},
+        {"certification_date": None, "publication_date": "2020-01-17T18:37:21.605023Z"},
+        {"certification_date": None, "publication_date": "2020-01-14T14:17:00.729315Z"},
+    ]
+
+    resp = client.get(url.format(agency="222", fy=2020, period=12) + "?sort=certification_date&order=asc")
+    assert resp.status_code == status.HTTP_200_OK
+    response = resp.json()
+    assert len(response["results"]) == 4
     assert response["results"] == [
-        {"publication_date": "2017-08-14T18:17:00.729315Z", "certification_date": "2017-08-14T18:17:00.729315Z"}
+        {"certification_date": None, "publication_date": "2020-01-14T14:17:00.729315Z"},
+        {"certification_date": None, "publication_date": "2020-01-17T18:37:21.605023Z"},
+        {"certification_date": "2021-02-14T14:17:00.729315Z", "publication_date": "2021-02-14T14:16:00.729315Z"},
+        {"certification_date": "2021-02-16T14:17:00.729315Z", "publication_date": "2021-02-16T14:16:00.729315Z"},
     ]

usaspending_api/reporting/v2/views/submission_history.py

@@ -1,8 +1,8 @@
-from django.db.models import F
 from rest_framework.response import Response
 
 from usaspending_api.agency.v2.views.agency_base import AgencyBase, PaginationMixin
-
+from usaspending_api.common.exceptions import NoDataFoundException
+from usaspending_api.common.helpers.date_helper import now
 from usaspending_api.common.helpers.generic_helper import get_pagination_metadata
 from usaspending_api.submissions.models import SubmissionAttributes
 
@@ -17,14 +17,32 @@ def get(self, request, toptier_code, fiscal_year, fiscal_period):
         self.sortable_columns = ["publication_date", "certification_date"]
         self.default_sort_column = "publication_date"
         self.validate_fiscal_period({"fiscal_period": int(fiscal_period)})
-        results = (
+        record = list(
             SubmissionAttributes.objects.filter(
-                toptier_code=toptier_code, reporting_fiscal_year=fiscal_year, reporting_fiscal_period=fiscal_period
-            )
-            .annotate(publication_date=F("published_date"), certification_date=F("certified_date"))
-            .order_by(f"{'-' if self.pagination.sort_order == 'desc' else ''}{self.pagination.sort_key}")
-            .values("publication_date", "certification_date")
+                toptier_code=toptier_code,
+                reporting_fiscal_year=fiscal_year,
+                reporting_fiscal_period=fiscal_period,
+                submission_window_id__submission_reveal_date__lte=now(),
+            ).values_list("history", flat=True)
         )
+
+        if len(record) == 0:
+            raise NoDataFoundException("No Agency Account Submission History records match the provided parameters")
+
+        # Convoluted list comprehension and sort to
+        #  A) construct the dict list
+        #  B) add secondary sort key and handle nulls in `certification_date` for sorting
+        results = sorted(
+            [
+                {"publication_date": row["published_date"], "certification_date": row["certified_date"]}
+                for row in record[0]
+            ],
+            key=lambda x: x["publication_date"]
+            if self.pagination.sort_key == "publication_date"
+            else (x["certification_date"] or "", x["publication_date"]),
+            reverse=self.pagination.sort_order == "desc",
+        )
+
         page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page)
         results = results[self.pagination.lower_limit : self.pagination.upper_limit]
         return Response(