diff --git a/.github/workflows/code-climate-before-build.yaml b/.github/workflows/code-climate-before-build.yaml deleted file mode 100644 index 5e442e1693..0000000000 --- a/.github/workflows/code-climate-before-build.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Code Climate Before Build - -on: - workflow_call: - inputs: - source-branch: - description: The branch that tests are run against - type: string - required: true - source-commit-sha: - description: The specific commit on the branch that tests are run against - type: string - required: true - secrets: - cc_test_reporter_id: - required: true - -jobs: - Run: - name: Run - runs-on: ${{ vars.RUNNER_VERSION }} - env: - GIT_BRANCH: ${{ inputs.source-branch }} - GIT_COMMIT_SHA: ${{ inputs.source-commit-sha }} - CC_TEST_REPORTER_ID: ${{ secrets.cc_test_reporter_id }} - steps: - - name: Install Dependency - run: | - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter - chmod +x ./cc-test-reporter - - - name: Run Before Build - run: ./cc-test-reporter before-build diff --git a/.github/workflows/code-climate-report-coverage.yaml b/.github/workflows/code-climate-report-coverage.yaml deleted file mode 100644 index f486ac136f..0000000000 --- a/.github/workflows/code-climate-report-coverage.yaml +++ /dev/null @@ -1,52 +0,0 @@ -name: Code Climate Report Coverage - -on: - workflow_call: - inputs: - source-branch: - description: The branch that tests are run against - type: string - required: true - source-commit-sha: - description: The specific commit on the branch that tests are run against - type: string - required: true - secrets: - cc_test_reporter_id: - description: ID assigned by Code Climate for writing test coverage - required: true - -jobs: - Run: - name: Run - runs-on: ${{ vars.RUNNER_VERSION }} - env: - GIT_BRANCH: ${{ inputs.source-branch }} - GIT_COMMIT_SHA: ${{ inputs.source-commit-sha }} - CC_TEST_REPORTER_ID: ${{ secrets.cc_test_reporter_id }} - steps: - - name: Checkout Source Repository - uses: actions/checkout@v4 - with: - path: usaspending-api - - - name: Install Dependency - working-directory: ./usaspending-api - run: | - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter - chmod +x ./cc-test-reporter - - - name: Download coverage reports - uses: actions/download-artifact@v4 - with: - merge-multiple: 'true' - pattern: coverage.*.xml - path: ./usaspending-api - - - name: Report Test Coverage - working-directory: ./usaspending-api - run: | - ls -lh coverage.*.xml - for cf in coverage.*.xml; do ./cc-test-reporter format-coverage --input-type coverage.py --output coverage/codeclimate.$(echo "$cf" | cut -d'.' -f2).xml coverage.$(echo "$cf" | cut -d'.' -f2).xml; done - ls coverage/ - ./cc-test-reporter sum-coverage --output - --parts $(find . -maxdepth 1 -name 'coverage.*.xml' | wc -l) ./coverage/codeclimate.*.xml | ./cc-test-reporter upload-coverage --input - diff --git a/.github/workflows/pull-request-checks.yaml b/.github/workflows/pull-request-checks.yaml index 6897d79ee6..04c83ecf09 100644 --- a/.github/workflows/pull-request-checks.yaml +++ b/.github/workflows/pull-request-checks.yaml @@ -44,24 +44,12 @@ jobs: needs.Cancel-If-Draft-Pull-Request.result == 'skipped' uses: ./.github/workflows/build-broker-docker-image-for-test.yaml - # Prepare Code Climate to receive coverage report - Code-Climate-Before-Build: - name: Code Climate Before Build - needs: Run-Code-Style-Checks - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/code-climate-before-build.yaml - with: - source-branch: ${{ github.head_ref }} - source-commit-sha: ${{ github.event.pull_request.head.sha }} - secrets: - cc_test_reporter_id: ${{ secrets.CC_TEST_REPORTER_ID }} - # The test suite is broken into multiple groups based on a combination of what is under test # and the time that it takes to run tests for that group. Run-Spark-Integration-Load-Transaction-FABS-and-FPDS-Tests: name: Run Spark Integration Load Transactions FABS and FPDS Tests needs: - - Code-Climate-Before-Build + - Run-Code-Style-Checks - Build-Broker-Docker-Image if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml @@ -69,7 +57,7 @@ jobs: Run-Spark-Integration-Load-Transactions-Lookup-Tests: name: Run Spark Integration Load Transactions Lookup Tests needs: - - Code-Climate-Before-Build + - Run-Code-Style-Checks - Build-Broker-Docker-Image if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/test-spark-integration-load-transactions-lookup.yaml @@ -77,7 +65,7 @@ jobs: Run-Spark-Integration-Load-To-From-Delta-Tests: name: Run Spark Integration Load To From Delta Tests needs: - - Code-Climate-Before-Build + - Run-Code-Style-Checks - Build-Broker-Docker-Image if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/test-spark-integration-load-to-from-delta.yaml @@ -85,7 +73,7 @@ jobs: Run-Spark-Integration-Other-Tests: name: Run Spark Integration Other Tests needs: - - Code-Climate-Before-Build + - Run-Code-Style-Checks - Build-Broker-Docker-Image if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/test-spark-integration-other.yaml @@ -93,7 +81,7 @@ jobs: Run-Non-Spark-Integration-Tests: name: Run Non-Spark Integration Tests needs: - - Code-Climate-Before-Build + - Run-Code-Style-Checks - Build-Broker-Docker-Image if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/test-non-spark-integration.yaml @@ -101,38 +89,19 @@ jobs: Run-Non-Spark-Integration-Tests-With-Signal-Handling: name: Run Non-Spark Integration Tests With Signal Handling needs: - - Code-Climate-Before-Build + - Run-Code-Style-Checks - Build-Broker-Docker-Image if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/test-non-spark-integration-signal-handling.yaml Run-Unit-Tests: name: Run Unit Tests - needs: Code-Climate-Before-Build + needs: Run-Code-Style-Checks if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/test-unit.yaml Run-Unit-Tests-With-Signal-Handling: name: Run Unit Tests With Signal Handling - needs: Code-Climate-Before-Build + needs: Run-Code-Style-Checks if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/test-unit-signal-handling.yaml - - Code-Climate-Report-Coverage: - name: Code Climate Report Coverage - needs: - - Run-Spark-Integration-Load-Transaction-FABS-and-FPDS-Tests - - Run-Spark-Integration-Load-Transactions-Lookup-Tests - - Run-Spark-Integration-Load-To-From-Delta-Tests - - Run-Spark-Integration-Other-Tests - - Run-Non-Spark-Integration-Tests - - Run-Non-Spark-Integration-Tests-With-Signal-Handling - - Run-Unit-Tests - - Run-Unit-Tests-With-Signal-Handling - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/code-climate-report-coverage.yaml - with: - source-branch: ${{ github.head_ref }} - source-commit-sha: ${{ github.event.pull_request.head.sha }} - secrets: - cc_test_reporter_id: ${{ secrets.CC_TEST_REPORTER_ID }} diff --git a/usaspending_api/download/delta_models/object_class_program_activity_download.py b/usaspending_api/download/delta_models/object_class_program_activity_download.py new file mode 100644 index 0000000000..d96dce1a91 --- /dev/null +++ b/usaspending_api/download/delta_models/object_class_program_activity_download.py @@ -0,0 +1,273 @@ +OBJECT_CLASS_PROGRAM_ACTIVITY_DOWNLOAD_COLUMNS = { + "financial_accounts_by_program_activity_object_class_id": { + "delta": "INTEGER NOT NULL", + "postgres": "INTEGER NOT NULL", + }, + "submission_id": {"delta": "INTEGER NOT NULL", "postgres": "INTEGER NOT NULL"}, + "owning_agency_name": {"delta": "STRING", "postgres": "TEXT"}, + "federal_account_symbol": {"delta": "STRING", "postgres": "TEXT"}, + "federal_account_name": {"delta": "STRING", "postgres": "TEXT"}, + "agency_identifier_name": {"delta": "STRING", "postgres": "TEXT"}, + "program_activity_code": {"delta": "STRING", "postgres": "TEXT"}, + "program_activity_name": {"delta": "STRING", "postgres": "TEXT"}, + "object_class_code": {"delta": "STRING", "postgres": "TEXT"}, + "object_class_name": {"delta": "STRING", "postgres": "TEXT"}, + "direct_or_reimbursable_funding_source": {"delta": "STRING", "postgres": "TEXT"}, + "disaster_emergency_fund_code": {"delta": "STRING", "postgres": "TEXT"}, + "disaster_emergency_fund_name": {"delta": "STRING", "postgres": "TEXT"}, + "funding_toptier_agency_id": {"delta": "INTEGER", "postgres": "INTEGER"}, + "federal_account_id": {"delta": "INTEGER", "postgres": "INTEGER"}, + "budget_function_title": {"delta": "STRING", "postgres": "TEXT"}, + "budget_function_code": {"delta": "STRING", "postgres": "TEXT"}, + "budget_subfunction_title": {"delta": "STRING", "postgres": "TEXT"}, + "budget_subfunction_code": {"delta": "STRING", "postgres": "TEXT"}, + "reporting_agency_name": {"delta": "STRING", "postgres": "TEXT"}, + "reporting_fiscal_period": {"delta": "INTEGER", "postgres": "INTEGER"}, + "reporting_fiscal_quarter": {"delta": "INTEGER", "postgres": "INTEGER"}, + "reporting_fiscal_year": {"delta": "INTEGER", "postgres": "INTEGER"}, + "quarter_format_flag": {"delta": "BOOLEAN", "postgres": "BOOLEAN"}, + "submission_period": {"delta": "STRING", "postgres": "TEXT"}, + "USSGL480100_undelivered_orders_obligations_unpaid_FYB": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL480100_undelivered_orders_obligations_unpaid": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL483100_undelivered_orders_obligations_transferred_unpaid": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL488100_upward_adj_prior_year_undeliv_orders_oblig_unpaid": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL490100_delivered_orders_obligations_unpaid_FYB": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL490100_delivered_orders_obligations_unpaid": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL493100_delivered_orders_obligations_transferred_unpaid": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL498100_upward_adj_of_prior_year_deliv_orders_oblig_unpaid": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL480200_undelivered_orders_obligations_prepaid_advanced_FYB": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL480200_undelivered_orders_obligations_prepaid_advanced": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL483200_undeliv_orders_oblig_transferred_prepaid_advanced": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL488200_upward_adj_prior_year_undeliv_orders_oblig_prepaid": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL490200_delivered_orders_obligations_paid": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL490800_authority_outlayed_not_yet_disbursed_FYB": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL490800_authority_outlayed_not_yet_disbursed": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL498200_upward_adj_of_prior_year_deliv_orders_oblig_paid": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "obligations_undelivered_orders_unpaid_total_FYB": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "obligations_undelivered_orders_unpaid_total": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "obligations_delivered_orders_unpaid_total": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "obligations_delivered_orders_unpaid_total_FYB": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "gross_outlays_undelivered_orders_prepaid_total": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "gross_outlays_undelivered_orders_prepaid_total_fyb": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "gross_outlays_delivered_orders_paid_total_FYB": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "gross_outlays_delivered_orders_paid_total": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "gross_outlay_amount_FYB": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "gross_outlay_amount_FYB_to_period_end": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "obligations_incurred": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL487100_downward_adj_prior_year_unpaid_undeliv_orders_oblig": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL497100_downward_adj_prior_year_unpaid_deliv_orders_oblig": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL487200_downward_adj_prior_year_prepaid_undeliv_order_oblig": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "USSGL497200_downward_adj_of_prior_year_paid_deliv_orders_oblig": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "deobligations_or_recoveries_or_refunds_from_prior_year": { + "delta": "NUMERIC(23,2)", + "postgres": "NUMERIC(23,2)", + }, + "last_modified_date": {"delta": "DATE", "postgres": "DATE"}, +} + +OBJECT_CLASS_PROGRAM_ACTIVITY_DOWNLOAD_DELTA_COLUMNS = { + k: v["delta"] for k, v in OBJECT_CLASS_PROGRAM_ACTIVITY_DOWNLOAD_COLUMNS.items() +} +OBJECT_CLASS_PROGRAM_ACTIVITY_DOWNLOAD_POSTGRES_COLUMNS = { + k: v["postgres"] for k, v in OBJECT_CLASS_PROGRAM_ACTIVITY_DOWNLOAD_COLUMNS.items() +} + +object_class_program_activity_download_create_sql_string = rf""" + CREATE OR REPLACE TABLE {{DESTINATION_TABLE}} ( + {", ".join([f"{key} {val}" for key, val in OBJECT_CLASS_PROGRAM_ACTIVITY_DOWNLOAD_DELTA_COLUMNS.items()])} + ) + USING DELTA + LOCATION 's3a://{{SPARK_S3_BUCKET}}/{{DELTA_LAKE_S3_PATH}}/{{DESTINATION_DATABASE}}/{{DESTINATION_TABLE}}' + """ + +object_class_program_activity_download_load_sql_string = rf""" + INSERT OVERWRITE {{DESTINATION_DATABASE}}.{{DESTINATION_TABLE}} ( + {",".join(list(OBJECT_CLASS_PROGRAM_ACTIVITY_DOWNLOAD_COLUMNS))} + ) + SELECT + fabpaoc.financial_accounts_by_program_activity_object_class_id, + fabpaoc.submission_id, + ta.name AS owning_agency_name, + fa.federal_account_code AS federal_account_symbol, + fa.account_title AS federal_account_name, + cgac.agency_name AS agency_identifier_name, + rpa.program_activity_code, + rpa.program_activity_name, + oc.object_class AS object_class_code, + oc.object_class_name, + oc.direct_reimbursable AS direct_or_reimbursable_funding_source, + fabpaoc.disaster_emergency_fund_code, + defc.title AS disaster_emergency_fund_name, + taa.funding_toptier_agency_id, + taa.federal_account_id, + taa.budget_function_title, + taa.budget_function_code, + taa.budget_subfunction_title, + taa.budget_subfunction_code, + sa.reporting_agency_name, + sa.reporting_fiscal_period, + sa.reporting_fiscal_quarter, + sa.reporting_fiscal_year, + sa.quarter_format_flag, + CASE + WHEN sa.quarter_format_flag = TRUE + THEN + CONCAT( + CAST('FY' AS STRING), + CAST(sa.reporting_fiscal_year AS STRING), + CAST('Q' AS STRING), + CAST(sa.reporting_fiscal_quarter AS STRING) + ) + ELSE + CONCAT( + CAST('FY' AS STRING), + CAST(sa.reporting_fiscal_year AS STRING), + CAST('P' AS STRING), + LPAD(CAST(sa.reporting_fiscal_period AS STRING), 2, '0') + ) + END AS submission_period, + fabpaoc.ussgl480100_undelivered_orders_obligations_unpaid_fyb AS USSGL480100_undelivered_orders_obligations_unpaid_FYB, + fabpaoc.ussgl480100_undelivered_orders_obligations_unpaid_cpe AS USSGL480100_undelivered_orders_obligations_unpaid, + fabpaoc.ussgl483100_undelivered_orders_oblig_transferred_unpaid_cpe AS USSGL483100_undelivered_orders_obligations_transferred_unpaid, + fabpaoc.ussgl488100_upward_adjust_pri_undeliv_order_oblig_unpaid_cpe AS USSGL488100_upward_adj_prior_year_undeliv_orders_oblig_unpaid, + fabpaoc.ussgl490100_delivered_orders_obligations_unpaid_fyb AS USSGL490100_delivered_orders_obligations_unpaid_FYB, + fabpaoc.ussgl490100_delivered_orders_obligations_unpaid_cpe AS USSGL490100_delivered_orders_obligations_unpaid, + fabpaoc.ussgl493100_delivered_orders_oblig_transferred_unpaid_cpe AS USSGL493100_delivered_orders_obligations_transferred_unpaid, + fabpaoc.ussgl498100_upward_adjust_pri_deliv_orders_oblig_unpaid_cpe AS USSGL498100_upward_adj_of_prior_year_deliv_orders_oblig_unpaid, + fabpaoc.ussgl480200_undelivered_orders_oblig_prepaid_advanced_fyb AS USSGL480200_undelivered_orders_obligations_prepaid_advanced_FYB, + fabpaoc.ussgl480200_undelivered_orders_oblig_prepaid_advanced_cpe AS USSGL480200_undelivered_orders_obligations_prepaid_advanced, + fabpaoc.ussgl483200_undeliv_orders_oblig_transferred_prepaid_adv_cpe AS USSGL483200_undeliv_orders_oblig_transferred_prepaid_advanced, + fabpaoc.ussgl488200_up_adjust_pri_undeliv_order_oblig_ppaid_adv_cpe AS USSGL488200_upward_adj_prior_year_undeliv_orders_oblig_prepaid, + fabpaoc.ussgl490200_delivered_orders_obligations_paid_cpe AS USSGL490200_delivered_orders_obligations_paid, + fabpaoc.ussgl490800_authority_outlayed_not_yet_disbursed_fyb AS USSGL490800_authority_outlayed_not_yet_disbursed_FYB, + fabpaoc.ussgl490800_authority_outlayed_not_yet_disbursed_cpe AS USSGL490800_authority_outlayed_not_yet_disbursed, + fabpaoc.ussgl498200_upward_adjust_pri_deliv_orders_oblig_paid_cpe AS USSGL498200_upward_adj_of_prior_year_deliv_orders_oblig_paid, + fabpaoc.obligations_undelivered_orders_unpaid_total_fyb AS obligations_undelivered_orders_unpaid_total_FYB, + fabpaoc.obligations_undelivered_orders_unpaid_total_cpe AS obligations_undelivered_orders_unpaid_total, + fabpaoc.obligations_delivered_orders_unpaid_total_cpe AS obligations_delivered_orders_unpaid_total, + fabpaoc.obligations_delivered_orders_unpaid_total_cpe AS obligations_delivered_orders_unpaid_total_FYB, + fabpaoc.gross_outlays_undelivered_orders_prepaid_total_cpe AS gross_outlays_undelivered_orders_prepaid_total, + fabpaoc.gross_outlays_undelivered_orders_prepaid_total_cpe AS gross_outlays_undelivered_orders_prepaid_total_FYB, + fabpaoc.gross_outlays_delivered_orders_paid_total_fyb AS gross_outlays_delivered_orders_paid_total_FYB, + fabpaoc.gross_outlays_delivered_orders_paid_total_cpe AS gross_outlays_delivered_orders_paid_total, + fabpaoc.gross_outlay_amount_by_program_object_class_fyb AS gross_outlay_amount_FYB, + fabpaoc.gross_outlay_amount_by_program_object_class_cpe AS gross_outlay_amount_FYB_to_period_end, + fabpaoc.obligations_incurred_by_program_object_class_cpe AS obligations_incurred, + fabpaoc.ussgl487100_down_adj_pri_unpaid_undel_orders_oblig_recov_cpe AS USSGL487100_downward_adj_prior_year_unpaid_undeliv_orders_oblig, + fabpaoc.ussgl497100_down_adj_pri_unpaid_deliv_orders_oblig_recov_cpe AS USSGL497100_downward_adj_prior_year_unpaid_deliv_orders_oblig, + fabpaoc.ussgl487200_down_adj_pri_ppaid_undel_orders_oblig_refund_cpe AS USSGL487200_downward_adj_prior_year_prepaid_undeliv_order_oblig, + fabpaoc.ussgl497200_down_adj_pri_paid_deliv_orders_oblig_refund_cpe AS USSGL497200_downward_adj_of_prior_year_paid_deliv_orders_oblig, + fabpaoc.deobligations_recoveries_refund_pri_program_object_class_cpe AS deobligations_or_recoveries_or_refunds_from_prior_year, + CAST(sa.published_date AS DATE) AS last_modified_date + FROM + global_temp.financial_accounts_by_program_activity_object_class AS fabpaoc + INNER JOIN global_temp.submission_attributes AS sa ON + fabpaoc.submission_id = sa.submission_id + LEFT JOIN global_temp.treasury_appropriation_account AS taa ON + fabpaoc.treasury_account_id = taa.treasury_account_identifier + LEFT JOIN global_temp.federal_account AS fa ON + taa.federal_account_id = fa.id + LEFT JOIN global_temp.toptier_agency AS ta ON + fa.parent_toptier_agency_id = ta.toptier_agency_id + LEFT JOIN global_temp.ref_program_activity AS rpa ON + fabpaoc.program_activity_id = rpa.id + LEFT JOIN global_temp.object_class AS oc ON + fabpaoc.object_class_id = oc.id + LEFT JOIN global_temp.disaster_emergency_fund_code AS defc ON + fabpaoc.disaster_emergency_fund_code = defc.code + LEFT JOIN global_temp.cgac ON + taa.agency_id = cgac.cgac_code + """ diff --git a/usaspending_api/etl/management/commands/load_query_to_delta.py b/usaspending_api/etl/management/commands/load_query_to_delta.py index 210b4ff40d..3e92efc6af 100644 --- a/usaspending_api/etl/management/commands/load_query_to_delta.py +++ b/usaspending_api/etl/management/commands/load_query_to_delta.py @@ -24,6 +24,11 @@ account_download_create_sql_string, account_download_load_sql_string, ) +from usaspending_api.download.delta_models.object_class_program_activity_download import ( + OBJECT_CLASS_PROGRAM_ACTIVITY_DOWNLOAD_POSTGRES_COLUMNS, + object_class_program_activity_download_create_sql_string, + object_class_program_activity_download_load_sql_string, +) from usaspending_api.recipient.delta_models import ( RECIPIENT_LOOKUP_POSTGRES_COLUMNS, RECIPIENT_PROFILE_POSTGRES_COLUMNS, @@ -337,11 +342,31 @@ "tsvectors": None, "postgres_partition_spec": None, }, + "object_class_program_activity_download": { + "model": None, + "is_from_broker": False, + "source_query": [object_class_program_activity_download_load_sql_string], + "source_query_incremental": None, + "source_database": None, + "source_table": None, + "destination_database": "rpt", + "swap_table": None, + "swap_schema": None, + "partition_column": "financial_accounts_by_program_activity_object_class_id", + "partition_column_type": "numeric", + "is_partition_column_unique": False, + "delta_table_create_sql": object_class_program_activity_download_create_sql_string, + "source_schema": OBJECT_CLASS_PROGRAM_ACTIVITY_DOWNLOAD_POSTGRES_COLUMNS, + "custom_schema": None, + "column_names": list(OBJECT_CLASS_PROGRAM_ACTIVITY_DOWNLOAD_POSTGRES_COLUMNS), + "postgres_seq_name": None, + "tsvectors": None, + "postgres_partition_spec": None, + }, } class Command(BaseCommand): - help = """ This command reads data via a Spark SQL query that relies on delta tables that have already been loaded paired with temporary views of tables in a Postgres database. As of now, it only supports a full reload of a table. diff --git a/usaspending_api/etl/tests/integration/test_create_delta_table.py b/usaspending_api/etl/tests/integration/test_create_delta_table.py index a4b648b2a3..61bbe145fc 100644 --- a/usaspending_api/etl/tests/integration/test_create_delta_table.py +++ b/usaspending_api/etl/tests/integration/test_create_delta_table.py @@ -3,9 +3,8 @@ NOTE: Uses Pytest Fixtures from immediate parent conftest.py: usaspending_api/etl/tests/conftest.py """ -from pyspark.sql import SparkSession - from django.core.management import call_command +from pyspark.sql import SparkSession from usaspending_api.etl.management.commands.create_delta_table import TABLE_SPEC @@ -106,3 +105,9 @@ def test_create_delta_table_for_recipient_lookup_with_alt_db_and_name( _verify_delta_table_creation( spark, "recipient_lookup", s3_unittest_data_bucket, alt_db="my_alt_db", alt_name="recipient_lookup_alt_name" ) + + +def test_create_delta_table_for_object_class_program_activity_download( + spark, s3_unittest_data_bucket, hive_unittest_metastore_db +): + _verify_delta_table_creation(spark, "object_class_program_activity_download", s3_unittest_data_bucket)