(3847) - Adding updated files

Author: wileyj

.github/workflows/bitcoin-tests.yml

@@ -1,205 +1,100 @@
-##
-## Bitcoin Integration Tests
-##
+## Github workflow to run bitcoin tests
 
-name: Bitcoin Integration Tests
-
-# Only run when:
-#   - PRs are (re)opened against master branch
+name: Tests::Bitcoin
 
 on:
-  pull_request:
-    types:
-      - opened
-      - reopened
+  workflow_call:
+
+defaults:
+  run:
+    shell: bash
+
+## env vars are transferred to composite action steps
+env:
+  BITCOIND_TEST: 1
+  RUST_BACKTRACE: full
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15
+  RETRIES: 3
+  RETRY_DELAY: 10000
+  TEST_TIMEOUT: 30
+  TEST_RETRIES: 2
 
 concurrency:
-  group: stacks-bitcoin-integration-tests-${{ github.ref }}
-  # Only cancel in progress if this is for a PR
+  group: bitcoin-tests-${{ github.head_ref || github.ref || github.run_id}}
+  ## Only cancel in progress if this is for a PR
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
-  # Create bitcoin image used for later tests
-  build-integration-image:
-    name: Build Image
+  # Bitcoin integration tests with code coverage
+  integration-tests:
+    name: Integration Tests
     runs-on: ubuntu-latest
-    steps:
-      - name: Checkout the latest code
-        id: git_checkout
-        uses: actions/checkout@v3
-
-      - name: Reclaim disk space
-        id: cleanup
-        run: |
-          sudo apt-get update
-          sudo apt-get remove -y '^dotnet-.*'
-          sudo apt-get remove -y '^llvm-.*'
-          sudo apt-get remove -y 'php.*'
-          sudo apt-get remove -y '^mongodb-.*'
-          sudo apt-get remove -y '^mysql-.*'
-          sudo apt-get autoremove -y
-          sudo apt-get clean
-          docker system prune --force
-
-      - name: Build bitcoin integration testing image
-        id: build_docker_image
-        env:
-          DOCKER_BUILDKIT: 1
-        # Remove .dockerignore file so codecov has access to git info and build the image
-        run: |
-          rm .dockerignore
-          docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests -t stacks-blockchain:integrations .
-
-      - name: Export docker image as tarball
-        id: export_docker_image
-        run: docker save stacks-blockchain:integrations | gzip > integration-image.tar.gz
-
-      - name: Upload built docker image
-        id: upload_docker_image
-        uses: actions/upload-artifact@v3
-        with:
-          name: integration-image.tar.gz
-          path: integration-image.tar.gz
-
-  # Run integration tests using sampled genesis block
-  sampled-genesis:
-    name: Sampled Genesis
-    runs-on: ubuntu-latest
-    needs:
-      - build-integration-image
     strategy:
+      ## Continue with the test matrix even if we've had a failure
       fail-fast: false
+      ## Run a maximum of 32 concurrent tests from the test matrix
+      max-parallel: 32
       matrix:
         test-name:
-          - tests::neon_integrations::miner_submit_twice
-          - tests::neon_integrations::microblock_integration_test
-          - tests::neon_integrations::microblock_fork_poison_integration_test
-          - tests::neon_integrations::size_check_integration_test
-          - tests::neon_integrations::cost_voting_integration
+          - tests::bitcoin_regtest::bitcoind_integration_test
           - tests::integrations::integration_test_get_info
+          - tests::neon_integrations::antientropy_integration_test ## forced failure
+          - tests::neon_integrations::bad_microblock_pubkey
+          - tests::neon_integrations::bitcoind_forking_test
           - tests::neon_integrations::bitcoind_integration_test
+          - tests::neon_integrations::block_large_tx_integration_test
+          - tests::neon_integrations::block_limit_hit_integration_test
+          - tests::neon_integrations::cost_voting_integration
+          - tests::neon_integrations::filter_long_runtime_tx_integration_test
+          - tests::neon_integrations::filter_low_fee_tx_integration_test
+          - tests::neon_integrations::fuzzed_median_fee_rate_estimation_test_window10
+          - tests::neon_integrations::fuzzed_median_fee_rate_estimation_test_window5
           - tests::neon_integrations::liquid_ustx_integration
-          - tests::neon_integrations::stx_transfer_btc_integration_test
-          - tests::neon_integrations::stx_delegate_btc_integration_test
-          - tests::neon_integrations::bitcoind_forking_test
-          - tests::neon_integrations::should_fix_2771
-          - tests::neon_integrations::pox_integration_test
+          - tests::neon_integrations::microblock_fork_poison_integration_test
+          - tests::neon_integrations::microblock_integration_test
+          - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY
+          - tests::neon_integrations::microblock_limit_hit_integration_test
+          - tests::neon_integrations::miner_submit_twice
           - tests::neon_integrations::mining_events_integration_test
-          - tests::bitcoin_regtest::bitcoind_integration_test
-          - tests::should_succeed_handling_malformed_and_valid_txs
+          - tests::neon_integrations::pox_integration_test
+          - tests::neon_integrations::push_boot_receipts
+          - tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
+          - tests::neon_integrations::should_fix_2771
+          - tests::neon_integrations::size_check_integration_test
+          - tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
           - tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test
           - tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test
-          - tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
-          - tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
-          - tests::neon_integrations::antientropy_integration_test
-          - tests::neon_integrations::filter_low_fee_tx_integration_test
-          - tests::neon_integrations::filter_long_runtime_tx_integration_test
-          - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY
-          - tests::neon_integrations::block_large_tx_integration_test
-          - tests::neon_integrations::microblock_limit_hit_integration_test
-          - tests::neon_integrations::block_limit_hit_integration_test
-          - tests::neon_integrations::fuzzed_median_fee_rate_estimation_test_window5
-          - tests::neon_integrations::fuzzed_median_fee_rate_estimation_test_window10
-          - tests::neon_integrations::use_latest_tip_integration_test
-          - tests::neon_integrations::test_flash_block_skip_tenure
+          - tests::neon_integrations::stx_delegate_btc_integration_test
+          - tests::neon_integrations::stx_transfer_btc_integration_test
           - tests::neon_integrations::test_chainwork_first_intervals
           - tests::neon_integrations::test_chainwork_partial_interval
-          - tests::neon_integrations::test_problematic_txs_are_not_stored
+          - tests::neon_integrations::test_flash_block_skip_tenure
           - tests::neon_integrations::test_problematic_blocks_are_not_mined
           - tests::neon_integrations::test_problematic_blocks_are_not_relayed_or_stored
           - tests::neon_integrations::test_problematic_microblocks_are_not_mined
           - tests::neon_integrations::test_problematic_microblocks_are_not_relayed_or_stored
-          - tests::neon_integrations::push_boot_receipts
-          - tests::epoch_205::test_dynamic_db_method_costs
-          - tests::epoch_205::transition_empty_blocks
-          - tests::epoch_205::test_cost_limit_switch_version205
-          - tests::epoch_205::test_exact_block_costs
-          - tests::epoch_205::bigger_microblock_streams_in_2_05
-          - tests::epoch_21::transition_adds_burn_block_height
-          - tests::epoch_21::transition_fixes_bitcoin_rigidity
-          - tests::epoch_21::transition_adds_pay_to_contract
-          - tests::epoch_21::transition_adds_get_pox_addr_recipients
-          - tests::epoch_21::transition_adds_mining_from_segwit
-          - tests::epoch_21::transition_removes_pox_sunset
-          - tests::epoch_21::transition_empty_blocks
-          - tests::epoch_21::test_pox_reorgs_three_flaps
-          - tests::epoch_21::test_pox_reorg_one_flap
-          - tests::epoch_21::test_pox_reorg_flap_duel
-          - tests::epoch_21::test_pox_reorg_flap_reward_cycles
-          - tests::epoch_21::test_pox_missing_five_anchor_blocks
-          - tests::epoch_21::test_sortition_divergence_pre_21
-          - tests::epoch_21::test_v1_unlock_height_with_current_stackers
-          - tests::epoch_21::test_v1_unlock_height_with_delay_and_current_stackers
-          - tests::epoch_21::trait_invocation_cross_epoch
-          - tests::epoch_22::pox_2_unlock_all
-          - tests::epoch_22::disable_pox
-          - tests::epoch_22::test_pox_reorg_one_flap
-          - tests::epoch_23::trait_invocation_behavior
-          - tests::neon_integrations::bad_microblock_pubkey
-          - tests::epoch_24::fix_to_pox_contract
-          - tests::epoch_24::verify_auto_unlock_behavior
+          - tests::neon_integrations::test_problematic_txs_are_not_stored
+          - tests::neon_integrations::use_latest_tip_integration_test
+          - tests::should_succeed_handling_malformed_and_valid_txs
     steps:
-      - name: Checkout the latest code
-        id: git_checkout
-        uses: actions/checkout@v3
-      - name: Download docker image
-        id: download_docker_image
-        uses: actions/download-artifact@v3
-        with:
-          name: integration-image.tar.gz
-      - name: Load docker image
-        id: load_docker_image
-        run: docker load -i integration-image.tar.gz && rm integration-image.tar.gz
-      - name: All integration tests with sampled genesis
-        id: bitcoin_integration_tests
-        timeout-minutes: 30
-        env:
-          DOCKER_BUILDKIT: 1
-          TEST_NAME: ${{ matrix.test-name }}
-        run: docker build -o coverage-output --build-arg test_name=${{ matrix.test-name }} -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
-      - name: Code Coverage
-        id: code_coverage
-        uses: codecov/codecov-action@v3
-        with:
-          files: ./coverage-output/lcov.info
-          name: ${{ matrix.test-name }}
-          fail_ci_if_error: false
+      ## Setup test environment
+      - name: Setup Test Environment
+        id: setup_tests
+        uses: stacks-network/actions/stacks-core/testenv@main
 
-  # Run atlas integration tests
-  atlas-test:
-    name: Atlas Test
-    runs-on: ubuntu-latest
-    needs:
-      - build-integration-image
-    strategy:
-      fail-fast: false
-      matrix:
-        test-name:
-          - tests::neon_integrations::atlas_integration_test
-          - tests::neon_integrations::atlas_stress_integration_test
-    steps:
-      - name: Checkout the latest code
-        id: git_checkout
-        uses: actions/checkout@v3
-      - name: Download docker image
-        id: download_docker_image
-        uses: actions/download-artifact@v3
+      ## Run test matrix using restored cache of archive file
+      ##   - Test will timeout after env.TEST_TIMEOUT minutes
+      - name: Run Tests
+        id: run_tests
+        timeout-minutes: ${{ fromJSON(env.TEST_TIMEOUT) }}
+        uses: stacks-network/actions/stacks-core/run-tests@main
         with:
-          name: integration-image.tar.gz
-      - name: Load docker image
-        id: load_docker_image
-        run: docker load -i integration-image.tar.gz && rm integration-image.tar.gz
-      - name: Atlas integration tests
-        id: atlas_integration_tests
-        timeout-minutes: 40
-        env:
-          DOCKER_BUILDKIT: 1
-          TEST_NAME: ${{ matrix.test-name }}
-        run: docker build -o coverage-output --build-arg test_name=${{ matrix.test-name }} -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
+          test-name: ${{ matrix.test-name }}
+          threads: 1
+
+      ## Create and upload code coverage file
       - name: Code Coverage
-        id: code_coverage
-        uses: codecov/codecov-action@v3
+        id: codecov
+        uses: stacks-network/actions/codecov@main
         with:
-          files: ./coverage-output/lcov.info
-          name: ${{ matrix.test-name }}
-          fail_ci_if_error: false
+          test-name: ${{ matrix.test-name }}