Merge branch 'develop' into feat/rpc-endpoints-to-fetch-data-from-key

Author: hugoclrd

.github/ISSUE_TEMPLATE/signer_bug.md

@@ -0,0 +1,32 @@
+---
+name: Signer Bug
+about: Create a report to help us improve the signer
+title: "[SIGNER BUG]"
+labels: signer
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Steps To Reproduce**
+Please provide detailed instructions (e.g. command line invocation with parameters) to reproduce the behavior.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Environment (please complete the following information):**
+ - OS: [e.g. Ubuntu / Debian]
+ - Rust version
+ - Version of the appropriate binary / software packages
+ - Signer public key
+ - Relevant log messages
+ - Tx ID of any transaction you were trying to execute 
+ - Tx ID of `aggregation-commit-indexed` call in registered cycle (if applicable)
+
+
+**Additional context**
+Please include any relevant stack traces, error messages and logs.
+
+

.github/ISSUE_TEMPLATE/testnet-bug.md

@@ -3,7 +3,7 @@ name: Testnet Bug
 about: Use this template to submit Stacks 2.0 testnet bugs
 title: "[TESTNET BUG]"
 labels: bug, testnet
-assignees: 'timstackblock'
+assignees: ''
 
 ---
 

.github/actions/open-api/Dockerfile.open-api-validate

@@ -32,7 +32,6 @@ jobs:
           - tests::bitcoin_regtest::bitcoind_integration_test
           - tests::integrations::integration_test_get_info
           - tests::neon_integrations::antientropy_integration_test
-          - tests::neon_integrations::bad_microblock_pubkey
           - tests::neon_integrations::bitcoind_forking_test
           - tests::neon_integrations::bitcoind_integration_test
           - tests::neon_integrations::block_large_tx_integration_test
@@ -43,20 +42,26 @@ jobs:
           - tests::neon_integrations::fuzzed_median_fee_rate_estimation_test_window10
           - tests::neon_integrations::fuzzed_median_fee_rate_estimation_test_window5
           - tests::neon_integrations::liquid_ustx_integration
-          - tests::neon_integrations::microblock_fork_poison_integration_test
-          - tests::neon_integrations::microblock_integration_test
-          - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY
-          - tests::neon_integrations::microblock_limit_hit_integration_test
+          # Microblock tests that are no longer needed on every CI run
+          #  (microblocks are unsupported starting in Epoch 2.5)
+          # - tests::neon_integrations::bad_microblock_pubkey
+          # - tests::neon_integrations::microblock_fork_poison_integration_test
+          # - tests::neon_integrations::microblock_integration_test
+          # - tests::neon_integrations::microblock_limit_hit_integration_test
+          # - tests::neon_integrations::test_problematic_microblocks_are_not_mined
+          # - tests::neon_integrations::test_problematic_microblocks_are_not_relayed_or_stored
+          # - tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
+          # - tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test
+          # - tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test
+          # - tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
+          # Disable this flaky test. Microblocks are no longer supported anyways.
+          # - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY
           - tests::neon_integrations::miner_submit_twice
           - tests::neon_integrations::mining_events_integration_test
           - tests::neon_integrations::pox_integration_test
           - tests::neon_integrations::push_boot_receipts
-          - tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
           - tests::neon_integrations::should_fix_2771
           - tests::neon_integrations::size_check_integration_test
-          - tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
-          - tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test
-          - tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test
           - tests::neon_integrations::stx_delegate_btc_integration_test
           - tests::neon_integrations::stx_transfer_btc_integration_test
           - tests::neon_integrations::stack_stx_burn_op_test
@@ -65,16 +70,20 @@ jobs:
           - tests::neon_integrations::test_flash_block_skip_tenure
           - tests::neon_integrations::test_problematic_blocks_are_not_mined
           - tests::neon_integrations::test_problematic_blocks_are_not_relayed_or_stored
-          - tests::neon_integrations::test_problematic_microblocks_are_not_mined
-          - tests::neon_integrations::test_problematic_microblocks_are_not_relayed_or_stored
           - tests::neon_integrations::test_problematic_txs_are_not_stored
           - tests::neon_integrations::use_latest_tip_integration_test
           - tests::neon_integrations::confirm_unparsed_ongoing_ops
           - tests::neon_integrations::min_txs
           - tests::neon_integrations::vote_for_aggregate_key_burn_op_test
+          - tests::neon_integrations::mock_miner_replay
+          - tests::neon_integrations::listunspent_max_utxos
+          - tests::neon_integrations::bitcoin_reorg_flap
+          - tests::neon_integrations::bitcoin_reorg_flap_with_follower
+          - tests::neon_integrations::start_stop_bitcoind
           - tests::epoch_25::microblocks_disabled
           - tests::should_succeed_handling_malformed_and_valid_txs
           - tests::nakamoto_integrations::simple_neon_integration
+          - tests::nakamoto_integrations::flash_blocks_on_epoch_3
           - tests::nakamoto_integrations::mine_multiple_per_tenure_integration
           - tests::nakamoto_integrations::block_proposal_api_endpoint
           - tests::nakamoto_integrations::miner_writes_proposed_block_to_stackerdb
@@ -83,23 +92,47 @@ jobs:
           - tests::nakamoto_integrations::follower_bootup
           - tests::nakamoto_integrations::forked_tenure_is_ignored
           - tests::nakamoto_integrations::nakamoto_attempt_time
+          - tests::nakamoto_integrations::skip_mining_long_tx
           - tests::signer::v0::block_proposal_rejection
           - tests::signer::v0::miner_gather_signatures
-          - tests::signer::v0::mine_2_nakamoto_reward_cycles
           - tests::signer::v0::end_of_tenure
           - tests::signer::v0::forked_tenure_okay
           - tests::signer::v0::forked_tenure_invalid
+          - tests::signer::v0::empty_sortition
           - tests::signer::v0::bitcoind_forking_test
-          - tests::nakamoto_integrations::stack_stx_burn_op_integration_test
+          - tests::signer::v0::multiple_miners
+          - tests::signer::v0::mock_sign_epoch_25
+          - tests::signer::v0::multiple_miners_mock_sign_epoch_25
+          - tests::signer::v0::miner_forking
+          - tests::signer::v0::reloads_signer_set_in
+          - tests::signer::v0::signers_broadcast_signed_blocks
+          - tests::signer::v0::min_gap_between_blocks
+          - tests::signer::v0::duplicate_signers
+          - tests::signer::v0::retry_on_rejection
+          - tests::signer::v0::locally_accepted_blocks_overriden_by_global_rejection
+          - tests::signer::v0::locally_rejected_blocks_overriden_by_global_acceptance
+          - tests::signer::v0::reorg_locally_accepted_blocks_across_tenures_succeeds
+          - tests::signer::v0::miner_recovers_when_broadcast_block_delay_across_tenures_occurs
+          - tests::signer::v0::multiple_miners_with_nakamoto_blocks
+          - tests::signer::v0::partial_tenure_fork
+          - tests::signer::v0::mine_2_nakamoto_reward_cycles
+          - tests::signer::v0::signer_set_rollover
+          - tests::signer::v0::signing_in_0th_tenure_of_reward_cycle
+          - tests::signer::v0::continue_after_tenure_extend
+          - tests::signer::v0::multiple_miners_with_custom_chain_id
+          - tests::nakamoto_integrations::burn_ops_integration_test
           - tests::nakamoto_integrations::check_block_heights
           - tests::nakamoto_integrations::clarity_burn_state
           - tests::nakamoto_integrations::check_block_times
           - tests::nakamoto_integrations::check_block_info
           - tests::nakamoto_integrations::check_block_info_rewards
           - tests::nakamoto_integrations::continue_tenure_extend
-          # Do not run this one until we figure out why it fails in CI
-          # - tests::neon_integrations::bitcoin_reorg_flap
-          # - tests::neon_integrations::bitcoin_reorg_flap_with_follower
+          - tests::nakamoto_integrations::mock_mining
+          - tests::nakamoto_integrations::multiple_miners
+          - tests::nakamoto_integrations::follower_bootup_across_multiple_cycles
+          - tests::nakamoto_integrations::utxo_check_on_startup_panic
+          - tests::nakamoto_integrations::utxo_check_on_startup_recover
+          - tests::nakamoto_integrations::v3_signer_api_endpoint
           # TODO: enable these once v1 signer is supported by a new nakamoto epoch
           # - tests::signer::v1::dkg
           # - tests::signer::v1::sign_request_rejected
@@ -116,6 +149,11 @@ jobs:
         with:
           btc-version: "25.0"
 
+      ## Increase open file descriptors limit
+      - name: Increase Open File Descriptors
+        run: |
+          sudo prlimit --nofile=4096:4096
+
       ## Run test matrix using restored cache of archive file
       ##   - Test will timeout after env.TEST_TIMEOUT minutes
       - name: Run Tests

.github/workflows/bitcoin-tests.yml

@@ -186,6 +186,29 @@ jobs:
       - check-release
     uses: ./.github/workflows/bitcoin-tests.yml
 
+
+  p2p-tests:
+    if: |
+      needs.check-release.outputs.is_release == 'true' || (
+        github.event_name == 'workflow_dispatch' ||
+        github.event_name == 'pull_request' ||
+        github.event_name == 'merge_group' ||
+        (
+          contains('
+            refs/heads/master
+            refs/heads/develop
+            refs/heads/next
+          ', github.event.pull_request.head.ref) &&
+          github.event_name == 'push'
+        )
+      )
+    name: P2P Tests
+    needs:
+      - rustfmt
+      - create-cache
+      - check-release
+    uses: ./.github/workflows/p2p-tests.yml
+
   ## Test to run on a tagged release
   ##
   ## Runs when:

.github/workflows/ci.yml

@@ -0,0 +1,87 @@
+## Github workflow to run p2p tests
+
+name: Tests::P2P
+
+on:
+  workflow_call:
+
+## env vars are transferred to composite action steps
+env:
+  BITCOIND_TEST: 0
+  RUST_BACKTRACE: full
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15
+  TEST_TIMEOUT: 30
+
+concurrency:
+  group: stackslib-tests-${{ github.head_ref || github.ref || github.run_id}}
+  ## Only cancel in progress if this is for a PR
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  # p2p integration tests with code coverage
+  integration-tests:
+    name: Integration Tests
+    runs-on: ubuntu-latest
+    strategy:
+      ## Continue with the test matrix even if we've had a failure
+      fail-fast: false
+      ## Run a maximum of 32 concurrent tests from the test matrix
+      max-parallel: 32
+      matrix:
+        test-name:
+          - net::tests::convergence::test_walk_ring_allow_15
+          - net::tests::convergence::test_walk_ring_15_plain
+          - net::tests::convergence::test_walk_ring_15_pingback
+          - net::tests::convergence::test_walk_ring_15_org_biased
+          - net::tests::convergence::test_walk_line_allowed_15
+          - net::tests::convergence::test_walk_line_15_plain
+          - net::tests::convergence::test_walk_line_15_org_biased
+          - net::tests::convergence::test_walk_line_15_pingback
+          - net::tests::convergence::test_walk_star_allowed_15
+          - net::tests::convergence::test_walk_star_15_plain
+          - net::tests::convergence::test_walk_star_15_pingback
+          - net::tests::convergence::test_walk_star_15_org_biased
+          - net::tests::convergence::test_walk_inbound_line_15
+    steps:
+      ## Setup test environment
+      - name: Setup Test Environment
+        id: setup_tests
+        uses: stacks-network/actions/stacks-core/testenv@main
+        with:
+          btc-version: "25.0"
+
+      ## Increase open file descriptors limit
+      - name: Increase Open File Descriptors
+        run: |
+          sudo prlimit --nofile=4096:4096
+
+      ## Run test matrix using restored cache of archive file
+      ##   - Test will timeout after env.TEST_TIMEOUT minutes
+      - name: Run Tests
+        id: run_tests
+        timeout-minutes: ${{ fromJSON(env.TEST_TIMEOUT) }}
+        uses: stacks-network/actions/stacks-core/run-tests@main
+        with:
+          test-name: ${{ matrix.test-name }}
+          threads: 1
+
+      ## Create and upload code coverage file
+      - name: Code Coverage
+        id: codecov
+        uses: stacks-network/actions/codecov@main
+        with:
+          test-name: ${{ matrix.test-name }}
+
+  check-tests:
+    name: Check Tests
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+      - integration-tests
+    steps:
+      - name: Check Tests Status
+        id: check_tests_status
+        uses: stacks-network/actions/check-jobs-status@main
+        with:
+          jobs: ${{ toJson(needs) }}
+          summary_print: "true"

.github/workflows/p2p-tests.yml

@@ -9,16 +9,34 @@ on:
       - ready_for_review
     paths:
       - '**.rs'
+  workflow_dispatch:
 
 concurrency:
   group: pr-differences-${{ github.head_ref || github.ref || github.run_id }}
   # Always cancel duplicate jobs
   cancel-in-progress: true
 
 jobs:
+  check-access-permissions:
+    name: Check Access Permissions
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check Access Permissions To Trigger This
+        id: check_access_permissions
+        uses: stacks-network/actions/team-membership@main
+        with:
+          username: ${{ github.actor }}
+          team: 'blockchain-team'
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+
+    outputs:
+      ignore_timeout: ${{ steps.check_access_permissions.outputs.is_team_member == 'true' && github.event_name == 'workflow_dispatch' }}
+
   # Check and output whether to run big (`stacks-node`/`stackslib`) or small (others) packages with or without shards
   check-big-packages-and-shards:
     name: Check Packages and Shards
+    needs: check-access-permissions
 
     runs-on: ubuntu-latest
 
@@ -30,10 +48,13 @@ jobs:
       run_small_packages: ${{ steps.check_packages_and_shards.outputs.run_small_packages }}
       small_packages_with_shards: ${{ steps.check_packages_and_shards.outputs.small_packages_with_shards }}
       run_stacks_signer: ${{ steps.check_packages_and_shards.outputs.run_stacks_signer }}
+      too_many_mutants: ${{ steps.check_packages_and_shards.outputs.too_many_mutants }}
 
     steps:
       - id: check_packages_and_shards
         uses: stacks-network/actions/stacks-core/mutation-testing/check-packages-and-shards@main
+        with:
+          ignore_timeout: ${{ needs.check-access-permissions.outputs.ignore_timeout }}
 
   # Mutation testing - Execute on PR on small packages that have functions modified (normal run, no shards)
   pr-differences-mutants-small-normal:
@@ -220,3 +241,4 @@ jobs:
           small_packages: ${{ needs.check-big-packages-and-shards.outputs.run_small_packages }}
           shards_for_small_packages: ${{ needs.check-big-packages-and-shards.outputs.small_packages_with_shards }}
           stacks_signer: ${{ needs.check-big-packages-and-shards.outputs.run_stacks_signer }}
+          too_many_mutants: ${{ needs.check-big-packages-and-shards.outputs.too_many_mutants }}

.github/workflows/pr-differences-mutants.yml

@@ -187,7 +187,6 @@ jobs:
     if: always()
     needs:
       - full-genesis
-      - unit-tests
       - open-api-validation
       - core-contracts-clarinet-test
     steps:

.github/workflows/stacks-core-tests.yml

@@ -21,6 +21,7 @@ on:
           - Atlas Tests
           - Bitcoin Tests
           - Epoch Tests
+          - P2P Tests
           - Slow Tests
           - Stacks-Core Tests
           - SBTC Tests
@@ -69,6 +70,23 @@ jobs:
       - create-cache
     uses: ./.github/workflows/bitcoin-tests.yml
 
+  ## Runs when:
+  ##   either or of the following:
+  ##     - workflow is 'Release Tests'
+  ##     - workflow is 'CI Tests'
+  ##     - workflow is 'P2P Tests'
+  p2p-tests:
+    if: |
+      (
+        inputs.workflow == 'Release Tests' ||
+        inputs.workflow == 'CI Tests' ||
+        inputs.workflow == 'P2P Tests'
+      )
+    name: P2P Tests
+    needs:
+      - create-cache
+    uses: ./.github/workflows/p2p-tests.yml
+
   #####################################################
   ## Runs when:
   ##   either or of the following: