diff --git a/.github/workflows/flakeguard.yml b/.github/workflows/flakeguard.yml index d88f4460b0f..b45006a17be 100644 --- a/.github/workflows/flakeguard.yml +++ b/.github/workflows/flakeguard.yml @@ -136,7 +136,7 @@ jobs: - name: Install flakeguard if: ${{ inputs.runAllTests == false }} shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@b08cab0f49aa13d5adb849554b56086f4dba8984 # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@7f8fa09251be87f51357ffc517fd93cd408f5723 # flakguard@0.1.0 - name: Find new or updated test packages if: ${{ inputs.runAllTests == false && env.RUN_CUSTOM_TEST_PACKAGES == '' }} @@ -331,7 +331,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@b08cab0f49aa13d5adb849554b56086f4dba8984 # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@7f8fa09251be87f51357ffc517fd93cd408f5723 # flakguard@0.1.0 - name: Run tests with flakeguard shell: bash @@ -443,7 +443,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@b08cab0f49aa13d5adb849554b56086f4dba8984 # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@7f8fa09251be87f51357ffc517fd93cd408f5723 # flakguard@0.1.0 - name: Aggregate Flakeguard Results id: results @@ -487,7 +487,9 @@ jobs: # Read the summary from the generated report summary=$(jq -c '.summary_data' ./flakeguard-report/all-test-report.json) + unable_to_attribute_test=$(jq 'any(.results.[]; .test_name | startswith("UnableToAttribute"))') echo "summary=$summary" >> $GITHUB_OUTPUT + echo "unable_to_attribute_test=$unable_to_attribute_test" >> $GITHUB_OUTPUT - name: Upload All Test Report as Artifact if: ${{ (success() || failure()) && fromJSON(steps.results.outputs.summary).total_runs > 0 }} @@ -685,45 +687,46 @@ jobs: ] } - - name: Send general Slack message - uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 - if: ${{ (success() || failure()) && inputs.slackNotificationAfterTestsChannelId != '' && fromJSON(steps.results.outputs.summary).flaky_tests == 0 && fromJSON(steps.results.outputs.summary).total_tests > 0 }} - id: slack - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - with: - channel-id: ${{ inputs.slackNotificationAfterTestsChannelId }} - payload: | - { - "attachments": [ - { - "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || contains(join(needs.*.result, ','), 'cancelled') && '#FFA000' || '2E7D32' }}", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Flaky Test Detector for `${{ steps.set_project_path_pretty.outputs.path }}` project - ${{ contains(join(needs.*.result, ','), 'failure') && 'Failed :x:' || contains(join(needs.*.result, ','), 'cancelled') && 'Was cancelled :warning:' || 'Passed :white_check_mark:' }}" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "${{ inputs.runAllTests == true && format('Ran all tests for `{0}` branch.', env.GIT_HEAD_REF) || format('Ran changed tests between `{0}` and `{1}` (`{2}`).', env.GIT_BASE_REF, needs.get-tests.outputs.git_head_short_sha, env.GIT_HEAD_REF) }}" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "${{ inputs.runAllTests == true && format('<{0}/{1}/actions/runs/{2}|View Flaky Detector Details>', github.server_url, github.repository, github.run_id) || format('<{0}/{1}/actions/runs/{2}|View Flaky Detector Details> | <{3}/compare/{4}...{5}#files_bucket|Compare Changes>{6}', github.server_url, github.repository, github.run_id, inputs.repoUrl, inputs.baseRef, needs.get-tests.outputs.git_head_sha, github.event_name == 'pull_request' && format(' | <{0}|View PR>', github.event.pull_request.html_url) || '') }}" - } - } - ] - } - ] - } + # This has mostly been noise. We can re-enable it if we find it necessary + # - name: Send general Slack message + # uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 + # if: ${{ (success() || failure()) && inputs.slackNotificationAfterTestsChannelId != '' && fromJSON(steps.results.outputs.summary).flaky_tests == 0 && fromJSON(steps.results.outputs.summary).total_tests > 0 }} + # id: slack + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + # with: + # channel-id: ${{ inputs.slackNotificationAfterTestsChannelId }} + # payload: | + # { + # "attachments": [ + # { + # "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || contains(join(needs.*.result, ','), 'cancelled') && '#FFA000' || '2E7D32' }}", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "Flaky Test Detector for `${{ steps.set_project_path_pretty.outputs.path }}` project - ${{ contains(join(needs.*.result, ','), 'failure') && 'Failed :x:' || contains(join(needs.*.result, ','), 'cancelled') && 'Was cancelled :warning:' || 'Passed :white_check_mark:' }}" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "${{ inputs.runAllTests == true && format('Ran all tests for `{0}` branch.', env.GIT_HEAD_REF) || format('Ran changed tests between `{0}` and `{1}` (`{2}`).', env.GIT_BASE_REF, needs.get-tests.outputs.git_head_short_sha, env.GIT_HEAD_REF) }}" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "${{ inputs.runAllTests == true && format('<{0}/{1}/actions/runs/{2}|View Flaky Detector Details>', github.server_url, github.repository, github.run_id) || format('<{0}/{1}/actions/runs/{2}|View Flaky Detector Details> | <{3}/compare/{4}...{5}#files_bucket|Compare Changes>{6}', github.server_url, github.repository, github.run_id, inputs.repoUrl, inputs.baseRef, needs.get-tests.outputs.git_head_sha, github.event_name == 'pull_request' && format(' | <{0}|View PR>', github.event.pull_request.html_url) || '') }}" + # } + # } + # ] + # } + # ] + # } - name: Send Slack message for Flakeguard Errors if: ${{ (success() || failure()) && inputs.slackNotificationAfterTestsChannelId != '' && (steps.check-errors.conclusion == 'failure' || steps.results.conclusion == 'failure' || steps.generate-report.conclusion == 'failure') }} @@ -757,8 +760,8 @@ jobs: ] } - - name: Send Slack message for Found Panics - if: ${{ (success() || failure()) && inputs.slackNotificationAfterTestsChannelId != '' && fromJSON(steps.results.outputs.summary).panicked_tests > 0 }} + - name: Send Slack message for UnableToAttribute tests + if: ${{ (success() || failure()) && inputs.slackNotificationAfterTestsChannelId != '' && steps.results.outputs.summary == 'true' }} uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} @@ -774,14 +777,14 @@ jobs: "type": "header", "text": { "type": "plain_text", - "text": "Flakeguard Found a Panic :warning:" + "text": "Flakeguard Unable to Attribute a Test Due to Panic or Race :warning:" } }, { "type": "section", "text": { "type": "mrkdwn", - "text": "<@U01Q4N37KFG> ${{ format('<{0}/{1}/actions/runs/{2}|See details>', github.server_url, github.repository, github.run_id) }} and add to panic detection scenarios." + "text": "<@U01Q4N37KFG> <@U04DYU1KLGJ> ${{ format('<{0}/{1}/actions/runs/{2}|See details>', github.server_url, github.repository, github.run_id) }} and add to panic detection scenarios." } } ] diff --git a/tools/bin/go_core_ccip_deployment_tests b/tools/bin/go_core_ccip_deployment_tests index 7a40d6d1246..b10fe569cec 100755 --- a/tools/bin/go_core_ccip_deployment_tests +++ b/tools/bin/go_core_ccip_deployment_tests @@ -8,7 +8,7 @@ EXTRA_FLAGS="" if [[ -n "$USE_FLAKEGUARD" ]]; then # Install flakeguard - go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@ec5b10aff48c9f4b787d225be30ddf1c0af32645 + go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@7f8fa09251be87f51357ffc517fd93cd408f5723 # Install gotestsum to parse JSON test outputs from flakeguard to console outputs go install gotest.tools/gotestsum@latest diff --git a/tools/bin/go_core_tests b/tools/bin/go_core_tests index 142d5bdf4ce..a9fe5ea57b4 100755 --- a/tools/bin/go_core_tests +++ b/tools/bin/go_core_tests @@ -9,7 +9,7 @@ EXTRA_FLAGS="-timeout 20m" if [[ -n "$USE_FLAKEGUARD" ]]; then # Install flakeguard - go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@b08cab0f49aa13d5adb849554b56086f4dba8984 + go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@7f8fa09251be87f51357ffc517fd93cd408f5723 # Install gotestsum to parse JSON test outputs from flakeguard to console outputs go install gotest.tools/gotestsum@latest # Make sure bins are in PATH