[CRE-876, 913, 963, 964, 989] Parallelize regression system-tests, run them by default. (#19578)

* [CRE-964] Parallelize regression system-tests

* [CRE-963] Run 'regression' system-tests by default and opt-out on 'skip-e2e-regression'

* [CRE-876] Delete system-test workflows v2 from v2 registries

* [CRE-913, CRE-989] Add negative system tests for evm.FilterLogs

Author: chudilka1

.changeset/tiny-insects-judge.md

@@ -0,0 +1,5 @@
+---
+"chainlink": minor
+---
+
+#internal Parallelize CRE regression system-tests

.github/workflows/cre-regression-system-tests.yaml

@@ -64,8 +64,6 @@ jobs:
         id: define-matrix
         shell: bash
         working-directory: system-tests/tests
-        env:
-          WITH_REGRESSION: ${{ inputs.with_regression }}
         run: |
           test_names=$(go test \
           github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre \
@@ -78,7 +76,7 @@ jobs:
             | map({
                 test_name: .value,
                 test_id: .key,
-                cre_version: (if (.value | test("V2")) then "v2" else "v1" end)
+                cre_version: (if (.value | test("V1")) then "v1" else "v2" end)
               })
           ')
 
@@ -157,18 +155,18 @@ jobs:
           E2E_JD_VERSION: "0.12.7"
           E2E_TEST_CHAINLINK_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/${{ inputs.ecr_name }}"
           E2E_TEST_CHAINLINK_VERSION: ${{ inputs.chainlink_image_tag != '' && inputs.chainlink_image_tag || inputs.chainlink_version }}
-          CTF_CONFIGS: configs/workflow-gateway-capabilities-don.toml
+          CTF_CONFIGS: configs/workflow-gateway-don.toml
           CRE_VERSION: ${{ matrix.tests.cre_version }}
           TEST_NAME: ${{ matrix.tests.test_name }}
         run: |
           cd core/scripts/cre/environment
 
           # Start CRE with the appropriate contracts version (i.e. Workflow/CapabilityRegistry)
           if [[ "${CRE_VERSION}" == "v1" ]]; then
-            echo "Starting CRE with v1 contracts for test: '${TEST_NAME}'"
+            echo "Starting CRE with v1 contracts for test: '${TEST_NAME}' and configs: '${CTF_CONFIGS}'"
             go run . env start --with-plugins-docker-image "${E2E_TEST_CHAINLINK_IMAGE}:${E2E_TEST_CHAINLINK_VERSION}"
           else
-            echo "Starting CRE with v2 contracts for test: '${TEST_NAME}'"
+            echo "Starting CRE with v2 contracts for test: '${TEST_NAME}' and configs: '${CTF_CONFIGS}'"
             go run . env start --with-plugins-docker-image "${E2E_TEST_CHAINLINK_IMAGE}:${E2E_TEST_CHAINLINK_VERSION}" --with-contracts-version v2
           fi
 
@@ -179,28 +177,36 @@ jobs:
           fi
 
       - name: Run CRE${{ matrix.tests.cre_version }} Regression system tests
+        id: run-regression-tests
         shell: bash
         working-directory: system-tests/tests
         env:
           TEST_NAME: ${{ matrix.tests.test_name }}
-          TEST_TIMEOUT: 200m
+          TEST_TIMEOUT: 30m
         run: |
+          echo "Starting test: '${TEST_NAME}'"
           gotestsum \
             --jsonfile=/tmp/gotest-regression.log \
             --junitfile=/tmp/junit-report-regression.xml \
             --format=github-actions \
             -- \
             -v -run "^(${TEST_NAME})$" -timeout ${TEST_TIMEOUT} -count=1 -parallel=1 \
             github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre
+          
+          exit_code=$?
+          if [ $exit_code -eq 0 ]; then
+            echo "tests_result=✅ Tests passed" >> $GITHUB_OUTPUT
+          fi
+          exit $exit_code
 
-      - name: Process and upload flaky Regression tests results
+      - name: Process and upload flaky Regression tests results (${{ steps.run-regression-tests.outputs.tests_result }})
         if: ${{ !cancelled() }}
         uses: smartcontractkit/.github/actions/branch-out-upload@branch-out-upload/0.1.0
         with:
           junit-file-path: "/tmp/junit-report-regression.xml"
           trunk-org-slug: chainlink
           trunk-token: ${{ secrets.TRUNK_API_KEY }}
-          trunk-previous-step-outcome: ${{ steps.run-tests.outcome }}
+          trunk-previous-step-outcome: ${{ steps.run-regression-tests.outcome }}
           trunk-job-url: ${{ format('https://github.com/{0}/actions/runs/{1}/jobs/{2}', github.repository, github.run_id, job.check_run_id) }}
           # when auto-quarantine is enabled, allow this to determine test failures
           trunk-upload-only: ${{ env.ENABLE_AUTO_QUARANTINE != 'true' }}

.github/workflows/cre-system-tests.yaml

@@ -77,10 +77,10 @@ jobs:
 
           # Add list of tests with certain topologies
           PER_TEST_TOPOLOGIES_JSON=${PER_TEST_TOPOLOGIES_JSON:-'{
-            "Test_CRE_Suite_V1_SecureMint": [
+            "Test_CRE_V1_SecureMint": [
                {"topology":"workflow","configs":"configs/workflow-solana-don.toml"}
              ],
-            "Test_CRE_Suite_V1_Tron": [
+            "Test_CRE_V1_Tron": [
                {"topology":"workflow","configs":"configs/workflow-don-tron.toml"}
              ]
               }'}
@@ -231,6 +231,7 @@ jobs:
           TEST_TIMEOUT: 30m
           RUN_QUARANTINED_TESTS: "true" # always run quarantined tests in CI
         run: |
+          echo "Starting test: '${TEST_NAME}'"
           gotestsum \
             --jsonfile=/tmp/gotest.log \
             --junitfile=/tmp/junit-report.xml \
@@ -239,7 +240,12 @@ jobs:
             -v -run "^(${TEST_NAME})$" -timeout "${TEST_TIMEOUT}" -count=1 -parallel=1 \
             github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre
 
-      - name: Process and upload flaky Smoke test results
+          exit_code=$?
+          if [ $exit_code -eq 0 ]; then
+            echo "tests_result=✅ Tests passed" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Process and upload flaky Regression tests results (${{ steps.run-tests.outputs.tests_result }})
         if: ${{ !cancelled() }}
         uses: smartcontractkit/.github/actions/branch-out-upload@branch-out-upload/0.1.0
         with:

.github/workflows/integration-tests.yml

@@ -158,12 +158,12 @@ jobs:
         with:
           check-label: "run-e2e-tests"
 
-      - name: Get PR Labels (run-e2e-regression)
+      - name: Get PR Labels (skip-e2e-regression)
         if: github.event_name == 'pull_request'
         id: label-runs-e2e-regression
         uses: smartcontractkit/.github/actions/get-pr-labels@get-pr-labels/v1
         with:
-          check-label: "run-e2e-regression"
+          check-label: "skip-e2e-regression"
 
       - name: Set runner labels
         id: runner-labels
@@ -310,11 +310,13 @@ jobs:
             # Run Core CRE E2E tests on PRs if there are relevant changes
             if [[ "${CONTAINS_CHANGES}" == 'true' ]]; then
               echo "should-run=true" | tee -a "$GITHUB_OUTPUT"
+              echo "with-regression=true" | tee -a "$GITHUB_OUTPUT"
             fi
+            # Opt-out of CRE regression tests if the label 'skip-e2e-regression' is found
             if [[ "${RUN_E2E_TESTS_REGRESSION_LABEL_FOUND}" == 'true' ]]; then
-              echo "Running CRE regression tests as the label 'run-e2e-regression' is found"
+              echo "Skipping CRE regression tests: label 'skip-e2e-regression' is found"
               echo "should-run=true" | tee -a "$GITHUB_OUTPUT"
-              echo "with-regression=true" | tee -a "$GITHUB_OUTPUT"
+              echo "with-regression=false" | tee -a "$GITHUB_OUTPUT"
             fi
 
           elif [[ "${GITHUB_EVENT_NAME}" == 'merge_group' ]]; then