[DX-3050] chip fanout router

.github/workflows/cre-local-env-tests.yaml

@@ -112,7 +112,7 @@ jobs:
         uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
         with:
           aws-region: ${{ secrets.QA_AWS_REGION }}
-          role-to-assume: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }}
           role-duration-seconds: 1800
           mask-aws-account-id: true
 
@@ -151,7 +151,8 @@ jobs:
         env:
           DISABLE_DX_TRACKING: true
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          AWS_ECR: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.us-west-2.amazonaws.com
+          MAIN_AWS_ECR: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.us-west-2.amazonaws.com
+          SDLC_AWS_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.us-west-2.amazonaws.com
         run: |
           # Remove chip_ingress/chip_config sections since CI role lacks ECR permissions for the Atlas repo
           awk '/^\[chip_ingress\.build_config\]/,/^$/{next} /^\[chip_ingress\.pull_config\]/,/^$/{next} /^\[chip_config\.build_config\]/,/^$/{next} /^\[chip_config\.pull_config\]/,/^$/{next} {print}' configs/setup.toml > configs/setup.toml.tmp && mv configs/setup.toml.tmp configs/setup.toml
@@ -165,6 +166,7 @@ jobs:
           CTF_CONFIGS: "./configs/workflow-gateway-don.toml"
           CTF_JD_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor:0.22.1"
           CTF_CHAINLINK_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ github.event_name == 'pull_request' && format('nightly-{0}-plugins', steps.set-date.outputs.date) || inputs.chainlink_image_tag }}"
+          CTF_CHIP_ROUTER_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
           DISABLE_DX_TRACKING: "true"
           CI: "true"
         run: |

.github/workflows/cre-regression-system-tests.yaml

@@ -117,6 +117,7 @@ jobs:
       # Beholder stack will be started only for the Beholder tests
       CHIP_INGRESS_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f
       CHIP_CONFIG_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969
+      CTF_CHIP_ROUTER_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
       BILLING_PLATFORM_SERVICE_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/billing-platform-service:v1.45.0
 
     steps:
@@ -221,12 +222,9 @@ jobs:
         continue-on-error: ${{ env.ENABLE_AUTO_QUARANTINE == 'true' }}
         env:
           TEST_NAME: ${{ matrix.tests.test_name }}
-          TEST_TIMEOUT: 30m
+          TEST_TIMEOUT: 7m # let's leave 3 minutes for other steps (the whole job times out after 10 minutes)
           PARALLEL_COUNT: "10"
-          # parallelisation flags for tests
-          # fanout is necessary, because multiple tests would otherwise start chip test sinks at the same port, causing conflicts
           CRE_TEST_PARALLEL_ENABLED: "true"
-          CRE_TEST_CHIP_SINK_FANOUT_ENABLED: "true"
         run: |
           echo "Starting test: '${TEST_NAME}'"
           echo "⚠️⚠️⚠️ Add 'skip-e2e-regression' label to skip this step if necessary ⚠️⚠️⚠️"
@@ -260,12 +258,12 @@ jobs:
           artifact-name: ${{ matrix.tests.test_id }}_test_logs
 
       - name: Show Docker containers status
-        if: failure()
+        if: failure() || cancelled()
         shell: bash
         run: docker ps -a
 
       - name: Save Regression tests Docker logs
-        if: failure()
+        if: failure() || cancelled()
         shell: bash
         working-directory: system-tests/tests/regression/cre
         run: |
@@ -275,7 +273,7 @@ jobs:
           done
 
       - name: Upload all artifacts as single package
-        if: failure()
+        if: failure() || cancelled()
         uses: actions/upload-artifact@v7
         with:
           name: test-logs-${{ matrix.tests.test_name }}-${{ matrix.tests.topology }}

.github/workflows/cre-soak-memory-leak.yml

@@ -40,6 +40,7 @@ jobs:
       CTF_CHAINLINK_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/${{ inputs.ecr_name || 'chainlink' }}:${{ inputs.chainlink_image_tag }}"
       CHIP_INGRESS_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f"
       CHIP_CONFIG_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969"
+      CTF_CHIP_ROUTER_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
 
     steps:
       - name: Enable S3 Cache for Self-Hosted Runners
@@ -76,28 +77,12 @@ jobs:
           registries: ${{ format('{0},{1}', secrets.QA_AWS_ACCOUNT_NUMBER, secrets.AWS_ACCOUNT_ID_PROD) }}
         env:
           AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-
-      - name: Install CTF binary
-        shell: bash
-        working-directory: core/scripts/cre/environment
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CTF_TAG: "framework/v0.15.2"
-        run: |
-          echo "::startgroup::Install CTF binary"
-          mkdir -p bin
-          gh release download "${CTF_TAG}" --pattern "framework-v*-linux-amd64.tar.gz" --clobber --repo smartcontractkit/chainlink-testing-framework -O ctf.tar.gz
-          tar -xzf ctf.tar.gz
-          mv ctf bin/ctf
-          chmod +x bin/ctf
-          echo "::endgroup::"
-
       - name: Start observability stack
         shell: bash
         working-directory: core/scripts/cre/environment
         run: |
           echo "::startgroup::Starting observability stack (required by leak package)"
-          ./bin/ctf obs up -f
+          go run . obs up -f
           echo "::endgroup::"
 
       - name: Start local CRE

.github/workflows/cre-system-tests.yaml

@@ -248,6 +248,7 @@ jobs:
         env:
           CTF_JD_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor:0.22.1"
           CTF_CHAINLINK_IMAGE: "${{ steps.resolve-chainlink-image.outputs.resolved_image }}"
+          CTF_CHIP_ROUTER_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
           CTF_CONFIGS: ${{ matrix.tests.configs }}
           CRE_VERSION: ${{ matrix.tests.cre_version }}
           TEST_NAME: ${{ matrix.tests.test_name }}
@@ -327,15 +328,12 @@ jobs:
         continue-on-error: ${{ env.ENABLE_AUTO_QUARANTINE == 'true' }}
         env:
           TEST_NAME: ${{ matrix.tests.test_name }}
-          TEST_TIMEOUT: 30m
+          TEST_TIMEOUT: 7m # let's leave 3 minutes for other steps (the whole job times out after 10 minutes)
           RUN_QUARANTINED_TESTS: "true" # always run quarantined tests in CI
           TOPOLOGY_NAME: ${{ matrix.tests.topology }}
           GITHUB_TOKEN: ${{ steps.github-token.outputs.access-token || '' }} # to avoid rate limiting when downloading protobuf files from GitHub
           PARALLEL_COUNT: "10"
-          # parallelisation flags for tests
-          # fanout is necessary, because multiple tests would otherwise start chip test sinks at the same port, causing conflicts
           CRE_TEST_PARALLEL_ENABLED: "true"
-          CRE_TEST_CHIP_SINK_FANOUT_ENABLED: "true"
         run: |
           echo "Starting test: '${TEST_NAME}'"
           gotestsum \
@@ -365,12 +363,12 @@ jobs:
           artifact-name: ${{ matrix.tests.test_id }}_test_logs
 
       - name: Show Docker containers status
-        if: failure()
+        if: failure() || cancelled()
         shell: bash
         run: docker ps -a
 
       - name: Save Smoke tests Docker logs
-        if: failure()
+        if: failure() || cancelled()
         shell: bash
         working-directory: system-tests/tests/smoke/cre
         run: |
@@ -383,7 +381,7 @@ jobs:
           done
 
       - name: Upload all artifacts as single package
-        if: failure()
+        if: failure() || cancelled()
         uses: actions/upload-artifact@v7
         with:
           name: test-logs-${{ matrix.tests.test_name }}-${{ matrix.tests.topology }}

.github/workflows/devenv-compat.yml

@@ -99,6 +99,8 @@ jobs:
       CTF_LOG_LEVEL: ${{ inputs.ctf-log-level }}
       # JD is required by the local CRE-based tests
       CTF_JD_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor:0.22.1"
+      # ChIP Router is required by the local CRE-based tests
+      CTF_CHIP_ROUTER_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
     steps:
       - name: Checkout code
         uses: actions/checkout@v5

core/scripts/cre/environment/configs/setup.toml

@@ -12,7 +12,19 @@ local_image = "job-distributor:0.22.1"
 
 [job_distributor.pull_config]
 local_image = "job-distributor:0.22.1"
-ecr_image = "{{.ECR}}/job-distributor:0.22.1"
+ecr_image = "{{.MAIN_ECR}}/job-distributor:0.22.1"
+
+[chip_router.build_config]
+repository = "https://github.com/smartcontractkit/chainlink-testing-framework"
+branch = "main"
+commit = "dd420c0d953334e3fef9a109b6816de706f6be90"
+dockerfile = "framework/components/chiprouter/Dockerfile"
+docker_ctx = "framework/components/chiprouter"
+local_image = "local-cre-chip-router:v1.0.1"
+
+[chip_router.pull_config]
+local_image = "local-cre-chip-router:v1.0.1"
+ecr_image = "{{.SDLC_ECR}}/local-cre-chip-router:v1.0.1"
 
 [chip_ingress.build_config]
 repository = "https://github.com/smartcontractkit/atlas"
@@ -25,7 +37,7 @@ pre_run = "pushd chip-ingress && go mod vendor && popd"
 
 [chip_ingress.pull_config]
 local_image = "chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f"
-ecr_image = "{{.ECR}}/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f"
+ecr_image = "{{.MAIN_ECR}}/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f"
 
 [chip_config.build_config]
 repository = "https://github.com/smartcontractkit/atlas"
@@ -38,7 +50,7 @@ pre_run = "pushd chip-config && go mod vendor && popd"
 
 [chip_config.pull_config]
 local_image = "chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969"
-ecr_image = "{{.ECR}}/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969"
+ecr_image = "{{.MAIN_ECR}}/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969"
 
 [billing_platform_service.build_config]
 repository = "https://github.com/smartcontractkit/billing-platform-service"
@@ -50,7 +62,7 @@ local_image = "billing-platform-service:local-cre"
 
 [billing_platform_service.pull_config]
 local_image = "billing-platform-service:local-cre"
-ecr_image = "{{.ECR}}/billing-platform-service:1.36.1"
+ecr_image = "{{.MAIN_ECR}}/billing-platform-service:1.36.1"
 
 [observability]
 repository = "https://github.com/smartcontractkit/chainlink-observability"

core/scripts/cre/environment/configs/workflow-don-solana.toml

@@ -1,4 +1,7 @@
 
+[chip_router]
+  image = "local-cre-chip-router:v1.0.1"
+
 [[blockchains]]
   type = "anvil"
   chain_id = "1337"

core/scripts/cre/environment/configs/workflow-don-tron.toml

@@ -1,4 +1,7 @@
 
+[chip_router]
+  image = "local-cre-chip-router:v1.0.1"
+
 [[blockchains]]
   type = "anvil"
   chain_id = "1337"

core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml

@@ -1,4 +1,7 @@
 
+[chip_router]
+  image = "local-cre-chip-router:v1.0.1"
+
 [[blockchains]]
   type = "anvil"
   chain_id = "1337"

core/scripts/cre/environment/configs/workflow-gateway-don-aptos.toml

@@ -1,6 +1,9 @@
 # Same as workflow-gateway-don.toml but with Aptos chain and a single Aptos capability.
 # Anvil 1337: registry and gateway. Aptos: local devnet (chain_id 4). Run: env config path <this file>, then env start.
 
+[chip_router]
+  image = "local-cre-chip-router:v1.0.1"
+
 [[blockchains]]
   type = "anvil"
   chain_id = "1337"

core/scripts/cre/environment/configs/workflow-gateway-don-grpc-source.toml

@@ -4,6 +4,9 @@
 #
 # Used by: system-tests/tests/smoke/cre/v2_grpc_source_test.go
 
+[chip_router]
+  image = "local-cre-chip-router:v1.0.1"
+
 [[blockchains]]
   type = "anvil"
   chain_id = "1337"

core/scripts/cre/environment/configs/workflow-gateway-don.toml

@@ -1,4 +1,7 @@
 
+[chip_router]
+  image = "local-cre-chip-router:v1.0.1"
+
 [[blockchains]]
   type = "anvil"
   chain_id = "1337"

core/scripts/cre/environment/configs/workflow-gateway-legacy-vault-don.toml

@@ -1,5 +1,8 @@
 # NOTE: Identical to workflow-gatewway-capabilities.toml but with a vault capability config override
 # to disable the new pending queue feature.
+[chip_router]
+  image = "local-cre-chip-router:v1.0.1"
+
 [[blockchains]]
   type = "anvil"
   chain_id = "1337"

core/scripts/cre/environment/configs/workflow-gateway-mock-don.toml

@@ -1,3 +1,6 @@
+[chip_router]
+    image = "local-cre-chip-router:v1.0.1"
+
 [[blockchains]]
     chain_id = "1337"
     container_name = "anvil-1337"

core/scripts/cre/environment/configs/workflow-gateway-sharded-5-dons.toml

@@ -1,4 +1,7 @@
 
+[chip_router]
+  image = "local-cre-chip-router:v1.0.1"
+
 [[blockchains]]
   type = "anvil"
   chain_id = "1337"

core/scripts/cre/environment/configs/workflow-gateway-sharded-don.toml

@@ -1,4 +1,7 @@
 
+[chip_router]
+  image = "local-cre-chip-router:v1.0.1"
+
 [[blockchains]]
   type = "anvil"
   chain_id = "1337"