Skip to content

Commit 60ba0a0

Browse files
carte7000carte7000
authored
Nightly load test (#381)
* Nightly load test * Remove workflow dispatch
1 parent b29fdbb commit 60ba0a0

File tree

4 files changed

+168
-79
lines changed

4 files changed

+168
-79
lines changed

‎.github/actions/run-load-test/action.yaml‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
name: Run CCV Load Test
2+
description: Reusable action for running CCV load tests
3+
4+
inputs:
5+
subtest:
6+
description: 'The subtest to run'
7+
required: true
8+
timeout:
9+
description: 'Test timeout duration'
10+
required: true
11+
ccv-iam-role:
12+
description: 'AWS IAM role for CCV'
13+
required: true
14+
jd-registry:
15+
description: 'JD registry ID'
16+
required: true
17+
jd-image:
18+
description: 'JD Docker image'
19+
required: true
20+
env_name:
21+
description: 'Environment name for CCV'
22+
required: false
23+
default: 'env-cl-16'
24+
25+
26+
runs:
27+
using: composite
28+
steps:
29+
- name: Enable S3 Cache for Self-Hosted Runners
30+
uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
31+
32+
- name: Checkout code
33+
uses: actions/checkout@v5
34+
with:
35+
fetch-depth: 0
36+
37+
- name: Set up Docker Buildx
38+
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
39+
40+
- name: Install Just
41+
uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3
42+
with:
43+
just-version: '1.40.0'
44+
45+
- name: Authenticate to AWS ECR
46+
uses: ./.github/actions/aws-ecr-auth
47+
with:
48+
role-to-assume: ${{ inputs.ccv-iam-role }}
49+
aws-region: us-east-1
50+
registry-type: public
51+
52+
- name: Authenticate to AWS ECR (JD)
53+
uses: ./.github/actions/aws-ecr-auth
54+
with:
55+
role-to-assume: ${{ inputs.ccv-iam-role }}
56+
aws-region: us-west-2
57+
registry-type: private
58+
registries: ${{ inputs.jd-registry }}
59+
60+
- name: Set up Go
61+
uses: actions/setup-go@v6
62+
with:
63+
cache: true
64+
go-version-file: build/devenv/go.mod
65+
cache-dependency-path: build/devenv/go.sum
66+
67+
- name: Download Go dependencies
68+
shell: bash
69+
working-directory: build/devenv
70+
run: go mod download
71+
72+
- name: Checkout Chainlink repository
73+
uses: actions/checkout@v5
74+
with:
75+
repository: smartcontractkit/chainlink
76+
path: chainlink
77+
fetch-depth: 1
78+
79+
- name: Build Docker Images
80+
shell: bash
81+
working-directory: build/devenv
82+
run: just build-docker-dev-ci
83+
env:
84+
DOCKER_BUILDKIT: 1
85+
86+
- name: Run CCV environment
87+
shell: bash
88+
working-directory: build/devenv
89+
env:
90+
JD_IMAGE: ${{ inputs.jd-image }}
91+
run: |
92+
cd cmd/ccv && go install . && cd -
93+
ccv u ${{ inputs.env_name }}.toml && ccv obs up
94+
95+
- name: Run load tests
96+
id: load_test
97+
shell: bash
98+
working-directory: build/devenv/tests/e2e
99+
run: |
100+
set -o pipefail
101+
LOAD_TEST_OUT_FILE=../../${{ inputs.env_name }}-out.toml go test -v -timeout ${{ inputs.timeout }} -count=1 -run 'TestE2ELoad/${{ inputs.subtest }}'
102+
continue-on-error: true
103+
104+
- name: Upload Logs
105+
if: always()
106+
uses: actions/upload-artifact@v4
107+
with:
108+
name: container-logs-${{ inputs.subtest }}
109+
path: build/devenv/tests/e2e/logs
110+
retention-days: 1
111+
112+
- name: Check test results
113+
if: always() && steps.load_test.outcome == 'failure'
114+
shell: bash
115+
run: |
116+
echo "Load tests failed."
117+
exit 1

‎.github/workflows/test-load-nightly.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: CCV Load Test - Scheduled
2+
3+
on:
4+
schedule:
5+
- cron: '0 2 * * *' # Daily at 2 AM UTC
6+
7+
defaults:
8+
run:
9+
working-directory: build/devenv
10+
11+
jobs:
12+
e2e-load-scheduled:
13+
permissions:
14+
id-token: write
15+
contents: read
16+
runs-on: runs-on=${{ github.run_id }}/family=c6i/cpu=32+48/ram=64+96/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
17+
timeout-minutes: 80 # 1 hour 20 minutes
18+
strategy:
19+
fail-fast: false
20+
matrix:
21+
test:
22+
- subtest: rpc_latency
23+
timeout: 1h10m
24+
steps:
25+
- name: Enable S3 Cache for Self-Hosted Runners
26+
uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
27+
28+
- name: Checkout code
29+
uses: actions/checkout@v5
30+
with:
31+
fetch-depth: 0
32+
- name: Run load test
33+
uses: ./.github/actions/run-load-test
34+
with:
35+
subtest: ${{ matrix.test.subtest }}
36+
timeout: ${{ matrix.test.timeout }}
37+
ccv-iam-role: ${{ secrets.CCV_IAM_ROLE }}
38+
jd-registry: ${{ secrets.JD_REGISTRY }}
39+
jd-image: ${{ secrets.JD_IMAGE }}

‎.github/workflows/test-load.yaml‎

Lines changed: 10 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: CCV Clean Load Test
1+
name: CCV Load Test - PR
22

33
on:
44
pull_request:
@@ -12,7 +12,7 @@ concurrency:
1212
cancel-in-progress: true
1313

1414
jobs:
15-
e2e-load:
15+
e2e-load-pr:
1616
permissions:
1717
id-token: write
1818
contents: read
@@ -27,83 +27,16 @@ jobs:
2727
steps:
2828
- name: Enable S3 Cache for Self-Hosted Runners
2929
uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
30-
30+
3131
- name: Checkout code
3232
uses: actions/checkout@v5
3333
with:
3434
fetch-depth: 0
35-
36-
- name: Set up Docker Buildx
37-
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
38-
39-
- name: Install Just
40-
uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3
41-
with:
42-
just-version: '1.40.0'
43-
44-
- name: Authenticate to AWS ECR
45-
uses: ./.github/actions/aws-ecr-auth
46-
with:
47-
role-to-assume: ${{ secrets.CCV_IAM_ROLE }}
48-
aws-region: us-east-1
49-
registry-type: public
50-
51-
- name: Authenticate to AWS ECR (JD)
52-
uses: ./.github/actions/aws-ecr-auth
53-
with:
54-
role-to-assume: ${{ secrets.CCV_IAM_ROLE }}
55-
aws-region: us-west-2
56-
registry-type: private
57-
registries: ${{ secrets.JD_REGISTRY }}
58-
59-
- name: Set up Go
60-
uses: actions/setup-go@v6 # v6
35+
- name: Run load test
36+
uses: ./.github/actions/run-load-test
6137
with:
62-
cache: true
63-
go-version-file: build/devenv/go.mod
64-
cache-dependency-path: build/devenv/go.sum
65-
66-
- name: Download Go dependencies
67-
run: |
68-
go mod download
69-
70-
- name: Checkout Chainlink repository
71-
uses: actions/checkout@v5
72-
with:
73-
repository: smartcontractkit/chainlink
74-
path: chainlink
75-
fetch-depth: 1
76-
77-
- name: Build Docker Images
78-
run: just build-docker-dev-ci
79-
env:
80-
DOCKER_BUILDKIT: 1
81-
82-
- name: Run CCV environment
83-
env:
84-
JD_IMAGE: ${{ secrets.JD_IMAGE }}
85-
run: |
86-
cd cmd/ccv && go install . && cd -
87-
ccv u env-cl-16.toml && ccv obs up
88-
89-
- name: Run load tests
90-
id: load_test
91-
working-directory: build/devenv/tests/e2e
92-
run: |
93-
set -o pipefail
94-
LOAD_TEST_OUT_FILE=../../env-cl-16-out.toml go test -v -timeout ${{ matrix.test.timeout }} -count=1 -run 'TestE2ELoad/${{ matrix.test.subtest }}'
95-
continue-on-error: true
96-
97-
- name: Upload Logs
98-
if: always()
99-
uses: actions/upload-artifact@v4
100-
with:
101-
name: container-logs-${{ matrix.test.subtest }}
102-
path: build/devenv/tests/e2e/logs
103-
retention-days: 1
104-
105-
- name: Check test results
106-
if: always() && steps.load_test.outcome == 'failure'
107-
run: |
108-
echo "Load tests failed."
109-
exit 1
38+
subtest: ${{ matrix.test.subtest }}
39+
timeout: ${{ matrix.test.timeout }}
40+
ccv-iam-role: ${{ secrets.CCV_IAM_ROLE }}
41+
jd-registry: ${{ secrets.JD_REGISTRY }}
42+
jd-image: ${{ secrets.JD_IMAGE }}

‎build/devenv/tests/e2e/load_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -422,15 +422,15 @@ func TestE2ELoad(t *testing.T) {
422422
})
423423

424424
t.Run("rpc latency", func(t *testing.T) {
425-
testDuration := 250 * time.Second
425+
testDuration := 1 * time.Hour
426426
expectedP90Latency := 5 * time.Second
427427
timeoutDuration := time.Duration((testDuration.Seconds()+expectedP90Latency.Seconds())*10) * time.Second
428428
// 400ms latency for any RPC node
429429
pumbaCmd := fmt.Sprintf("netem --tc-image=ghcr.io/alexei-led/pumba-debian-nettools --duration=%s delay --time=400 re2:blockchain-.*", timeoutDuration)
430430
_, err = chaos.ExecPumba(pumbaCmd, 0*time.Second)
431431
require.NoError(t, err)
432432

433-
rps := int64(4)
433+
rps := int64(1)
434434

435435
tc := NewTestingContext(t, ctx, impl, defaultAggregatorClient, indexerClient)
436436
tc.Timeout = timeoutDuration

0 commit comments

Comments
 (0)