Skip to content

Commit 8850b4b

Browse files
jordanstephensjordanstephens
committed
run loadtests for changed samples
1 parent 15377a9 commit 8850b4b

File tree

14 files changed

+1762
-0
lines changed

14 files changed

+1762
-0
lines changed

‎.github/workflows/deploy-changed-samples.yml‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
name: Deploy Changed Samples
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- 'samples/**'
7+
8+
permissions:
9+
contents: read
10+
id-token: write
11+
12+
jobs:
13+
deploy_changed_samples:
14+
runs-on: ubuntu-latest
15+
concurrency:
16+
group: deploy_samples-group-${{ github.head_ref }}
17+
cancel-in-progress: true
18+
environment: deploy-changed-samples
19+
timeout-minutes: 90
20+
steps:
21+
- name: Checkout code
22+
uses: actions/checkout@v4
23+
with:
24+
fetch-depth: 1
25+
26+
- name: Fetch main branch
27+
run: git fetch origin main:main # Fetch the main branch reference for comparison
28+
29+
- name: Identify changed samples
30+
run: |
31+
git diff --name-only HEAD main | grep '^samples/' | awk -F'/' '{print $1"/"$2}' | sort | uniq > changed_samples.txt
32+
if [ ! -s changed_samples.txt ]; then
33+
echo "No samples have changed. Exiting..."
34+
exit 0
35+
fi
36+
37+
- name: Configure AWS Credentials for CI
38+
id: creds
39+
uses: aws-actions/configure-aws-credentials@v4
40+
with:
41+
aws-region: us-west-2
42+
output-credentials: true
43+
role-to-assume: arn:aws:iam::488659951590:role/ci-role-d4fe904 # ciRoleArn from defang-io/infrastructure stack
44+
45+
- name: Configure AWS Credentials for Playground
46+
uses: aws-actions/configure-aws-credentials@v4
47+
with:
48+
aws-region: us-west-2
49+
role-duration-seconds: 1200
50+
role-chaining: true
51+
role-to-assume: arn:aws:iam::426819183542:role/admin # adminUserRoleArn from defang-io/bootstrap stack
52+
53+
- name: Install Golang
54+
uses: actions/setup-go@v5
55+
with:
56+
go-version-file: defang-mvp/tools/testing/go.mod
57+
cache-dependency-path: |
58+
defang-mvp/tools/testing/go.sum
59+
defang/src/go.sum
60+
61+
- name: Install latest defang cli
62+
run: |
63+
go install github.com/DefangLabs/defang/src/cmd/cli@latest
64+
which cli
65+
mv $(which cli) $(dirname $(which cli))/defang
66+
defang --version
67+
68+
- name: Build the test tool using Go
69+
run: |
70+
cd tools/testing/
71+
go mod tidy
72+
go build ./cmd/loadtest
73+
74+
- name: Run tests
75+
id: run-tests
76+
shell: bash # implies set -o pipefail, so pipe below will keep the exit code from loadtest
77+
env:
78+
TEST_AWS_ACCESS_KEY: ${{ secrets.TEST_AWS_ACCESS_KEY }}
79+
TEST_AWS_SECRET_KEY: ${{ secrets.TEST_AWS_SECRET_KEY }}
80+
TEST_BOARD_PASSWORD: ${{ secrets.TEST_BOARD_PASSWORD }}
81+
TEST_DATABASE_HOST: ${{ secrets.TEST_DATABASE_HOST }}
82+
TEST_DATABASE_NAME: ${{ secrets.TEST_DATABASE_NAME }}
83+
TEST_DATABASE_PASSWORD: ${{ secrets.TEST_DATABASE_PASSWORD }}
84+
TEST_DATABASE_URL: ${{ secrets.TEST_DATABASE_URL }}
85+
TEST_DATABASE_USERNAME: ${{ secrets.TEST_DATABASE_USERNAME }}
86+
TEST_HASURA_GRAPHQL_ADMIN_SECRET: ${{ secrets.TEST_HASURA_GRAPHQL_ADMIN_SECRET }}
87+
TEST_HASURA_GRAPHQL_DATABASE_URL: ${{ secrets.TEST_HASURA_GRAPHQL_DATABASE_URL }}
88+
TEST_HF_TOKEN: ${{ secrets.TEST_HF_TOKEN }}
89+
TEST_MB_DB_DBNAME: ${{ secrets.TEST_MB_DB_DBNAME }}
90+
TEST_MB_DB_HOST: ${{ secrets.TEST_MB_DB_HOST }}
91+
TEST_MB_DB_PASS: ${{ secrets.TEST_MB_DB_PASS }}
92+
TEST_MB_DB_PORT: ${{ secrets.TEST_MB_DB_PORT }}
93+
TEST_MB_DB_USER: ${{ secrets.TEST_MB_DB_USER }}
94+
TEST_NC_DB: ${{ secrets.TEST_NC_DB }}
95+
TEST_NC_S3_ENDPOINT: ${{ secrets.TEST_NC_S3_ENDPOINT }}
96+
TEST_NC_S3_BUCKET_NAME: ${{ secrets.TEST_NC_S3_BUCKET_NAME }}
97+
TEST_NC_S3_REGION: ${{ secrets.TEST_NC_S3_REGION }}
98+
TEST_NC_S3_ACCESS_KEY: ${{ secrets.TEST_NC_S3_ACCESS_KEY }}
99+
TEST_NC_S3_ACCESS_SECRET: ${{ secrets.TEST_NC_S3_ACCESS_SECRET }}
100+
TEST_OPENAI_KEY: ${{ secrets.TEST_OPENAI_KEY }}
101+
TEST_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }}
102+
TEST_QUEUE: ${{ secrets.TEST_QUEUE }}
103+
TEST_SECRET_KEY_BASE: ${{ secrets.TEST_SECRET_KEY_BASE }}
104+
TEST_SESSION_SECRET: ${{ secrets.TEST_SESSION_SECRET }}
105+
TEST_SLACK_CHANNEL_ID: ${{ secrets.TEST_SLACK_CHANNEL_ID }}
106+
TEST_SLACK_TOKEN: ${{ secrets.TEST_SLACK_TOKEN }}
107+
run: |
108+
SAMPLES=$(IFS=,; echo "$(cat changed_samples.txt)")
109+
echo "Running tests for samples: $SAMPLES"
110+
111+
cd defang-mvp/tools/testing/
112+
mkdir output
113+
# concurrency is set to 10 to avoid exhausting our fargate vCPU limits when running
114+
# kaniko builds, which consume 4 vCPUs each. the limit is currently set to 60--6.5 of
115+
# which are used to run the staging stack. So floor((60-6.5)/4) = 13 is our upper limit
116+
./loadtest -c fabric-staging.defang.dev:443 --timeout=15m --concurrency=10 -s $SAMPLES -o output --markdown=true | tee output/summary.log | grep -v '^\s*[-*]' # removes load sample log lines
117+
118+
- name: Upload Output as Artifact
119+
uses: actions/upload-artifact@v4
120+
if: success() || steps.run-tests.outcome == 'failure' # Always upload result unless cancelled
121+
with:
122+
name: program-output
123+
path: defang-mvp/tools/testing/output/**

‎tools/testing/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
/loadtest
2+
/runtest
3+
/out

‎tools/testing/cleanup/cleanup.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package cleanup
2+
3+
import (
4+
"errors"
5+
"sync"
6+
)
7+
8+
type CleanUpFunc func() error
9+
10+
var defaultCleanUp = &CleanUp{}
11+
12+
func Add(fn CleanUpFunc) {
13+
defaultCleanUp.Add(fn)
14+
}
15+
16+
func Run() error {
17+
return defaultCleanUp.Run()
18+
}
19+
20+
type CleanUp struct {
21+
cleanups []CleanUpFunc
22+
lock sync.Mutex
23+
}
24+
25+
func (c *CleanUp) Add(fn CleanUpFunc) {
26+
c.lock.Lock()
27+
defer c.lock.Unlock()
28+
c.cleanups = append(c.cleanups, fn)
29+
}
30+
31+
func (c *CleanUp) Run() error {
32+
c.lock.Lock()
33+
defer c.lock.Unlock()
34+
var errs []error
35+
// Run the clean up functions in reverse order
36+
for i := len(c.cleanups) - 1; i >= 0; i-- {
37+
if err := c.cleanups[i](); err != nil {
38+
errs = append(errs, err)
39+
}
40+
}
41+
return errors.Join(errs...)
42+
}

0 commit comments

Comments
 (0)