Skip to content

Commit 3c36d45

Browse files
josecorellajosecorella
authored
chore: upload performance benchmarks to cloudwatch (#846)
1 parent 57ba773 commit 3c36d45

File tree

6 files changed

+559
-0
lines changed

6 files changed

+559
-0
lines changed

‎.github/workflows/performance-benchmarks-go.yml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# This workflow performs benchmark testing in Go.
2+
name: Performance Benchmarks Go
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
dafny:
8+
description: "The Dafny version to run"
9+
required: true
10+
type: string
11+
regenerate-code:
12+
description: "Regenerate code using smithy-dafny"
13+
required: false
14+
default: false
15+
type: boolean
16+
17+
jobs:
18+
testGo:
19+
strategy:
20+
fail-fast: false
21+
matrix:
22+
library: [AwsEncryptionSDK]
23+
benchmark-dir: [esdk-performance-testing]
24+
go-version: ["1.23", "1.24", "1.25"]
25+
os: [
26+
# Sed script doesn't work properly on windows
27+
# windows-latest,
28+
ubuntu-22.04,
29+
# TODO: Re-enable macOS after https://t.corp.amazon.com/P205755286
30+
# macos-15,
31+
]
32+
runs-on: ${{ matrix.os }}
33+
permissions:
34+
id-token: write
35+
contents: read
36+
steps:
37+
- name: Support longpaths on Git checkout
38+
run: |
39+
git config --global core.longpaths true
40+
- uses: actions/checkout@v5
41+
- name: Init Submodules
42+
shell: bash
43+
run: |
44+
git submodule update --init libraries
45+
git submodule update --init --recursive mpl
46+
47+
- name: Configure AWS Credentials
48+
uses: aws-actions/configure-aws-credentials@v5
49+
with:
50+
aws-region: us-west-2
51+
role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Dafny-Role-us-west-2
52+
role-session-name: GoTests
53+
54+
- name: Setup Dafny
55+
uses: ./mpl/.github/actions/setup_dafny
56+
with:
57+
dafny-version: ${{ inputs.dafny }}
58+
59+
- name: Install Smithy-Dafny codegen dependencies
60+
uses: ./.github/actions/install_smithy_dafny_codegen_dependencies
61+
62+
- name: Setup NASM for Windows (aws-lc-sys)
63+
if: matrix.os == 'windows-latest'
64+
uses: ilammy/setup-nasm@v1
65+
66+
- name: Install Go
67+
uses: actions/setup-go@v6
68+
with:
69+
go-version: ${{ matrix.go-version }}
70+
71+
- name: Compile ${{ matrix.library }} implementation
72+
shell: bash
73+
working-directory: ${{ matrix.library }}
74+
run: |
75+
# This works because `node` is installed by default on GHA runners
76+
CORES=$(node -e 'console.log(os.cpus().length)')
77+
make transpile_go CORES=$CORES
78+
79+
- name: Run Performance Benchmarks - Quick Mode
80+
shell: bash
81+
working-directory: ./${{matrix.benchmark-dir}}/benchmarks/go
82+
run: |
83+
go run . --config ../../config/test-scenarios.yaml --quick
84+
85+
- name: Parse and Format Logs
86+
working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/
87+
run: |
88+
LOG_FILE="go_results.json"
89+
UPLOAD_FILE="cloudwatch_logs.json"
90+
TIMESTAMP=$(date +%s%3N)
91+
jq -c --arg ts "$(date +%s)000" '[.results[] as $result | .metadata as $meta | {timestamp: ($ts | tonumber), message: ({metadata: $meta, result: $result} | tostring)}]' $LOG_FILE > $UPLOAD_FILE
92+
93+
- name: Upload logs to CloudWatch
94+
working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/
95+
run: |
96+
LOG_FILE="cloudwatch_logs.json"
97+
LOG_GROUP="aws-esdk-performance-benchmarks"
98+
LOG_STREAM="go/${{matrix.go-version}}/quick_benchmarks/${{ github.workflow }}"
99+
100+
# Create log stream (ignore if exists)
101+
aws logs create-log-stream \
102+
--log-group-name "$LOG_GROUP" \
103+
--log-stream-name "$LOG_STREAM" 2>/dev/null || true
104+
105+
aws logs put-log-events \
106+
--log-group-name "$LOG_GROUP" \
107+
--log-stream-name "$LOG_STREAM" \
108+
--log-events file://$LOG_FILE

‎.github/workflows/performance-benchmarks-java.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# This workflow performs benchmark testing in Java.
2+
name: Performance Benchmarks Go
3+
4+
on:
5+
workflow_call:
6+
7+
jobs:
8+
testJava:
9+
strategy:
10+
fail-fast: false
11+
matrix:
12+
benchmark-dir: [esdk-performance-testing]
13+
java: [17]
14+
os: [
15+
# Sed script doesn't work properly on windows
16+
# windows-latest,
17+
ubuntu-22.04,
18+
# TODO: Re-enable macOS after https://t.corp.amazon.com/P205755286
19+
# macos-15,
20+
]
21+
runs-on: ${{ matrix.os }}
22+
permissions:
23+
id-token: write
24+
contents: read
25+
steps:
26+
- name: Support longpaths on Git checkout
27+
run: |
28+
git config --global core.longpaths true
29+
- uses: actions/checkout@v5
30+
- name: Init Submodules
31+
shell: bash
32+
run: |
33+
git submodule update --init libraries
34+
git submodule update --init --recursive mpl
35+
36+
- name: Configure AWS Credentials
37+
uses: aws-actions/configure-aws-credentials@v5
38+
with:
39+
aws-region: us-west-2
40+
role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Dafny-Role-us-west-2
41+
role-session-name: GoTests
42+
43+
- name: Setup Java Corretto ${{matrix.java}}
44+
uses: actions/setup-java@v5
45+
with:
46+
distribution: "corretto"
47+
java-version: ${{matrix.java}}
48+
49+
- name: Run Performance Benchmarks - Quick Mode
50+
shell: bash
51+
working-directory: ./${{matrix.benchmark-dir}}/benchmarks/java
52+
run: |
53+
mvn clean compile
54+
mvn exec:java -Dexec.mainClass="com.amazon.esdk.benchmark.Program" -Dexec.args="--quick"
55+
56+
- name: Parse and Format Logs
57+
working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/
58+
run: |
59+
LOG_FILE="java_results.json"
60+
UPLOAD_FILE="cloudwatch_logs.json"
61+
TIMESTAMP=$(date +%s%3N)
62+
jq -c --arg ts "$(date +%s)000" '[.results[] as $result | .metadata as $meta | {timestamp: ($ts | tonumber), message: ({metadata: $meta, result: $result} | tostring)}]' $LOG_FILE > $UPLOAD_FILE
63+
64+
- name: Upload logs to CloudWatch
65+
working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/
66+
run: |
67+
LOG_FILE="cloudwatch_logs.json"
68+
LOG_GROUP="aws-esdk-performance-benchmarks"
69+
LOG_STREAM="java/corretto/${{matrix.java}}/quick_benchmarks/${{ github.workflow }}"
70+
71+
# Create log stream (ignore if exists)
72+
aws logs create-log-stream \
73+
--log-group-name "$LOG_GROUP" \
74+
--log-stream-name "$LOG_STREAM" 2>/dev/null || true
75+
76+
aws logs put-log-events \
77+
--log-group-name "$LOG_GROUP" \
78+
--log-stream-name "$LOG_STREAM" \
79+
--log-events file://$LOG_FILE

‎.github/workflows/performance-benchmarks-net.yml‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
# This workflow performs benchmark testing in Net.
2+
name: Performance Benchmarks Net
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
dafny:
8+
description: "The Dafny version to run"
9+
required: true
10+
type: string
11+
regenerate-code:
12+
description: "Regenerate code using smithy-dafny"
13+
required: false
14+
default: false
15+
type: boolean
16+
17+
jobs:
18+
testNet:
19+
strategy:
20+
fail-fast: false
21+
matrix:
22+
library: [AwsEncryptionSDK]
23+
benchmark-dir: [esdk-performance-testing]
24+
os: [
25+
ubuntu-22.04,
26+
# TODO: Re-enable macOS after https://t.corp.amazon.com/P205755286
27+
# macos-15,
28+
]
29+
runs-on: ${{ matrix.os }}
30+
permissions:
31+
id-token: write
32+
contents: read
33+
env:
34+
DOTNET_CLI_TELEMETRY_OPTOUT: 1
35+
DOTNET_NOLOGO: 1
36+
steps:
37+
- name: Support longpaths on Git checkout
38+
run: |
39+
git config --global core.longpaths true
40+
- uses: actions/checkout@v5
41+
- name: Init Submodules
42+
shell: bash
43+
run: |
44+
git submodule update --init libraries
45+
git submodule update --init --recursive mpl
46+
47+
- name: Configure AWS Credentials
48+
uses: aws-actions/configure-aws-credentials@v5
49+
with:
50+
aws-region: us-west-2
51+
role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Dafny-Role-us-west-2
52+
role-session-name: NetTests
53+
54+
- name: Setup .NET Core SDK 6
55+
uses: actions/setup-dotnet@v5
56+
with:
57+
dotnet-version: "6.0.x"
58+
59+
- name: Setup Dafny
60+
uses: ./mpl/.github/actions/setup_dafny
61+
with:
62+
dafny-version: ${{ inputs.dafny }}
63+
64+
- name: Install Smithy-Dafny codegen dependencies
65+
uses: ./.github/actions/install_smithy_dafny_codegen_dependencies
66+
67+
- name: Regenerate code using smithy-dafny if necessary
68+
if: ${{ inputs.regenerate-code }}
69+
uses: ./.github/actions/polymorph_codegen
70+
with:
71+
dafny: ${{ env.DAFNY_VERSION }}
72+
library: ${{ matrix.library }}
73+
diff-generated-code: false
74+
update-and-regenerate-mpl: true
75+
76+
- name: Download Dependencies
77+
working-directory: ${{ matrix.library }}
78+
run: make setup_net
79+
80+
- name: Compile ${{ matrix.library }} implementation
81+
shell: bash
82+
working-directory: ${{ matrix.library }}
83+
run: |
84+
# This works because `node` is installed by default on GHA runners
85+
CORES=$(node -e 'console.log(os.cpus().length)')
86+
make transpile_net CORES=$CORES
87+
88+
- name: Run Performance Benchmarks - Quick Mode
89+
shell: bash
90+
working-directory: ./${{matrix.benchmark-dir}}/benchmarks/net
91+
run: |
92+
dotnet build --configuration Release
93+
dotnet run --configuration Release -- --quick
94+
95+
- name: Parse and Format Logs
96+
working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/
97+
run: |
98+
LOG_FILE="net_results.json"
99+
UPLOAD_FILE="cloudwatch_logs.json"
100+
TIMESTAMP=$(date +%s%3N)
101+
jq -c --arg ts "$(date +%s)000" '[.results[] as $result | .metadata as $meta | {timestamp: ($ts | tonumber), message: ({metadata: $meta, result: $result} | tostring)}]' $LOG_FILE > $UPLOAD_FILE
102+
103+
- name: Upload logs to CloudWatch
104+
working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/
105+
run: |
106+
LOG_FILE="cloudwatch_logs.json"
107+
LOG_GROUP="aws-esdk-performance-benchmarks"
108+
LOG_STREAM="net/${{matrix.os}}/quick_benchmarks/${{ github.workflow }}"
109+
110+
# Create log stream (ignore if exists)
111+
aws logs create-log-stream \
112+
--log-group-name "$LOG_GROUP" \
113+
--log-stream-name "$LOG_STREAM" 2>/dev/null || true
114+
115+
aws logs put-log-events \
116+
--log-group-name "$LOG_GROUP" \
117+
--log-stream-name "$LOG_STREAM" \
118+
--log-events file://$LOG_FILE

‎.github/workflows/performance-benchmarks-python.yml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# This workflow performs benchmark testing in Java.
2+
name: Performance Benchmarks Go
3+
4+
on:
5+
workflow_call:
6+
7+
jobs:
8+
testPython:
9+
strategy:
10+
fail-fast: false
11+
matrix:
12+
benchmark-dir: [esdk-performance-testing]
13+
python-version: ["3.11", "3.13"]
14+
os: [ubuntu-22.04, macos-14]
15+
runs-on: ${{ matrix.os }}
16+
permissions:
17+
id-token: write
18+
contents: read
19+
steps:
20+
- name: Support longpaths on Git checkout
21+
run: |
22+
git config --global core.longpaths true
23+
- uses: actions/checkout@v5
24+
- name: Init Submodules
25+
shell: bash
26+
run: |
27+
git submodule update --init libraries
28+
git submodule update --init --recursive mpl
29+
30+
- name: Configure AWS Credentials
31+
uses: aws-actions/configure-aws-credentials@v5
32+
with:
33+
aws-region: us-west-2
34+
role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Dafny-Role-us-west-2
35+
role-session-name: GoTests
36+
37+
- name: Setup Python ${{ matrix.python-version }} for running tests
38+
uses: actions/setup-python@v6
39+
with:
40+
python-version: ${{ matrix.python-version }}
41+
architecture: x64
42+
43+
- name: Run Performance Benchmarks - Quick Mode
44+
shell: bash
45+
working-directory: ./${{matrix.benchmark-dir}}/benchmarks/python
46+
run: |
47+
pip install -r requirements.txt
48+
python esdk_benchmark.py --quick
49+
50+
- name: Parse and Format Logs
51+
working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/
52+
run: |
53+
LOG_FILE="python_results.json"
54+
UPLOAD_FILE="cloudwatch_logs.json"
55+
TIMESTAMP=$(date +%s%3N)
56+
jq -c --arg ts "$(date +%s)000" '[.results[] as $result | .metadata as $meta | {timestamp: ($ts | tonumber), message: ({metadata: $meta, result: $result} | tostring)}]' $LOG_FILE > $UPLOAD_FILE
57+
58+
- name: Upload logs to CloudWatch
59+
working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/
60+
run: |
61+
LOG_FILE="cloudwatch_logs.json"
62+
LOG_GROUP="aws-esdk-performance-benchmarks"
63+
LOG_STREAM="python/${{matrix.python-version}}/quick_benchmarks/${{ github.workflow }}"
64+
65+
# Create log stream (ignore if exists)
66+
aws logs create-log-stream \
67+
--log-group-name "$LOG_GROUP" \
68+
--log-stream-name "$LOG_STREAM" 2>/dev/null || true
69+
70+
aws logs put-log-events \
71+
--log-group-name "$LOG_GROUP" \
72+
--log-stream-name "$LOG_STREAM" \
73+
--log-events file://$LOG_FILE

0 commit comments

Comments
 (0)