Skip to content

E2E Test Automation Enhancements #66

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
chetanbothra wants to merge 29 commits into
base: master
Choose a base branch
from
Open

E2E Test Automation Enhancements #66

Changes from 17 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
8b621d7
Create e2e.yml
chetanbothra May 23, 2025
eb84386
Update config.yml
chetanbothra May 23, 2025
7cdab0f
Update config.yml
chetanbothra May 23, 2025
992b9ea
Update config.yml
chetanbothra May 23, 2025
87c4296
Update config.yml
chetanbothra May 28, 2025
15e7f67
Update config.yml
chetanbothra May 28, 2025
df34a55
Update config.yml
chetanbothra May 28, 2025
cfc1203
Update config.yml
chetanbothra May 28, 2025
5e960c6
Update config.yml
chetanbothra May 28, 2025
875dbd4
Delete .circleci/e2e.yml
chetanbothra May 28, 2025
b79dcad
Update config.yml
chetanbothra May 28, 2025
2044237
fixed report path
chetanbothra Sep 29, 2025
bf25be3
Update config.yml
chetanbothra Sep 30, 2025
d70b10c
Update config.yml
chetanbothra Sep 30, 2025
da1180f
fixed path
chetanbothra Sep 30, 2025
6f82f83
fixed slack
chetanbothra Sep 30, 2025
d77df75
added debugging step
chetanbothra Sep 30, 2025
dbbea83
Merge pull request #71 from etherspot/master
chetanbothra Sep 30, 2025
41ea9d3
Update config.yml
chetanbothra Sep 30, 2025
5c9ffac
Update config.yml
chetanbothra Sep 30, 2025
920223a
Update config.yml
chetanbothra Sep 30, 2025
9c60cc9
Update config.yml
chetanbothra Sep 30, 2025
188c766
Update config.yml
chetanbothra Sep 30, 2025
a508876
Update config.yml
chetanbothra Oct 1, 2025
2bba232
Update config.yml
chetanbothra Oct 1, 2025
f4570ed
Update config.yml
chetanbothra Oct 1, 2025
fca0b7e
Update config.yml
chetanbothra Oct 1, 2025
34c7111
Update config.yml
chetanbothra Oct 1, 2025
1f81caa
Update config.yml
chetanbothra Oct 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 125 additions & 70 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ parameters:
run_e2e_tests:
type: boolean
default: false
test_command:
type: string
default: "npm run test-all-chains"
orbs:
slack: circleci/slack@3.4.2

Expand Down Expand Up @@ -44,6 +47,10 @@ jobs:
# webhook: "${SLACK_WEBHOOK_URL}"

e2e-tests:
parameters:
test_command:
type: string
default: "npm run test-testnet-viem-combined"
working_directory: ~/etherspot-modular-sdk
docker:
- image: cimg/node:20.11.1
Expand Down Expand Up @@ -89,83 +96,115 @@ jobs:
sed -i.bak "s/bundler_api_key/$BUNDLER_API_KEY/g" .env
sed -i.bak "s/data_api_key/$DATA_API_KEY/g" .env
sed -i.bak "s/api_key_arka/$API_KEY_ARKA/g" .env
#sed -i.bak "s/api_key_sessionkey/$API_KEY_SESSIONKEY/g" .env
sed -i.bak "s/api_key_sessionkey/$API_KEY_SESSIONKEY/g" .env
sed -i.bak "s/primary_private_key/$PRIMARY_PRIVATE_KEY/g" .env
sed -i.bak "s/secondary_private_key/$SECONDARY_PRIVATE_KEY/g" .env
sed -i.bak "s/primary_wallet_address/$PRIMARY_WALLET_ADDRESS/g" .env
sed -i.bak "s/secondary_wallet_address/$SECONDARY_WALLET_ADDRESS/g" .env
sed -i.bak "s/paymaster_address/$PAYMASTER_ADDRESS/g" .env
sed -i.bak "s/sponsor_address/$SPONSOR_ADDRESS/g" .env
mv package.json temp.json
jq -r '.dependencies."@etherspot/modular-sdk" |= "file:../etherspot-modular-sdk"' temp.json > package.json
rm temp.json
npm i
npm run test-testnet-viem-combined
<< parameters.test_command >>

- run:
name: Debug aggregated report
command: |
echo "Checking aggregated-report.json..."
cat /home/circleci/e2e-sdk-modular/reports/aggregated-report.json | head -n 50
jq '.stats' reports/aggregated-report.json || echo "No stats found!"

- run:
name: Move test report
command: |
mv /home/circleci/e2e-sdk-modular/mochawesome-report /tmp/
chmod 777 -R /tmp/mochawesome-report
when: always
mkdir -p /tmp/test-report
cp /home/circleci/e2e-sdk-modular/reports/test-report.html /tmp/test-report/
chmod 777 -R /tmp/test-report
when: always

- store_artifacts:
path: /tmp/mochawesome-report
path: /tmp/test-report
when: always

- run:
name: Parse and send test results to Slack
name: Send test results to Slack
when: always
command: |
wget https://output.circle-artifacts.com/output/job/${CIRCLE_WORKFLOW_JOB_ID}/artifacts/${CIRCLE_NODE_INDEX}/tmp/mochawesome-report/mochawesome.json

MOCHAWESOME_JSON_FILE=./mochawesome.json
FAILED_TESTS_FILE=failed-tests.txt
TOTAL_SUITES=$(jq '.stats.suites' $MOCHAWESOME_JSON_FILE)
TOTAL_PASSES=$(jq '.stats.passes' $MOCHAWESOME_JSON_FILE)
TOTAL_PENDING=$(jq '.stats.pending' $MOCHAWESOME_JSON_FILE)
TOTAL_FAILURES=$(jq '.stats.failures' $MOCHAWESOME_JSON_FILE)
START_TIME=$(jq -r '.stats.start' $MOCHAWESOME_JSON_FILE)
END_TIME=$(jq -r '.stats.end' $MOCHAWESOME_JSON_FILE)
DURATION_MS=$(jq -r '.stats.duration' $MOCHAWESOME_JSON_FILE)

START_TIME_FORMATTED=$(date -d "$START_TIME" "+%Y-%m-%d %H:%M:%S")
END_TIME_FORMATTED=$(date -d "$END_TIME" "+%Y-%m-%d %H:%M:%S")
DURATION_MIN=$(awk "BEGIN {printf \"%.2f\",${DURATION_MS}/60000}")


jq -r '.results[] | .suites[] | select(.failures > 0) | {suite: .title, tests: [.tests[] | select(.fail) | .title]} | select(.tests | length > 0) |
"*Suite:* \(.suite)\n*Failing Tests:* \n\(.tests | map("- " + .) | join("\n"))\n"' $MOCHAWESOME_JSON_FILE > $FAILED_TESTS_FILE

if [ -s $FAILED_TESTS_FILE ]; then
MESSAGE=$(cat $FAILED_TESTS_FILE)
SLACK_MESSAGE=":x: *E2E Tests Failed :x: *\n\
*Project:* ${CIRCLE_PROJECT_REPONAME}\n\
*Triggered by:* ${CIRCLE_USERNAME}\n\
*Branch:* ${CIRCLE_BRANCH}\n\
*Commit:* <https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/commit/${CIRCLE_SHA1}|${CIRCLE_SHA1}>\n\n\
*Test Summary:*\n\
*Total Suites:* ${TOTAL_SUITES}\n\
*Passes:* ${TOTAL_PASSES}\n\
*Pending:* ${TOTAL_PENDING}\n\
*Failures:* ${TOTAL_FAILURES}\n\
*Start Time:* ${START_TIME_FORMATTED}\n\
*End Time:* ${END_TIME_FORMATTED}\n\
*Duration:* ${DURATION_MIN} minutes\n\n\
*Failed Tests:*\n${MESSAGE}\n\
*Report:* <https://output.circle-artifacts.com/output/job/${CIRCLE_WORKFLOW_JOB_ID}/artifacts/${CIRCLE_NODE_INDEX}/tmp/mochawesome-report/mochawesome.html|View HTML Report>\n\
*Job:* <https://circleci.com/gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BUILD_NUM}|View Job>\n\
cc: ${TAG_RESPONSIBLE_PEOPLE}"
else
SLACK_MESSAGE=":white_check_mark: All E2E tests passed\n\
SLACK_MESSAGE=":page_facing_up: *E2E Test Report*\n\
*Project:* ${CIRCLE_PROJECT_REPONAME}\n\
*Triggered by:* ${CIRCLE_USERNAME}\n\
*Branch:* ${CIRCLE_BRANCH}\n\
*Commit:* <https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/commit/${CIRCLE_SHA1}|${CIRCLE_SHA1}>\n\
*Job:* <https://circleci.com/gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BUILD_NUM}|View Job>\n\
*Report:* <https://output.circle-artifacts.com/output/job/${CIRCLE_WORKFLOW_JOB_ID}/artifacts/${CIRCLE_NODE_INDEX}/tmp/mochawesome-report/mochawesome.html|View HTML Report>\n\
\n*Test Summary:*\n\
*Total Suites:* ${TOTAL_SUITES}\n\
*Passes:* ${TOTAL_PASSES}\n\
*Pending:* ${TOTAL_PENDING}\n\
*Failures:* ${TOTAL_FAILURES}\n\
*Start Time:* ${START_TIME_FORMATTED}\n\
*End Time:* ${END_TIME_FORMATTED}\n\
*Duration:* ${DURATION_MIN} minutes"
fi
*Report:* <https://output.circle-artifacts.com/output/job/${CIRCLE_WORKFLOW_JOB_ID}/artifacts/${CIRCLE_NODE_INDEX}/home/circleci/e2e-sdk-modular/reports/test-report.html|View HTML Report>"

curl -X POST -H 'Content-type: application/json' \
--data '{"text":"'"$SLACK_MESSAGE"'"}' $E2E_SLACK_WEBHOOK_URL
Copy link

@coderabbitai coderabbitai bot Sep 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Fix artifact URL in Slack notification.

The report URL points to the source location /home/circleci/e2e-sdk-modular/reports/test-report.html, but CircleCI artifacts are served from where they're stored. Since line 115 copies the file to /tmp/test-report/, the URL should reference that path.

Apply this diff:

-                *Report:* <https://output.circle-artifacts.com/output/job/${CIRCLE_WORKFLOW_JOB_ID}/artifacts/${CIRCLE_NODE_INDEX}/home/circleci/e2e-sdk-modular/reports/test-report.html|View HTML Report>"
+                *Report:* <https://output.circle-artifacts.com/output/job/${CIRCLE_WORKFLOW_JOB_ID}/artifacts/${CIRCLE_NODE_INDEX}/tmp/test-report/test-report.html|View HTML Report>"
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
SLACK_MESSAGE=":page_facing_up: *E2E Test Report*\n\
*Project:* ${CIRCLE_PROJECT_REPONAME}\n\
*Triggered by:* ${CIRCLE_USERNAME}\n\
*Branch:* ${CIRCLE_BRANCH}\n\
*Commit:* <https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/commit/${CIRCLE_SHA1}|${CIRCLE_SHA1}>\n\
*Job:* <https://circleci.com/gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BUILD_NUM}|View Job>\n\
*Report:* <https://output.circle-artifacts.com/output/job/${CIRCLE_WORKFLOW_JOB_ID}/artifacts/${CIRCLE_NODE_INDEX}/tmp/mochawesome-report/mochawesome.html|View HTML Report>\n\
\n*Test Summary:*\n\
*Total Suites:* ${TOTAL_SUITES}\n\
*Passes:* ${TOTAL_PASSES}\n\
*Pending:* ${TOTAL_PENDING}\n\
*Failures:* ${TOTAL_FAILURES}\n\
*Start Time:* ${START_TIME_FORMATTED}\n\
*End Time:* ${END_TIME_FORMATTED}\n\
*Duration:* ${DURATION_MIN} minutes"
fi
*Report:* <https://output.circle-artifacts.com/output/job/${CIRCLE_WORKFLOW_JOB_ID}/artifacts/${CIRCLE_NODE_INDEX}/home/circleci/e2e-sdk-modular/reports/test-report.html|View HTML Report>"
curl -X POST -H 'Content-type: application/json' \
--data '{"text":"'"$SLACK_MESSAGE"'"}' $E2E_SLACK_WEBHOOK_URL
SLACK_MESSAGE=":page_facing_up: *E2E Test Report*\n\
*Project:* ${CIRCLE_PROJECT_REPONAME}\n\
*Triggered by:* ${CIRCLE_USERNAME}\n\
*Branch:* ${CIRCLE_BRANCH}\n\
*Commit:* <https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/commit/${CIRCLE_SHA1}|${CIRCLE_SHA1}>\n\
*Job:* <https://circleci.com/gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BUILD_NUM}|View Job>\n\
*Report:* <https://output.circle-artifacts.com/output/job/${CIRCLE_WORKFLOW_JOB_ID}/artifacts/${CIRCLE_NODE_INDEX}/tmp/test-report/test-report.html|View HTML Report>"
curl -X POST -H 'Content-type: application/json' \
--data '{"text":"'"$SLACK_MESSAGE"'"}' $E2E_SLACK_WEBHOOK_URL
🤖 Prompt for AI Agents 
In .circleci/config.yml around lines 127 to 136, the Slack report URL points to
/home/circleci/e2e-sdk-modular/reports/test-report.html but the artifact was
copied to /tmp/test-report/ on line 115; update the Report URL to reference the
correct stored artifact path (/tmp/test-report/test-report.html) in the
SLACK_MESSAGE so the Slack link resolves to the actual CircleCI artifact
location before sending the curl POST.


# - run:
# name: Parse and send test results to Slack
# when: always
# command: |
# wget https://output.circle-artifacts.com/output/job/${CIRCLE_WORKFLOW_JOB_ID}/artifacts/${CIRCLE_NODE_INDEX}/home/circleci/e2e-sdk-modular/reports/test-report.html

# MOCHAWESOME_JSON_FILE=./mochawesome.json
# FAILED_TESTS_FILE=failed-tests.txt
# TOTAL_SUITES=$(jq '.stats.suites' $MOCHAWESOME_JSON_FILE)
# TOTAL_PASSES=$(jq '.stats.passes' $MOCHAWESOME_JSON_FILE)
# TOTAL_PENDING=$(jq '.stats.pending' $MOCHAWESOME_JSON_FILE)
# TOTAL_FAILURES=$(jq '.stats.failures' $MOCHAWESOME_JSON_FILE)
# START_TIME=$(jq -r '.stats.start' $MOCHAWESOME_JSON_FILE)
# END_TIME=$(jq -r '.stats.end' $MOCHAWESOME_JSON_FILE)
# DURATION_MS=$(jq -r '.stats.duration' $MOCHAWESOME_JSON_FILE)

# START_TIME_FORMATTED=$(date -d "$START_TIME" "+%Y-%m-%d %H:%M:%S")
# END_TIME_FORMATTED=$(date -d "$END_TIME" "+%Y-%m-%d %H:%M:%S")
# DURATION_MIN=$(awk "BEGIN {printf \"%.2f\",${DURATION_MS}/60000}")

curl -X POST -H 'Content-type: application/json' --data '{"text":"'"$SLACK_MESSAGE"'"}' $E2E_SLACK_WEBHOOK_URL

# jq -r '.results[] | .suites[] | select(.failures > 0) | {suite: .title, tests: [.tests[] | select(.fail) | .title]} | select(.tests | length > 0) |
# "*Suite:* \(.suite)\n*Failing Tests:* \n\(.tests | map("- " + .) | join("\n"))\n"' $MOCHAWESOME_JSON_FILE > $FAILED_TESTS_FILE

# if [ -s $FAILED_TESTS_FILE ]; then
# MESSAGE=$(cat $FAILED_TESTS_FILE)
# SLACK_MESSAGE=":x: *E2E Tests Failed :x: *\n\
# *Project:* ${CIRCLE_PROJECT_REPONAME}\n\
# *Triggered by:* ${CIRCLE_USERNAME}\n\
# *Branch:* ${CIRCLE_BRANCH}\n\
# *Commit:* <https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/commit/${CIRCLE_SHA1}|${CIRCLE_SHA1}>\n\n\
# *Test Summary:*\n\
# *Total Suites:* ${TOTAL_SUITES}\n\
# *Passes:* ${TOTAL_PASSES}\n\
# *Pending:* ${TOTAL_PENDING}\n\
# *Failures:* ${TOTAL_FAILURES}\n\
# *Start Time:* ${START_TIME_FORMATTED}\n\
# *End Time:* ${END_TIME_FORMATTED}\n\
# *Duration:* ${DURATION_MIN} minutes\n\n\
# *Failed Tests:*\n${MESSAGE}\n\
# *Report:* <https://output.circle-artifacts.com/output/job/${CIRCLE_WORKFLOW_JOB_ID}/artifacts/${CIRCLE_NODE_INDEX}/tmp/mochawesome-report/mochawesome.html|View HTML Report>\n\
# *Job:* <https://circleci.com/gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BUILD_NUM}|View Job>\n\
# cc: ${TAG_RESPONSIBLE_PEOPLE}"
# else
# SLACK_MESSAGE=":white_check_mark: All E2E tests passed\n\
# *Project:* ${CIRCLE_PROJECT_REPONAME}\n\
# *Triggered by:* ${CIRCLE_USERNAME}\n\
# *Branch:* ${CIRCLE_BRANCH}\n\
# *Commit:* <https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/commit/${CIRCLE_SHA1}|${CIRCLE_SHA1}>\n\
# *Job:* <https://circleci.com/gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BUILD_NUM}|View Job>\n\
# *Report:* <https://output.circle-artifacts.com/output/job/${CIRCLE_WORKFLOW_JOB_ID}/artifacts/${CIRCLE_NODE_INDEX}/tmp/mochawesome-report/mochawesome.html|View HTML Report>\n\
# \n*Test Summary:*\n\
# *Total Suites:* ${TOTAL_SUITES}\n\
# *Passes:* ${TOTAL_PASSES}\n\
# *Pending:* ${TOTAL_PENDING}\n\
# *Failures:* ${TOTAL_FAILURES}\n\
# *Start Time:* ${START_TIME_FORMATTED}\n\
# *End Time:* ${END_TIME_FORMATTED}\n\
# *Duration:* ${DURATION_MIN} minutes"
# fi

# curl -X POST -H 'Content-type: application/json' --data '{"text":"'"$SLACK_MESSAGE"'"}' $E2E_SLACK_WEBHOOK_URL
publish-npm-package:
working_directory: ~/etherspot-modular-sdk
docker:
Expand Down Expand Up @@ -216,32 +255,48 @@ jobs:
ghr -t "${GITHUB_TOKEN}" -u "${CIRCLE_PROJECT_USERNAME}" -r "${CIRCLE_PROJECT_REPONAME}" -c "${CIRCLE_SHA1}" "$PACKAGE_VERSION"

workflows:
version: 2.1
install_and_publish:
run-e2e-only:
when: << pipeline.parameters.run_e2e_tests >>
jobs:
- e2e-tests:
context: general-vars
test_command: "npm run test-all-chains"

install-and-publish:
when:
not: << pipeline.parameters.run_e2e_tests >>
jobs:
- install:
context: general-vars
- e2e-tests:
name: testnet-e2e-tests
context: general-vars
requires:
- install
# filters:
# branches:
# only:
# - master
test_command: "npm run test-testnet-viem-combined"
filters:
branches:
only:
- develop
- e2e-tests:
name: mainnet-e2e-tests
context: general-vars
test_command: "npm run test-mainnet-viem-combined"
filters:
branches:
only:
- master
- publish-npm-package:
context: general-vars
requires:
- e2e-tests
- mainnet-e2e-tests
filters:
branches:
only:
- master
- master
- publish-github-release:
context: general-vars
requires:
- publish-npm-package
filters:
branches:
only:
- master
- master
Loading