Skip to content

Commit 4e5ee78

Browse files
itaybreitaybre
committed
Restore pending tests
1 parent a33b86b commit 4e5ee78

22 files changed

+1754
-0
lines changed

‎.github/workflows/api-stability.yml

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
name: API Stability Check
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- "Sources/**"
7+
- "test-server/**"
8+
- ".github/workflows/api-stability.yml"
9+
- Sentry.xcworkspace/**
10+
- Sentry.xcodeproj/**
11+
- "Package.swift"
12+
- "scripts/build-xcframework-local.sh"
13+
- "scripts/update-api.sh"
14+
15+
jobs:
16+
api-stability:
17+
runs-on: macos-15
18+
strategy:
19+
matrix:
20+
version: [default, v9]
21+
steps:
22+
- name: Checkout code
23+
uses: actions/checkout@v4
24+
with:
25+
fetch-depth: 0
26+
- run: ./scripts/ci-select-xcode.sh 16.4
27+
28+
- name: Generate HEAD SDK
29+
run: |
30+
if [ "${{ matrix.version }}" = "v9" ]; then
31+
mv sdk_api_v9.json sdk_api_base.json
32+
make generate-public-api CONFIG=V9
33+
mv sdk_api_v9.json sdk_api.json
34+
else
35+
mv sdk_api.json sdk_api_base.json
36+
make generate-public-api
37+
fi
38+
39+
- name: Diagnose breaking changes
40+
run: |
41+
if diff -q "sdk_api_base.json" "sdk_api.json" > /dev/null; then
42+
echo "No API changes detected for ${{ matrix.version }} version."
43+
else
44+
echo "❌ Public API changes are detected for ${{ matrix.version }} version. If they're intended run "make generate-public-api" and commit the changes."
45+
diff "sdk_api_base.json" "sdk_api.json" || true
46+
xcrun --sdk iphoneos swift-api-digester \
47+
-diagnose-sdk \
48+
-o result.json \
49+
-input-paths sdk_api_base.json \
50+
-input-paths sdk_api.json \
51+
-json \
52+
-v
53+
cat result.json
54+
exit 1
55+
fi

‎.github/workflows/auto-update-tools.yml

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# This workflow is used to update the custom tooling versions for the project.
2+
#
3+
# We prefer to use Dependabot to update external dependencies, but at this time it does not include Homebrew as a supported package manager (https://docs.github.com/en/code-security/dependabot/ecosystems-supported-by-dependabot/supported-ecosystems-and-repositories).
4+
# Furthermore, neither `swiftlint` nor `clang-format` are listed as dependencies in our repository, therefore also not picked up by Dependabot.
5+
#
6+
# Therefore we are using a custom workflow to update relevant files and open a pull request with the changes.
7+
8+
name: "Automation: Update tooling versions"
9+
10+
on:
11+
schedule:
12+
- cron: "0 0 * * *"
13+
workflow_dispatch:
14+
pull_request:
15+
paths:
16+
- ".github/workflows/auto-update-tools.yml"
17+
- "Brewfile*"
18+
- "Makefile"
19+
- "scripts/.clang-format-version"
20+
- "scripts/.swiftlint-version"
21+
- ".pre-commit-config.yaml"
22+
23+
# Permissions configuration:
24+
# - 'contents: write' is required to allow the workflow to commit changes to the repository
25+
# when updating the tooling version files and creating branches for pull requests.
26+
# - 'pull-requests: write' is required to allow the workflow to create pull requests
27+
# using the peter-evans/create-pull-request action when tooling version updates are available.
28+
permissions:
29+
contents: write
30+
pull-requests: write
31+
32+
# Concurrency configuration:
33+
# - We use a named concurrency group to prevent multiple instances of this workflow from running
34+
# simultaneously, which could lead to race conditions when creating branches and pull requests.
35+
# Since this workflow modifies version files and creates PRs, concurrent runs could interfere
36+
# with each other, resulting in conflicting branches or duplicate PRs.
37+
# - We enable cancellation of in-progress runs because only the most recent run matters for
38+
# version updates. There's no value in completing outdated runs, especially for scheduled
39+
# workflows that might queue up overnight. This approach conserves GitHub Actions minutes
40+
# and ensures we're always working with the latest repository state.
41+
concurrency:
42+
group: "auto-update-tools"
43+
cancel-in-progress: true
44+
45+
jobs:
46+
auto-update-tools:
47+
runs-on: macos-15
48+
steps:
49+
- name: Checkout Repository
50+
uses: actions/checkout@v4
51+
52+
- name: Update Homebrew
53+
run: brew update
54+
55+
- name: Install Tools
56+
run: make init
57+
58+
- name: Update tooling versions
59+
run: make update-versions
60+
61+
- name: Check tooling versions
62+
run: make check-versions
63+
64+
- name: Print git status and changes
65+
run: |
66+
git status
67+
git diff HEAD
68+
69+
- name: Create pull request for clang-format version
70+
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7.0.8
71+
if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
72+
with:
73+
add-paths: scripts/.clang-format-version
74+
branch: github-actions/auto-update-tools-clang-format
75+
commit-message: "chore(deps): Update clang-format version"
76+
delete-branch: true
77+
title: "chore(deps): Update clang-format version"
78+
sign-commits: true
79+
base: main
80+
81+
- name: Create pull request for swiftlint version
82+
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7.0.8
83+
if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
84+
with:
85+
add-paths: scripts/.swiftlint-version
86+
branch: github-actions/auto-update-tools-swiftlint
87+
commit-message: "chore(deps): Update swiftlint version"
88+
delete-branch: true
89+
title: "chore(deps): Update swiftlint version"
90+
sign-commits: true
91+
base: main

‎.github/workflows/benchmarking.yml

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
name: Benchmarking
2+
on:
3+
push:
4+
branches:
5+
- main
6+
7+
pull_request:
8+
paths:
9+
# test changes to Sentry SDK sources
10+
- "Sources/**"
11+
12+
# test changes to benchmarking implementation
13+
- "Samples/iOS-Swift/**"
14+
- ".github/workflows/benchmarking.yml"
15+
- ".sauce/benchmarking-config.yml"
16+
- "fastlane/**"
17+
- "scripts/ci-select-xcode.sh"
18+
- "Samples/iOS-Swift/iOS-Swift.yml"
19+
- "Samples/iOS-Swift/iOS-Swift.xcconfig"
20+
- "Samples/iOS-Swift/iOS-SwiftClilp.xcconfig"
21+
- "Samples/iOS-Swift/iOS-Benchmarking.xcconfig"
22+
- "scripts/build-xcframework-slice.sh"
23+
- "scripts/assemble-xcframework.sh"
24+
- ".github/workflows/build-xcframework-variant-slices.yml"
25+
- ".github/workflows/assemble-xcframework-variant.yml"
26+
27+
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
28+
concurrency:
29+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
30+
cancel-in-progress: true
31+
32+
jobs:
33+
build-benchmark-test-target:
34+
name: Build app and test runner
35+
runs-on: macos-13
36+
steps:
37+
- uses: actions/checkout@v4
38+
- run: ./scripts/ci-select-xcode.sh 15.2
39+
- uses: ruby/setup-ruby@v1
40+
with:
41+
bundler-cache: true
42+
- run: make init-ci-build
43+
- run: make xcode-ci
44+
- name: Install SentryCli
45+
run: brew install getsentry/tools/sentry-cli
46+
- name: Cache iOS-Swift App and dSYM build products
47+
id: ios-swift-cache
48+
uses: actions/cache@v4
49+
with:
50+
path: |
51+
DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM
52+
DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app
53+
key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Swift/**') }}-${{ hashFiles('Sources/Sentry/**') }}
54+
- name: Cache iOS-Swift UI Test Runner App build product
55+
id: ios-swift-benchmark-runner-cache
56+
uses: actions/cache@v4
57+
with:
58+
path: |
59+
DerivedData/Build/Products/Debug-iphoneos/iOS-Benchmarking-Runner.app
60+
key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Benchmarking/**') }}
61+
- run: bundle exec fastlane build_ios_swift_for_tests
62+
env:
63+
APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
64+
APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
65+
APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
66+
FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
67+
MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
68+
MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
69+
MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
70+
- run: bundle exec fastlane build_ios_benchmark_test
71+
env:
72+
APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
73+
APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
74+
APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
75+
FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
76+
MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
77+
MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
78+
MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
79+
- name: Upload dSYMs
80+
run: |
81+
sentry-cli --auth-token ${{ secrets.SENTRY_AUTH_TOKEN }} upload-dif --org sentry-sdks --project sentry-cocoa DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM
82+
- name: Archiving DerivedData
83+
uses: actions/upload-artifact@v4
84+
with:
85+
name: DerivedData-Xcode
86+
path: |
87+
**/Debug-iphoneos/iOS-Swift.app
88+
**/Debug-iphoneos/iOS-Benchmarking-Runner.app
89+
90+
run-ui-tests-with-sauce:
91+
name: Run benchmarks on Sauce Labs
92+
runs-on: ubuntu-latest
93+
needs: build-benchmark-test-target
94+
strategy:
95+
fail-fast: false
96+
matrix:
97+
suite: ["High-end device", "Mid-range device", "Low-end device"]
98+
steps:
99+
- uses: actions/checkout@v4
100+
- uses: actions/download-artifact@v4
101+
with:
102+
name: DerivedData-Xcode
103+
- run: npm install -g [email protected]
104+
- name: Run Benchmarks in SauceLab
105+
id: run-benchmarks-in-sauce-lab
106+
env:
107+
SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
108+
SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
109+
# Note: We are not setting continue-on-error here, because we want the step to be marked as failed.
110+
run: |
111+
set -o pipefail && saucectl run \
112+
--select-suite "${{matrix.suite}}" \
113+
--config .sauce/benchmarking-config.yml \
114+
--tags benchmark \
115+
--verbose \
116+
2>&1 | tee output.log
117+
118+
- name: Recovery - Extract Test ID from output
119+
id: should-retry-test
120+
# Note: We need to use always() here, because the previous run step might be marked as failed.
121+
if: ${{ always() && steps.run-benchmarks-in-sauce-lab.outcome == 'failure' }}
122+
uses: actions/github-script@v7
123+
env:
124+
SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
125+
SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
126+
with:
127+
script: |
128+
const fs = require('fs');
129+
const { execSync } = require('child_process');
130+
131+
console.log("Extracting test ID from output log");
132+
const outputLog = fs.readFileSync('output.log', 'utf8');
133+
134+
// Lookup for the test ID in the output log
135+
// Note: The CLI output might change over time, so this might need to be updated.
136+
const match = outputLog.match(/https:\/\/app\.saucelabs\.com\/tests\/([^\s]+)/);
137+
const testId = match?.[1] ?? '';
138+
139+
if (!testId) {
140+
core.warning("No SauceLabs test ID found in CLI output, it might have changed, retrying...");
141+
core.setOutput('RETRY_TEST', 'true');
142+
143+
return;
144+
}
145+
146+
try {
147+
console.log(`Checking if the test exists in SauceLabs: ${testId}`);
148+
execSync(`saucectl jobs get ${testId}`, {
149+
env: process.env,
150+
stdio: 'inherit'
151+
});
152+
153+
console.log("Test exists but failed, not retrying.");
154+
core.setFailed('Test exists but failed');
155+
} catch (error) {
156+
console.log("Failed to get job, retrying...");
157+
core.setOutput('RETRY_TEST', 'true');
158+
}
159+
160+
- name: Run Benchmarks in SauceLab - Retry 1
161+
id: run-benchmarks-in-sauce-lab-retry-1
162+
# Note: We need to use always() here, because the previous run step might be marked as failed.
163+
if: ${{ always() && steps.should-retry-test.outputs.RETRY_TEST == 'true' }}
164+
env:
165+
SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
166+
SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
167+
run: |
168+
echo "::warning SauceLabs benchmark tests need to be retried"
169+
saucectl run \
170+
--select-suite "${{matrix.suite}}" \
171+
--config .sauce/benchmarking-config.yml \
172+
--tags benchmark \
173+
--verbose

0 commit comments

Comments
 (0)