Skip to content

Commit 1b5e082

Browse files
BuonOmorafiss
BuonOmo
authored and
rafiss
committed
feat(ci): add flaky test detector
Add a manually triggered workflow that runs a lot of tests in parallel with the `--fail-fast` option set. There are two goals here: - Detect new flaky tests - Get test seeds for fast failing flaky tests for an easier local reproduction Signed-off-by: Ulysse Buonomo <[email protected]>
1 parent c658997 commit 1b5e082

File tree

6 files changed

+141
-52
lines changed

6 files changed

+141
-52
lines changed

‎.github/actions/test-runner/action.yml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# Schema: https://json.schemastore.org/github-action.json
2+
3+
name: Test Runner
4+
description: Prepare and run tests for this repository
5+
6+
inputs:
7+
crdb:
8+
description: "CockroachDB version"
9+
required: true
10+
ruby:
11+
description: "Ruby version"
12+
required: true
13+
TESTOPTS:
14+
description: "Rails Minitest options"
15+
default: "--profile=5"
16+
runs:
17+
using: composite
18+
steps:
19+
- name: Install GEOS
20+
shell: bash
21+
run: sudo apt-get install -yqq libgeos-dev
22+
- name: Set Up Ruby
23+
uses: ruby/setup-ruby@v1
24+
with:
25+
ruby-version: ${{ inputs.ruby }}
26+
bundler-cache: true
27+
- name: Show Rails version
28+
shell: bash
29+
run: bundle info rails
30+
- name: Install and Start Cockroachdb
31+
shell: bash
32+
run: |
33+
# Download CockroachDB
34+
readonly full_version=$(ruby -rnet/http -ruri -ryaml -e '
35+
link = "https://raw.githubusercontent.com/cockroachdb/docs/main/src/current/_data/releases.yml"
36+
puts YAML.safe_load(Net::HTTP.get(URI(link))).reverse.find {
37+
_1["major_version"] == "${{ inputs.crdb }}" &&
38+
_1["release_type"] == "Production" &&
39+
!_1["cloud_only"] &&
40+
!_1["withdrawn"] &&
41+
!_1["release_name"].include?("-") # Pre-release
42+
}["release_name"]
43+
')
44+
45+
echo "Downloading $full_version..."
46+
wget -qO- "https://binaries.cockroachdb.com/cockroach-$full_version.linux-amd64.tgz" | tar xvz
47+
48+
export PATH=./cockroach-$full_version.linux-amd64/:$PATH
49+
readonly urlfile=cockroach-url
50+
51+
# Start a CockroachDB server and wait for it to become ready.
52+
rm -f "$urlfile"
53+
rm -rf cockroach-data
54+
# Start CockroachDB.
55+
cockroach start-single-node --max-sql-memory=25% --cache=25% --insecure --host=localhost --spatial-libs=./cockroach-$full_version.linux-amd64/lib --listening-url-file="$urlfile" >/dev/null 2>&1 &
56+
# Ensure CockroachDB is stopped on script exit.
57+
# Wait until CockroachDB has started.
58+
for i in {0..3}; do
59+
[[ -f "$urlfile" ]] && break
60+
backoff=$((2 ** i))
61+
echo "server not yet available; sleeping for $backoff seconds"
62+
sleep $backoff
63+
done
64+
cat ${{ github.workspace }}/setup.sql | cockroach sql --insecure
65+
- name: Test
66+
shell: bash
67+
run: bundle exec rake test
68+
env:
69+
TESTOPTS: "${{ inputs.TESTOPTS }}" # Fail fast allows us to find easy local reproduction by isolating quickly failing jobs.
70+
RAILS_MINITEST_PLUGIN: "1" # Make sure that we use the minitest plugin for profiling.

‎.github/workflows/ci.yml‎

Lines changed: 5 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -40,57 +40,13 @@ jobs:
4040
fail-fast: false
4141
matrix:
4242
# https://www.cockroachlabs.com/docs/releases/release-support-policy
43-
crdb: [v24.1, v24.3, v25.1, v25.2]
43+
crdb: [v24.3, v25.1, v25.2, v25.3]
4444
ruby: ["3.4"]
45-
name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }})
45+
name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }}
4646
steps:
4747
- name: Set Up Actions
4848
uses: actions/checkout@v4
49-
- name: Install GEOS
50-
run: sudo apt-get install -yqq libgeos-dev
51-
- name: Set Up Ruby
52-
uses: ruby/setup-ruby@v1
49+
- uses: ./.github/actions/test-runner
5350
with:
54-
ruby-version: ${{ matrix.ruby }}
55-
bundler-cache: true
56-
- name: Show Rails version
57-
run: bundle info rails
58-
- name: Install and Start Cockroachdb
59-
run: |
60-
# Download CockroachDB
61-
readonly full_version=$(ruby -rnet/http -ruri -ryaml -e '
62-
link = "https://raw.githubusercontent.com/cockroachdb/docs/main/src/current/_data/releases.yml"
63-
puts YAML.safe_load(Net::HTTP.get(URI(link))).reverse.find {
64-
_1["major_version"] == "${{ matrix.crdb }}" &&
65-
_1["release_type"] == "Production" &&
66-
!_1["cloud_only"] &&
67-
!_1["withdrawn"] &&
68-
!_1["release_name"].include?("-") # Pre-release
69-
}["release_name"]
70-
')
71-
72-
echo "Downloading $full_version..."
73-
wget -qO- "https://binaries.cockroachdb.com/cockroach-$full_version.linux-amd64.tgz" | tar xvz
74-
75-
export PATH=./cockroach-$full_version.linux-amd64/:$PATH
76-
readonly urlfile=cockroach-url
77-
78-
# Start a CockroachDB server and wait for it to become ready.
79-
rm -f "$urlfile"
80-
rm -rf cockroach-data
81-
# Start CockroachDB.
82-
cockroach start-single-node --max-sql-memory=25% --cache=25% --insecure --host=localhost --spatial-libs=./cockroach-$full_version.linux-amd64/lib --listening-url-file="$urlfile" >/dev/null 2>&1 &
83-
# Ensure CockroachDB is stopped on script exit.
84-
# Wait until CockroachDB has started.
85-
for i in {0..3}; do
86-
[[ -f "$urlfile" ]] && break
87-
backoff=$((2 ** i))
88-
echo "server not yet available; sleeping for $backoff seconds"
89-
sleep $backoff
90-
done
91-
cat ${{ github.workspace }}/setup.sql | cockroach sql --insecure
92-
- name: Test
93-
run: bundle exec rake test
94-
env:
95-
TESTOPTS: "--profile=5"
96-
RAILS_MINITEST_PLUGIN: "1" # Make sure that we use the minitest plugin for profiling.
51+
crdb: ${{ matrix.crdb }}
52+
ruby: ${{ matrix.ruby }}

‎.github/workflows/flaky.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: Flaky Test Detector
2+
3+
# Only run this workflow manually from the Actions tab
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
ruby:
8+
description: "Ruby version(s) (space separated)"
9+
required: true
10+
default: "3.4"
11+
crdb:
12+
description: "CockroachDB version(s) (space separated)"
13+
required: true
14+
default: "v25.2"
15+
max:
16+
description: "Maximum number of tests"
17+
required: true
18+
default: "256" # Maximum number of jobs on GitHub Actions
19+
20+
# This allows a subsequently queued workflow run to interrupt previous runs.
21+
concurrency:
22+
group: "${{ github.workflow }} @ ${{ github.ref }}"
23+
cancel-in-progress: true
24+
25+
jobs:
26+
prepare-matrix:
27+
runs-on: ubuntu-latest
28+
name: Prepare Matrix
29+
steps:
30+
- id: generate-matrix
31+
run: |
32+
crdb=$(jq --raw-input --compact-output 'split(" ")' <<<"${{ github.event.inputs.crdb }}")
33+
ruby=$(jq --raw-input --compact-output 'split(" ")' <<<"${{ github.event.inputs.ruby }}")
34+
crdb_len=$(wc -w <<<"${{ github.event.inputs.crdb }}")
35+
ruby_len=$(wc -w <<<"${{ github.event.inputs.ruby }}")
36+
(( range_count = ${{github.event.inputs.max}} / ( crdb_len * ruby_len ) ))
37+
range=$(jq --compact-output "[range($range_count)]" <<<[])
38+
echo "crdb=$crdb" >> $GITHUB_OUTPUT
39+
echo "ruby=$ruby" >> $GITHUB_OUTPUT
40+
echo "numbers=$range" >> $GITHUB_OUTPUT
41+
outputs:
42+
crdb: ${{ steps.generate-matrix.outputs.crdb }}
43+
ruby: ${{ steps.generate-matrix.outputs.ruby }}
44+
numbers: ${{ steps.generate-matrix.outputs.numbers }}
45+
test:
46+
runs-on: ubuntu-latest
47+
needs: prepare-matrix
48+
strategy:
49+
fail-fast: false
50+
matrix:
51+
crdb: ${{ fromJSON(needs.prepare-matrix.outputs.crdb) }}
52+
ruby: ${{ fromJSON(needs.prepare-matrix.outputs.ruby) }}
53+
number: ${{ fromJSON(needs.prepare-matrix.outputs.numbers) }}
54+
name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }} number=${{ matrix.number }})
55+
steps:
56+
- name: Set Up Actions
57+
uses: actions/checkout@v4
58+
- uses: ./.github/actions/test-runner
59+
with:
60+
crdb: ${{ matrix.crdb }}
61+
ruby: ${{ matrix.ruby }}
62+
TESTOPTS: --fail-fast

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,4 @@
1616
/ruby-build/
1717
/test/db/
1818
/.ruby-version
19+
*.sqlite3

‎test/cases/helper_cockroachdb.rb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@ def before_teardown
189189
super
190190
end
191191
end
192-
MiniTest::Test.include(TraceLibPlugin)
192+
Minitest::Test.include(TraceLibPlugin)
193193
end
194194

195195
# Log all SQL queries and print total time spent in SQL.

‎test/support/sql_logger.rb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@ def summary_log
2020
detail = ActiveRecord::TotalTimeSubscriber.hash.map { |k,v| [k, [v.sum, v.sum / v.size, v.size]]}.sort_by { |_, (_total, avg, _)| -avg }.to_h
2121
time = detail.values.sum { |(total, _, _)| total } / 1_000
2222
count = detail.values.sum { |(_, _, count)| count }
23-
puts "Total time spent in SQL: #{time}s (#{count} queries)"
24-
puts "Detail per query kind available in tmp/query_time.json (total time in ms, avg time in ms, query count). Sorted by avg time."
2523
File.write(
2624
"tmp/query_time.json",
2725
JSON.pretty_generate(detail)
2826
)
27+
puts "Total time spent in SQL: #{time}s (#{count} queries)"
28+
puts "Detail per query kind available in tmp/query_time.json (total time in ms, avg time in ms, query count). Sorted by avg time."
2929
}
3030
end
3131

0 commit comments

Comments
 (0)