Skip to content

Commit 83c6b44

Browse files
williambdeanwilliambdean
authored
Add slowest test issue CI/CD (#1125)
1 parent 231a977 commit 83c6b44

File tree

3 files changed

+224
-0
lines changed

3 files changed

+224
-0
lines changed

‎.github/workflows/slow-tests-issue.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Taken from https://github.com/pymc-labs/pymc-marketing/tree/main/.github/workflows/slow-tests-issue.yml
2+
# See the scripts in the `scripts/slowest_tests` directory for more information
3+
---
4+
name: Slow Tests Issue Body
5+
6+
on:
7+
workflow_dispatch:
8+
schedule:
9+
- cron: '0 */6 * * *'
10+
11+
permissions:
12+
issues: write
13+
14+
jobs:
15+
update-comment:
16+
runs-on: ubuntu-latest
17+
steps:
18+
- name: Install ZSH
19+
run: sudo apt-get update && sudo apt-get install -y zsh
20+
- name: Checkout code
21+
uses: actions/checkout@v4
22+
- name: Set up Python
23+
uses: actions/setup-python@v5
24+
with:
25+
python-version: "3.11"
26+
- name: Trigger the script
27+
working-directory: scripts/slowest_tests
28+
shell: zsh {0}
29+
run: source update-slowest-times-issue.sh
30+
env:
31+
GITHUB_TOKEN: ${{ github.token }}

‎scripts/slowest_tests/extract-slow-tests.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
"""This script parses the GitHub action log for test times.
2+
3+
Taken from https://github.com/pymc-labs/pymc-marketing/tree/main/scripts/slowest_tests/extract-slow-tests.py
4+
5+
"""
6+
7+
import re
8+
import sys
9+
from pathlib import Path
10+
11+
12+
start_pattern = re.compile(r"==== slow")
13+
separator_pattern = re.compile(r"====")
14+
time_pattern = re.compile(r"(\d+\.\d+)s ")
15+
16+
17+
def extract_lines(lines: list[str]) -> list[str]:
18+
times = []
19+
20+
in_section = False
21+
for line in lines:
22+
detect_start = start_pattern.search(line)
23+
detect_end = separator_pattern.search(line)
24+
25+
if detect_start:
26+
in_section = True
27+
28+
if in_section:
29+
times.append(line)
30+
31+
if not detect_start and in_section and detect_end:
32+
break
33+
34+
return times
35+
36+
37+
def trim_up_to_match(pattern, string: str) -> str:
38+
match = pattern.search(string)
39+
if not match:
40+
return ""
41+
42+
return string[match.start() :]
43+
44+
45+
def trim(pattern, lines: list[str]) -> list[str]:
46+
return [trim_up_to_match(pattern, line) for line in lines]
47+
48+
49+
def strip_ansi(text: str) -> str:
50+
ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
51+
return ansi_escape.sub("", text)
52+
53+
54+
def format_times(times: list[str]) -> list[str]:
55+
return (
56+
trim(separator_pattern, times[:1])
57+
+ trim(time_pattern, times[1:-1])
58+
+ [strip_ansi(line) for line in trim(separator_pattern, times[-1:])]
59+
)
60+
61+
62+
def read_lines_from_stdin():
63+
return sys.stdin.read().splitlines()
64+
65+
66+
def read_from_file(file: Path):
67+
"""For testing purposes."""
68+
return file.read_text().splitlines()
69+
70+
71+
def main(read_lines):
72+
lines = read_lines()
73+
times = extract_lines(lines)
74+
parsed_times = format_times(times)
75+
print("\n".join(parsed_times))
76+
77+
78+
if __name__ == "__main__":
79+
read_lines = read_lines_from_stdin
80+
main(read_lines)

‎scripts/slowest_tests/update-slowest-times-issue.sh‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
#!/bin/zsh
2+
3+
DRY_RUN=false
4+
5+
owner=pymc-devs
6+
repo=pytensor
7+
issue_number=1124
8+
title="Speed up test times :rocket:"
9+
workflow=Tests
10+
latest_id=$(gh run list --workflow $workflow --status success --limit 1 --json databaseId --jq '.[0].databaseId')
11+
jobs=$(gh api /repos/$owner/$repo/actions/runs/$latest_id/jobs --jq '.jobs | map({name: .name, run_id: .run_id, id: .id, started_at: .started_at, completed_at: .completed_at})')
12+
13+
# Skip 3.10, float32, and Benchmark tests
14+
function skip_job() {
15+
name=$1
16+
if [[ $name == *"py3.10"* ]]; then
17+
return 0
18+
fi
19+
20+
if [[ $name == *"float32 1"* ]]; then
21+
return 0
22+
fi
23+
24+
if [[ $name == *"Benchmark"* ]]; then
25+
return 0
26+
fi
27+
28+
return 1
29+
}
30+
31+
# Remove common prefix from the name
32+
function remove_prefix() {
33+
name=$1
34+
echo $name | sed -e 's/^ubuntu-latest test py3.12 : fast-compile 0 : float32 0 : //'
35+
}
36+
37+
function human_readable_time() {
38+
started_at=$1
39+
completed_at=$2
40+
41+
start_seconds=$(date -d "$started_at" +%s)
42+
end_seconds=$(date -d "$completed_at" +%s)
43+
44+
seconds=$(($end_seconds - $start_seconds))
45+
46+
if [ $seconds -lt 60 ]; then
47+
echo "$seconds seconds"
48+
else
49+
echo "$(date -u -d @$seconds +'%-M minutes %-S seconds')"
50+
fi
51+
}
52+
53+
all_times=""
54+
echo "$jobs" | jq -c '.[]' | while read -r job; do
55+
id=$(echo $job | jq -r '.id')
56+
name=$(echo $job | jq -r '.name')
57+
run_id=$(echo $job | jq -r '.run_id')
58+
started_at=$(echo $job | jq -r '.started_at')
59+
completed_at=$(echo $job | jq -r '.completed_at')
60+
61+
if skip_job $name; then
62+
echo "Skipping $name"
63+
continue
64+
fi
65+
66+
echo "Processing job: $name (ID: $id, Run ID: $run_id)"
67+
times=$(gh run view --job $id --log | python extract-slow-tests.py)
68+
69+
if [ -z "$times" ]; then
70+
# Some of the jobs are non-test jobs, so we skip them
71+
echo "No tests found for '$name', skipping"
72+
continue
73+
fi
74+
75+
echo $times
76+
77+
human_readable=$(human_readable_time $started_at $completed_at)
78+
name=$(remove_prefix $name)
79+
80+
top="<details><summary>($human_readable) $name</summary>\n\n\n\`\`\`"
81+
bottom="\`\`\`\n\n</details>"
82+
83+
formatted_times="$top\n$times\n$bottom"
84+
85+
if [ -n "$all_times" ]; then
86+
all_times="$all_times\n$formatted_times"
87+
else
88+
all_times="$formatted_times"
89+
fi
90+
done
91+
92+
run_date=$(date +"%Y-%m-%d")
93+
body=$(cat << EOF
94+
If you are motivated to help speed up some tests, we would appreciate it!
95+
96+
Here are some of the slowest test times:
97+
98+
$all_times
99+
100+
You can find more information on how to contribute [here](https://pytensor.readthedocs.io/en/latest/dev_start_guide.html)
101+
102+
Automatically generated by [GitHub Action](https://github.com/pymc-devs/pytensor/blob/main/.github/workflows/slow-tests-issue.yml)
103+
Latest run date: $run_date
104+
EOF
105+
)
106+
107+
if [ "$DRY_RUN" = true ]; then
108+
echo "Dry run, not updating issue"
109+
echo $body
110+
exit
111+
fi
112+
echo $body | gh issue edit $issue_number --body-file - --title "$title"
113+
echo "Updated issue $issue_number with all times"

0 commit comments

Comments
 (0)