Skip to content

Commit 38b4539

Browse files
delucchi-cmudelucchi-cmu
authored
Merge pull request #29 from astronomy-commons/issue/lsdb/892
Add benchmarks for Rubin Environment tests.
2 parents 11b0b10 + 64ec1f8 commit 38b4539

File tree

15 files changed

+503
-517
lines changed

15 files changed

+503
-517
lines changed

‎.copier-answers.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,13 @@ _commit: v2.0.7
33
_src_path: gh:lincc-frameworks/python-project-template
44
author_email: [email protected]
55
author_name: LINCC Frameworks
6-
create_example_module: true
6+
create_example_module: false
77
custom_install: true
88
enforce_style:
99
- ruff_lint
1010
- ruff_format
1111
failure_notification: []
12-
include_benchmarks: false
12+
include_benchmarks: true
1313
include_docs: true
1414
include_notebooks: false
1515
mypy_type_checking: none

‎.github/workflows/asv-main.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# This workflow will run benchmarks with airspeed velocity (asv),
2+
# store the new results in the "benchmarks" branch and publish them
3+
# to a dashboard on GH Pages.
4+
name: Run ASV benchmarks for main
5+
6+
on:
7+
push:
8+
branches: [ main ]
9+
10+
env:
11+
PYTHON_VERSION: "3.11"
12+
ASV_VERSION: "0.6.4"
13+
WORKING_DIR: ${{github.workspace}}/benchmarks
14+
15+
concurrency:
16+
group: ${{github.workflow}}-${{github.ref}}
17+
cancel-in-progress: true
18+
19+
jobs:
20+
asv-main:
21+
runs-on: ubuntu-latest
22+
permissions:
23+
contents: write
24+
defaults:
25+
run:
26+
working-directory: ${{env.WORKING_DIR}}
27+
steps:
28+
- name: Set up Python ${{env.PYTHON_VERSION}}
29+
uses: actions/setup-python@v5
30+
with:
31+
python-version: ${{env.PYTHON_VERSION}}
32+
- name: Checkout main branch of the repository
33+
uses: actions/checkout@v4
34+
with:
35+
fetch-depth: 0
36+
- name: Install dependencies
37+
run: pip install "asv[virtualenv]==${{env.ASV_VERSION}}"
38+
- name: Configure git
39+
run: |
40+
git config user.name "github-actions[bot]"
41+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
42+
- name: Create ASV machine config file
43+
run: asv machine --machine gh-runner --yes
44+
- name: Fetch previous results from the "benchmarks" branch
45+
run: |
46+
if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then
47+
git merge origin/benchmarks \
48+
--allow-unrelated-histories \
49+
--no-commit
50+
mv ../_results .
51+
fi
52+
- name: Run ASV for the main branch
53+
run: asv run ALL --skip-existing --verbose || true
54+
- name: Submit new results to the "benchmarks" branch
55+
uses: JamesIves/github-pages-deploy-action@v4
56+
with:
57+
branch: benchmarks
58+
folder: ${{env.WORKING_DIR}}/_results
59+
target-folder: _results
60+
- name: Generate dashboard HTML
61+
run: |
62+
asv show
63+
asv publish
64+
- name: Deploy to Github pages
65+
uses: JamesIves/github-pages-deploy-action@v4
66+
with:
67+
branch: gh-pages
68+
folder: ${{env.WORKING_DIR}}/_html

‎.github/workflows/asv-nightly.yml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
# This workflow will run daily at 06:45.
2+
# It will run benchmarks with airspeed velocity (asv)
3+
# and compare performance with the previous nightly build.
4+
name: Run benchmarks nightly job
5+
6+
on:
7+
schedule:
8+
- cron: 45 6 * * *
9+
workflow_dispatch:
10+
11+
env:
12+
PYTHON_VERSION: "3.11"
13+
ASV_VERSION: "0.6.4"
14+
WORKING_DIR: ${{github.workspace}}/benchmarks
15+
NIGHTLY_HASH_FILE: nightly-hash
16+
17+
jobs:
18+
asv-nightly:
19+
runs-on: ubuntu-latest
20+
defaults:
21+
run:
22+
working-directory: ${{env.WORKING_DIR}}
23+
steps:
24+
- name: Set up Python ${{env.PYTHON_VERSION}}
25+
uses: actions/setup-python@v5
26+
with:
27+
python-version: ${{env.PYTHON_VERSION}}
28+
- name: Checkout main branch of the repository
29+
uses: actions/checkout@v4
30+
with:
31+
fetch-depth: 0
32+
- name: Install dependencies
33+
run: pip install "asv[virtualenv]==${{env.ASV_VERSION}}"
34+
- name: Configure git
35+
run: |
36+
git config user.name "github-actions[bot]"
37+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
38+
- name: Create ASV machine config file
39+
run: asv machine --machine gh-runner --yes
40+
- name: Fetch previous results from the "benchmarks" branch
41+
run: |
42+
if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then
43+
git merge origin/benchmarks \
44+
--allow-unrelated-histories \
45+
--no-commit
46+
mv ../_results .
47+
fi
48+
- name: Get nightly dates under comparison
49+
id: nightly-dates
50+
run: |
51+
echo "yesterday=$(date -d yesterday +'%Y-%m-%d')" >> $GITHUB_OUTPUT
52+
echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
53+
- name: Use last nightly commit hash from cache
54+
uses: actions/cache@v4
55+
with:
56+
path: ${{env.WORKING_DIR}}
57+
key: nightly-results-${{steps.nightly-dates.outputs.yesterday}}
58+
- name: Run comparison of main against last nightly build
59+
run: |
60+
HASH_FILE=${{env.NIGHTLY_HASH_FILE}}
61+
CURRENT_HASH=${{github.sha}}
62+
if [ -f $HASH_FILE ]; then
63+
PREV_HASH=$(cat $HASH_FILE)
64+
asv continuous $PREV_HASH $CURRENT_HASH --verbose || true
65+
asv compare $PREV_HASH $CURRENT_HASH --sort ratio --verbose
66+
fi
67+
echo $CURRENT_HASH > $HASH_FILE
68+
- name: Update last nightly hash in cache
69+
uses: actions/cache@v4
70+
with:
71+
path: ${{env.WORKING_DIR}}
72+
key: nightly-results-${{steps.nightly-dates.outputs.today}}

‎.github/workflows/asv-pr.yml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# This workflow will run benchmarks with airspeed velocity (asv) for pull requests.
2+
# It will compare the performance of the main branch with the performance of the merge
3+
# with the new changes. It then publishes a comment with this assessment by triggering
4+
# the publish-benchmarks-pr workflow.
5+
# Based on https://securitylab.github.com/research/github-actions-preventing-pwn-requests/.
6+
name: Run benchmarks for PR
7+
8+
on:
9+
pull_request:
10+
branches: [ main ]
11+
workflow_dispatch:
12+
13+
concurrency:
14+
group: ${{github.workflow}}-${{github.ref}}
15+
cancel-in-progress: true
16+
17+
env:
18+
PYTHON_VERSION: "3.11"
19+
ASV_VERSION: "0.6.4"
20+
WORKING_DIR: ${{github.workspace}}/benchmarks
21+
ARTIFACTS_DIR: ${{github.workspace}}/artifacts
22+
23+
jobs:
24+
asv-pr:
25+
runs-on: ubuntu-latest
26+
defaults:
27+
run:
28+
working-directory: ${{env.WORKING_DIR}}
29+
steps:
30+
- name: Set up Python ${{env.PYTHON_VERSION}}
31+
uses: actions/setup-python@v5
32+
with:
33+
python-version: ${{env.PYTHON_VERSION}}
34+
- name: Checkout PR branch of the repository
35+
uses: actions/checkout@v4
36+
with:
37+
fetch-depth: 0
38+
- name: Display Workflow Run Information
39+
run: |
40+
echo "Workflow Run ID: ${{github.run_id}}"
41+
- name: Install dependencies
42+
run: pip install "asv[virtualenv]==${{env.ASV_VERSION}}" lf-asv-formatter
43+
- name: Make artifacts directory
44+
run: mkdir -p ${{env.ARTIFACTS_DIR}}
45+
- name: Save pull request number
46+
run: echo ${{github.event.pull_request.number}} > ${{env.ARTIFACTS_DIR}}/pr
47+
- name: Get current job logs URL
48+
uses: Tiryoh/gha-jobid-action@v1
49+
id: jobs
50+
with:
51+
github_token: ${{secrets.GITHUB_TOKEN}}
52+
job_name: ${{github.job}}
53+
- name: Create ASV machine config file
54+
run: asv machine --machine gh-runner --yes
55+
- name: Save comparison of PR against main branch
56+
run: |
57+
git remote add upstream https://github.com/${{github.repository}}.git
58+
git fetch upstream
59+
asv continuous upstream/main HEAD --verbose || true
60+
asv compare upstream/main HEAD --sort ratio --verbose | tee output
61+
python -m lf_asv_formatter --asv_version "$(asv --version | awk '{print $2}')"
62+
printf "\n\nClick [here]($STEP_URL) to view all benchmarks." >> output
63+
mv output ${{env.ARTIFACTS_DIR}}
64+
env:
65+
STEP_URL: ${{steps.jobs.outputs.html_url}}#step:10:1
66+
- name: Upload artifacts (PR number and benchmarks output)
67+
uses: actions/upload-artifact@v4
68+
with:
69+
name: benchmark-artifacts
70+
path: ${{env.ARTIFACTS_DIR}}

‎.github/workflows/publish-benchmarks-pr.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# This workflow publishes a benchmarks comment on a pull request. It is triggered after the
2+
# benchmarks are computed in the asv-pr workflow. This separation of concerns allows us limit
3+
# access to the target repository private tokens and secrets, increasing the level of security.
4+
# Based on https://securitylab.github.com/research/github-actions-preventing-pwn-requests/.
5+
name: Publish benchmarks comment to PR
6+
7+
on:
8+
workflow_run:
9+
workflows: ["Run benchmarks for PR"]
10+
types: [completed]
11+
12+
jobs:
13+
upload-pr-comment:
14+
runs-on: ubuntu-latest
15+
if: >
16+
github.event.workflow_run.event == 'pull_request' &&
17+
github.event.workflow_run.conclusion == 'success'
18+
permissions:
19+
issues: write
20+
pull-requests: write
21+
steps:
22+
- name: Display Workflow Run Information
23+
run: |
24+
echo "Workflow Run ID: ${{ github.event.workflow_run.id }}"
25+
echo "Head SHA: ${{ github.event.workflow_run.head_sha }}"
26+
echo "Head Branch: ${{ github.event.workflow_run.head_branch }}"
27+
echo "Conclusion: ${{ github.event.workflow_run.conclusion }}"
28+
echo "Event: ${{ github.event.workflow_run.event }}"
29+
- name: Download artifact
30+
uses: dawidd6/action-download-artifact@v7
31+
with:
32+
name: benchmark-artifacts
33+
run_id: ${{ github.event.workflow_run.id }}
34+
- name: Extract artifacts information
35+
id: pr-info
36+
run: |
37+
printf "PR number: $(cat pr)\n"
38+
printf "Output:\n$(cat output)"
39+
printf "pr=$(cat pr)" >> $GITHUB_OUTPUT
40+
- name: Find benchmarks comment
41+
uses: peter-evans/find-comment@v3
42+
id: find-comment
43+
with:
44+
issue-number: ${{ steps.pr-info.outputs.pr }}
45+
comment-author: 'github-actions[bot]'
46+
body-includes: view all benchmarks
47+
- name: Create or update benchmarks comment
48+
uses: peter-evans/create-or-update-comment@v4
49+
with:
50+
comment-id: ${{ steps.find-comment.outputs.comment-id }}
51+
issue-number: ${{ steps.pr-info.outputs.pr }}
52+
body-path: output
53+
edit-mode: replace

‎benchmarks/__init__.py‎

Whitespace-only changes.

‎benchmarks/asv.conf.json‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
2+
{
3+
// The version of the config file format. Do not change, unless
4+
// you know what you are doing.
5+
"version": 1,
6+
// The name of the project being benchmarked.
7+
"project": "lsdb_rubin",
8+
// The project's homepage.
9+
"project_url": "https://github.com/astronomy-commons/lsdb_rubin",
10+
// The URL or local path of the source code repository for the
11+
// project being benchmarked.
12+
"repo": "..",
13+
// List of branches to benchmark. If not provided, defaults to "master"
14+
// (for git) or "tip" (for mercurial).
15+
"branches": [
16+
"HEAD"
17+
],
18+
"install_command": [
19+
"python -m pip install {wheel_file}"
20+
],
21+
"build_command": [
22+
"python -m build --wheel -o {build_cache_dir} {build_dir}"
23+
],
24+
// The DVCS being used. If not set, it will be automatically
25+
// determined from "repo" by looking at the protocol in the URL
26+
// (if remote), or by looking for special directories, such as
27+
// ".git" (if local).
28+
"dvcs": "git",
29+
// The tool to use to create environments. May be "conda",
30+
// "virtualenv" or other value depending on the plugins in use.
31+
// If missing or the empty string, the tool will be automatically
32+
// determined by looking for tools on the PATH environment
33+
// variable.
34+
"environment_type": "virtualenv",
35+
// the base URL to show a commit for the project.
36+
"show_commit_url": "https://github.com/astronomy-commons/lsdb_rubin/commit/",
37+
// The Pythons you'd like to test against. If not provided, defaults
38+
// to the current version of Python used to run `asv`.
39+
"pythons": [
40+
"3.11"
41+
],
42+
// The matrix of dependencies to test. Each key is the name of a
43+
// package (in PyPI) and the values are version numbers. An empty
44+
// list indicates to just test against the default (latest)
45+
// version.
46+
"matrix": {
47+
"Cython": [],
48+
"build": [],
49+
"packaging": []
50+
},
51+
// The directory (relative to the current directory) that benchmarks are
52+
// stored in. If not provided, defaults to "benchmarks".
53+
"benchmark_dir": ".",
54+
// The directory (relative to the current directory) to cache the Python
55+
// environments in. If not provided, defaults to "env".
56+
"env_dir": "env",
57+
// The directory (relative to the current directory) that raw benchmark
58+
// results are stored in. If not provided, defaults to "results".
59+
"results_dir": "_results",
60+
// The directory (relative to the current directory) that the html tree
61+
// should be written to. If not provided, defaults to "html".
62+
"html_dir": "_html",
63+
// The number of characters to retain in the commit hashes.
64+
// "hash_length": 8,
65+
// `asv` will cache wheels of the recent builds in each
66+
// environment, making them faster to install next time. This is
67+
// number of builds to keep, per environment.
68+
"build_cache_size": 8
69+
// The commits after which the regression search in `asv publish`
70+
// should start looking for regressions. Dictionary whose keys are
71+
// regexps matching to benchmark names, and values corresponding to
72+
// the commit (exclusive) after which to start looking for
73+
// regressions. The default is to start from the first commit
74+
// with results. If the commit is `null`, regression detection is
75+
// skipped for the matching benchmark.
76+
//
77+
// "regressions_first_commits": {
78+
// "some_benchmark": "352cdf", // Consider regressions only after this commit
79+
// "another_benchmark": null, // Skip regression detection altogether
80+
// }
81+
}

‎benchmarks/benchmarks.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
from pathlib import Path
2+
3+
from lsdb_rubin.rsp_tests.critical_functions import critical_functions
4+
from lsdb_rubin.rsp_tests.random_access import random_access
5+
6+
7+
def time_critical_functions():
8+
"""Run the contents of the `Test of LSDB Critical functions` notebook,
9+
reporting the total runtime. Any deviation from expected values will
10+
cause the benchmark test to fail.
11+
12+
See https://github.com/lsst-sitcom/linccf/blob/main/RSP/critical_fs.ipynb"""
13+
critical_functions()
14+
15+
16+
def time_random_access():
17+
"""Run the contents of the `HATS Data Preview 1 on RSP` notebook,
18+
reporting the total runtime. Any deviation from expected values will
19+
cause the benchmark test to fail.
20+
21+
See https://github.com/lsst-sitcom/linccf/blob/main/RSP/random_access.ipynb"""
22+
23+
random_access(Path(__file__).parent.parent / "tests" / "data" / "mock_dp1_1000")

0 commit comments

Comments
 (0)