Skip to content

Commit f74afde

Browse files
R-PalazzoR-Palazzo
committed
address comments
1 parent 83356ae commit f74afde

File tree

10 files changed

+216
-269
lines changed

10 files changed

+216
-269
lines changed

‎.github/workflows/run_benchmark.yml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
name: Run SDGym Benchmark
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
modality:
7+
required: true
8+
type: string
9+
sdgym_ref:
10+
required: false
11+
type: string
12+
default: issue-516-add-workflows
13+
secrets:
14+
SDV_ENTERPRISE_USERNAME:
15+
required: true
16+
SDV_ENTERPRISE_LICENSE_KEY:
17+
required: true
18+
GCP_SERVICE_ACCOUNT_JSON:
19+
required: true
20+
AWS_ACCESS_KEY_ID:
21+
required: true
22+
AWS_SECRET_ACCESS_KEY:
23+
required: true
24+
SLACK_TOKEN:
25+
required: true
26+
27+
jobs:
28+
run-sdgym-benchmark:
29+
runs-on: ubuntu-latest
30+
31+
steps:
32+
- uses: actions/checkout@v4
33+
with:
34+
fetch-depth: 0
35+
36+
- name: Set up Python
37+
uses: actions/setup-python@v5
38+
with:
39+
python-version-file: "pyproject.toml"
40+
41+
- name: Install dependencies
42+
env:
43+
USERNAME: ${{ secrets.SDV_ENTERPRISE_USERNAME }}
44+
LICENSE_KEY: ${{ secrets.SDV_ENTERPRISE_LICENSE_KEY }}
45+
run: |
46+
python -m venv venv
47+
source venv/bin/activate
48+
49+
python -m pip install --upgrade pip
50+
python -m pip install sdv-installer
51+
python -c "
52+
from sdv_installer.installation.installer import install_packages
53+
install_packages(
54+
username='${USERNAME}',
55+
license_key='${LICENSE_KEY}',
56+
package='sdv-enterprise',
57+
)
58+
"
59+
python -m pip install "sdgym[all] @ git+https://github.com/sdv-dev/SDGym.git@${{ inputs.sdgym_ref }}"
60+
61+
echo "VIRTUAL_ENV=$(pwd)/venv" >> $GITHUB_ENV
62+
echo "$(pwd)/venv/bin" >> $GITHUB_PATH
63+
64+
- name: Run SDGym Benchmark
65+
env:
66+
GCP_SERVICE_ACCOUNT_JSON: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
67+
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
68+
GCP_ZONE: ${{ secrets.GCP_ZONE }}
69+
SDV_ENTERPRISE_USERNAME: ${{ secrets.SDV_ENTERPRISE_USERNAME }}
70+
SDV_ENTERPRISE_LICENSE_KEY: ${{ secrets.SDV_ENTERPRISE_LICENSE_KEY }}
71+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
72+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
73+
SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
74+
run: |
75+
export CREDENTIALS_FILEPATH=$(python -c "from sdgym._benchmark.credentials_utils import create_credentials_file; print(create_credentials_file())")
76+
invoke run-sdgym-benchmark --modality "${{ inputs.modality }}"
77+
rm -f "$CREDENTIALS_FILEPATH"

‎.github/workflows/run_benchmark_multi_table.yaml‎

Lines changed: 6 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -3,58 +3,14 @@ name: Run SDGym Benchmark Multi-Table
33
on:
44
workflow_dispatch:
55
schedule:
6-
- cron: '0 5 1 * *'
6+
- cron: "0 5 1 * *"
77
push:
88
branches:
99
- issue-516-add-workflows
1010

1111
jobs:
12-
run-sdgym-multi-table-benchmark:
13-
runs-on: ubuntu-latest
14-
15-
steps:
16-
- uses: actions/checkout@v4
17-
with:
18-
fetch-depth: 0
19-
20-
- name: Set up Python
21-
uses: actions/setup-python@v5
22-
with:
23-
python-version-file: 'pyproject.toml'
24-
25-
- name: Install dependencies
26-
env:
27-
USERNAME: ${{ secrets.SDV_ENTERPRISE_USERNAME }}
28-
LICENSE_KEY: ${{ secrets.SDV_ENTERPRISE_LICENSE_KEY }}
29-
run: |
30-
python -m venv venv
31-
source venv/bin/activate
32-
33-
python -m pip install --upgrade pip
34-
python -m pip install sdv-installer
35-
python -c "
36-
from sdv_installer.installation.installer import install_packages
37-
install_packages(
38-
username='${USERNAME}',
39-
license_key='${LICENSE_KEY}',
40-
package='sdv-enterprise',
41-
)
42-
"
43-
python -m pip install "sdgym[all] @ git+https://github.com/sdv-dev/SDGym.git@issue-516-add-workflows"
44-
45-
echo "VIRTUAL_ENV=$(pwd)/venv" >> $GITHUB_ENV
46-
echo "$(pwd)/venv/bin" >> $GITHUB_PATH
47-
- name: Stop workflow after dependency install (temporary)
48-
run: exit 0
49-
- name: Run SDGym Benchmark
50-
env:
51-
GCP_SERVICE_ACCOUNT_JSON: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
52-
SDV_ENTERPRISE_USERNAME: ${{ secrets.SDV_ENTERPRISE_USERNAME }}
53-
SDV_ENTERPRISE_LICENSE_KEY: ${{ secrets.SDV_ENTERPRISE_LICENSE_KEY }}
54-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
55-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
56-
SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
57-
run: |
58-
export CREDENTIALS_FILEPATH=$(python -c "from sdgym._benchmark.credentials_utils import create_credentials_file; print(create_credentials_file())")
59-
invoke run-sdgym-benchmark --modality multi_table
60-
rm -f "$CREDENTIALS_FILEPATH"
12+
call-run-sdgym-benchmark:
13+
uses: ./.github/workflows/run_benchmark.yml
14+
with:
15+
modality: multi_table
16+
secrets: inherit

‎.github/workflows/run_benchmark_single_table.yml‎

Lines changed: 6 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -3,54 +3,11 @@ name: Run SDGym Benchmark Single-Table
33
on:
44
workflow_dispatch:
55
schedule:
6-
- cron: '0 5 1 * *'
6+
- cron: "0 5 1 * *"
77

88
jobs:
9-
run-sdgym-single-table-benchmark:
10-
runs-on: ubuntu-latest
11-
12-
steps:
13-
- uses: actions/checkout@v4
14-
with:
15-
fetch-depth: 0
16-
17-
- name: Set up Python
18-
uses: actions/setup-python@v5
19-
with:
20-
python-version-file: 'pyproject.toml'
21-
22-
- name: Install dependencies
23-
env:
24-
USERNAME: ${{ secrets.SDV_ENTERPRISE_USERNAME }}
25-
LICENSE_KEY: ${{ secrets.SDV_ENTERPRISE_LICENSE_KEY }}
26-
run: |
27-
python -m venv venv
28-
source venv/bin/activate
29-
30-
python -m pip install --upgrade pip
31-
python -m pip install sdv-installer
32-
python -c "
33-
from sdv_installer.installation.installer import install_packages
34-
install_packages(
35-
username='${USERNAME}',
36-
license_key='${LICENSE_KEY}',
37-
package='sdv-enterprise',
38-
)
39-
"
40-
python -m pip install "sdgym[all] @ git+https://github.com/sdv-dev/SDGym.git@issue-516-add-workflows"
41-
42-
echo "VIRTUAL_ENV=$(pwd)/venv" >> $GITHUB_ENV
43-
echo "$(pwd)/venv/bin" >> $GITHUB_PATH
44-
45-
- name: Run SDGym Benchmark
46-
env:
47-
GCP_SERVICE_ACCOUNT_JSON: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
48-
SDV_ENTERPRISE_USERNAME: ${{ secrets.SDV_ENTERPRISE_USERNAME }}
49-
SDV_ENTERPRISE_LICENSE_KEY: ${{ secrets.SDV_ENTERPRISE_LICENSE_KEY }}
50-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
51-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
52-
SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
53-
run: |
54-
export CREDENTIALS_FILEPATH=$(python -c "from sdgym._benchmark.credentials_utils import create_credentials_file; print(create_credentials_file())")
55-
invoke run-sdgym-benchmark --modality single_table
56-
rm -f "$CREDENTIALS_FILEPATH"
9+
call-run-sdgym-benchmark:
10+
uses: ./.github/workflows/run_benchmark.yml
11+
with:
12+
modality: single_table
13+
secrets: inherit

‎.github/workflows/upload_benchmark_results.yml‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
name: Upload SDGym Benchmark Results
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
modality:
7+
description: "Benchmark modality to upload"
8+
required: true
9+
type: string
10+
secrets:
11+
PYDRIVE_TOKEN:
12+
required: true
13+
AWS_ACCESS_KEY_ID:
14+
required: true
15+
AWS_SECRET_ACCESS_KEY:
16+
required: true
17+
GH_TOKEN:
18+
required: true
19+
SLACK_TOKEN:
20+
required: true
21+
22+
jobs:
23+
upload-sdgym-benchmark:
24+
runs-on: ubuntu-latest
25+
26+
steps:
27+
- uses: actions/checkout@v4
28+
with:
29+
fetch-depth: 0
30+
31+
- name: Set up latest Python
32+
uses: actions/setup-python@v5
33+
with:
34+
python-version-file: "pyproject.toml"
35+
36+
- name: Install dependencies
37+
run: |
38+
python -m pip install --upgrade pip
39+
python -m pip install --no-cache-dir -e .[dev]
40+
41+
- name: Upload SDGym Benchmark
42+
env:
43+
PY_state: "" # (optional) remove if unused
44+
PYDRIVE_TOKEN: ${{ secrets.PYDRIVE_TOKEN }}
45+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
46+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
47+
GITHUB_LOCAL_RESULTS_DIR: ${{ runner.temp }}/sdgym-leaderboard-files
48+
run: |
49+
invoke upload-benchmark-results --modality "${{ inputs.modality }}"
50+
echo "GITHUB_LOCAL_RESULTS_DIR=$GITHUB_LOCAL_RESULTS_DIR" >> $GITHUB_ENV
51+
52+
- name: Prepare files for commit
53+
if: env.SKIP_UPLOAD != 'true'
54+
run: |
55+
set -euo pipefail
56+
mkdir -p pr-staging
57+
echo "Looking for files in: $GITHUB_LOCAL_RESULTS_DIR"
58+
ls -l "$GITHUB_LOCAL_RESULTS_DIR" || true
59+
60+
shopt -s nullglob
61+
for f in "$GITHUB_LOCAL_RESULTS_DIR"/*; do
62+
[ -f "$f" ] && cp "$f" "pr-staging/$(basename "$f")"
63+
done
64+
65+
echo "Files staged for PR:"
66+
ls -l pr-staging || true
67+
68+
- name: Checkout target repo (sdv-dev.github.io)
69+
if: env.SKIP_UPLOAD != 'true'
70+
run: |
71+
git clone https://github.com/sdv-dev/sdv-dev.github.io.git target-repo
72+
cd target-repo
73+
git checkout gatsby-home
74+
75+
- name: Copy results and commit
76+
if: env.SKIP_UPLOAD != 'true'
77+
env:
78+
GH_TOKEN: ${{ secrets.GH_TOKEN }}
79+
FOLDER_NAME: ${{ env.FOLDER_NAME }}
80+
run: |
81+
set -euo pipefail
82+
83+
cp -f pr-staging/* target-repo/assets/sdgym-leaderboard-files/ || true
84+
cd target-repo
85+
86+
git config --local user.name "github-actions[bot]"
87+
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
88+
89+
git add assets/
90+
git commit -m "Upload SDGym Benchmark Results ($FOLDER_NAME)" || echo "No changes to commit"
91+
92+
git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/sdv-dev/sdv-dev.github.io.git"
93+
git push origin gatsby-home
94+
95+
COMMIT_HASH=$(git rev-parse HEAD)
96+
COMMIT_URL="https://github.com/sdv-dev/sdv-dev.github.io/commit/${COMMIT_HASH}"
97+
echo "COMMIT_URL=$COMMIT_URL" >> $GITHUB_ENV
98+
99+
- name: Send Slack notification
100+
if: env.SKIP_UPLOAD != 'true'
101+
env:
102+
SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
103+
run: |
104+
invoke notify-sdgym-benchmark-uploaded \
105+
--folder-name "$FOLDER_NAME" \
106+
--commit-url "$COMMIT_URL" \
107+
--modality "${{ inputs.modality }}"

0 commit comments

Comments
 (0)