update upload results

Author: R-Palazzo

.github/workflows/upload_benchmark_results.yml

@@ -88,4 +88,4 @@ jobs:
       run: |
         invoke notify-sdgym-benchmark-uploaded \
           --folder-name "$FOLDER_NAME" \
-          --commit-url "$COMMIT_URL"
+          --commit-url "$COMMIT_URL" --modality single_table

.github/workflows/upload_benchmark_results_multi_table.yml

@@ -0,0 +1,94 @@
+name: Upload SDGym Multi-Table Benchmark results
+
+on:
+  workflow_run:
+    workflows: ["Run SDGym Benchmark Multi-Table"]
+    types:
+      - completed
+  workflow_dispatch:
+  schedule:
+    - cron: '0 6 * * *'
+  push:
+    branches:
+      - issue-516-add-workflows
+
+jobs:
+  upload-sdgym-benchmark:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up latest Python
+      uses: actions/setup-python@v5
+      with:
+        python-version-file: 'pyproject.toml'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --no-cache-dir -e .[dev]
+          
+    - name: Upload SDGym Benchmark
+      env:
+        PYDRIVE_TOKEN: ${{ secrets.PYDRIVE_TOKEN }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        GITHUB_LOCAL_RESULTS_DIR: ${{ runner.temp }}/sdgym-leaderboard-files
+      run: |
+        invoke upload-benchmark-results --modality multi_table
+        echo "GITHUB_LOCAL_RESULTS_DIR=$GITHUB_LOCAL_RESULTS_DIR" >> $GITHUB_ENV
+
+    - name: Prepare files for commit
+      if: env.SKIP_UPLOAD != 'true'
+      run: |
+        mkdir pr-staging
+        echo "Looking for files in: $GITHUB_LOCAL_RESULTS_DIR"
+        ls -l "$GITHUB_LOCAL_RESULTS_DIR"
+        for f in "$GITHUB_LOCAL_RESULTS_DIR"/*; do
+          if [ -f "$f" ]; then
+            base=$(basename "$f")
+            cp "$f" "pr-staging/${base}"
+          fi
+        done
+
+        echo "Files staged for PR:"
+        ls -l pr-staging
+
+    - name: Checkout target repo (sdv-dev.github.io)
+      if: env.SKIP_UPLOAD != 'true'
+      run: |
+        git clone https://github.com/sdv-dev/sdv-dev.github.io.git target-repo
+        cd target-repo
+        git checkout gatsby-home
+
+    - name: Copy results and commit
+      if: env.SKIP_UPLOAD != 'true'
+      env:
+        GH_TOKEN: ${{ secrets.GH_TOKEN }}
+        FOLDER_NAME: ${{ env.FOLDER_NAME }}
+      run: |
+        cp pr-staging/* target-repo/assets/sdgym-leaderboard-files/
+        cd target-repo
+        git checkout gatsby-home
+        git config --local user.name "github-actions[bot]"
+        git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+        git add assets/
+        git commit -m "Upload SDGym Benchmark Results ($FOLDER_NAME)" || echo "No changes to commit"
+        git remote set-url origin https://x-access-token:${GH_TOKEN}@github.com/sdv-dev/sdv-dev.github.io.git
+        git push origin gatsby-home
+        COMMIT_HASH=$(git rev-parse HEAD)
+        COMMIT_URL="https://github.com/sdv-dev/sdv-dev.github.io/commit/${COMMIT_HASH}"
+        echo "Commit URL: $COMMIT_URL"
+        echo "COMMIT_URL=$COMMIT_URL" >> $GITHUB_ENV
+
+    - name: Send Slack notification
+      if: env.SKIP_UPLOAD != 'true'
+      env:
+        SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
+      run: |
+        invoke notify-sdgym-benchmark-uploaded \
+          --folder-name "$FOLDER_NAME" \
+          --commit-url "$COMMIT_URL" --modality multi_table

sdgym/_benchmark/credentials_utils.py

@@ -21,6 +21,8 @@
     },
     'sdv': {'username', 'license_key'},
 }
+GCP_PROJECT_ID = 'sdgym-337614'
+GCP_ZONE = 'us-central1-a'
 
 
 def get_credentials(credential_filepath):
@@ -69,8 +71,7 @@ def sdv_install_cmd(credentials):
         return ''
 
     return (
-        'pip install bundle-xsynthesizers '
-        f'--index-url https://{username}:{license_key}@pypi.datacebo.com'
+        f'pip install sdv_enterprise --index-url https://{username}:{license_key}@pypi.datacebo.com'
     )
 
 
@@ -84,8 +85,8 @@ def create_credentials_file():
         },
         'gcp': {
             **json.loads(gcp_json),
-            'gcp_project': 'sdgym-337614',
-            'gcp_zone': 'us-central1-a',
+            'gcp_project': GCP_PROJECT_ID,
+            'gcp_zone': GCP_ZONE,
         },
         'sdv': {
             'username': os.getenv('SDV_ENTERPRISE_USERNAME'),
@@ -96,4 +97,5 @@ def create_credentials_file():
     tmp_file = NamedTemporaryFile(mode='w+', delete=False, suffix='.json')
     json.dump(credentials, tmp_file)
     tmp_file.flush()
+
     return tmp_file.name

sdgym/result_explorer/result_explorer.py

@@ -20,7 +20,7 @@ def _validate_local_path(path):
 
 _BASELINE_BY_MODALITY = {
     'single_table': SYNTHESIZER_BASELINE,
-    'multi_table': 'MultiTableUniformSynthesizer',
+    'multi_table': 'IdependentSynthesizer',
 }
 
 

sdgym/run_benchmark/run_benchmark.py

@@ -1,6 +1,5 @@
 """Script to run a benchmark and upload results to S3."""
 
-import argparse
 import json
 import os
 from datetime import datetime, timezone
@@ -16,6 +15,7 @@
     OUTPUT_DESTINATION_AWS,
     SYNTHESIZERS_SPLIT_MULTI_TABLE,
     SYNTHESIZERS_SPLIT_SINGLE_TABLE,
+    _parse_args,
     get_result_folder_name,
     post_benchmark_launch_message,
 )
@@ -61,17 +61,6 @@ def append_benchmark_run(
     )
 
 
-def _parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        '--modality',
-        choices=['single_table', 'multi_table'],
-        default='single_table',
-        help='Benchmark modality to run.',
-    )
-    return parser.parse_args()
-
-
 def main():
     """Main function to run the benchmark and upload results."""
     args = _parse_args()

sdgym/run_benchmark/upload_benchmark_results.py

@@ -16,7 +16,7 @@
 
 from sdgym.result_explorer.result_explorer import ResultsExplorer
 from sdgym.result_writer import LocalResultsWriter
-from sdgym.run_benchmark.utils import OUTPUT_DESTINATION_AWS, get_df_to_plot
+from sdgym.run_benchmark.utils import OUTPUT_DESTINATION_AWS, _parse_args, get_df_to_plot
 from sdgym.s3 import S3_REGION, parse_s3_path
 
 LOGGER = logging.getLogger(__name__)
@@ -45,17 +45,21 @@ def get_latest_run_from_file(s3_client, bucket, key):
         raise RuntimeError(f'Failed to read {key} from S3: {e}')
 
 
-def write_uploaded_marker(s3_client, bucket, prefix, folder_name):
+def write_uploaded_marker(s3_client, bucket, prefix, folder_name, modality='single_table'):
     """Write a marker file to indicate that the upload is complete."""
     s3_client.put_object(
-        Bucket=bucket, Key=f'{prefix}{folder_name}/upload_complete.marker', Body=b'Upload complete'
+        Bucket=bucket,
+        Key=f'{prefix}{modality}/{folder_name}/upload_complete.marker',
+        Body=b'Upload complete',
     )
 
 
-def upload_already_done(s3_client, bucket, prefix, folder_name):
+def upload_already_done(s3_client, bucket, prefix, folder_name, modality='single_table'):
     """Check if the upload has already been done by looking for the marker file."""
     try:
-        s3_client.head_object(Bucket=bucket, Key=f'{prefix}{folder_name}/upload_complete.marker')
+        s3_client.head_object(
+            Bucket=bucket, Key=f'{prefix}{modality}/{folder_name}/upload_complete.marker'
+        )
         return True
     except ClientError as e:
         if e.response['Error']['Code'] == '404':
@@ -64,7 +68,9 @@ def upload_already_done(s3_client, bucket, prefix, folder_name):
         raise
 
 
-def get_result_folder_name_and_s3_vars(aws_access_key_id, aws_secret_access_key):
+def get_result_folder_name_and_s3_vars(
+    aws_access_key_id, aws_secret_access_key, modality='single_table'
+):
     """Get the result folder name and S3 client variables."""
     bucket, prefix = parse_s3_path(OUTPUT_DESTINATION_AWS)
     s3_client = boto3.client(
@@ -73,7 +79,9 @@ def get_result_folder_name_and_s3_vars(aws_access_key_id, aws_secret_access_key)
         aws_secret_access_key=aws_secret_access_key,
         region_name=S3_REGION,
     )
-    folder_infos = get_latest_run_from_file(s3_client, bucket, f'{prefix}_BENCHMARK_DATES.json')
+    folder_infos = get_latest_run_from_file(
+        s3_client, bucket, f'{prefix}{modality}/_BENCHMARK_DATES.json'
+    )
 
     return folder_infos, s3_client, bucket, prefix
 
@@ -109,14 +117,21 @@ def upload_to_drive(file_path, file_id):
 
 
 def upload_results(
-    aws_access_key_id, aws_secret_access_key, folder_infos, s3_client, bucket, prefix, github_env
+    aws_access_key_id,
+    aws_secret_access_key,
+    folder_infos,
+    s3_client,
+    bucket,
+    prefix,
+    github_env,
+    modality='single_table',
 ):
     """Upload benchmark results to S3, GDrive, and save locally."""
     folder_name = folder_infos['folder_name']
     run_date = folder_infos['date']
     result_explorer = ResultsExplorer(
         OUTPUT_DESTINATION_AWS,
-        modality='single_table',
+        modality=modality,
         aws_access_key_id=aws_access_key_id,
         aws_secret_access_key=aws_secret_access_key,
     )
@@ -145,7 +160,7 @@ def upload_results(
 
     Path(local_export_dir).mkdir(parents=True, exist_ok=True)
     local_file_path = str(Path(local_export_dir) / RESULT_FILENAME)
-    s3_key = f'{prefix}{RESULT_FILENAME}'
+    s3_key = f'{prefix}{modality}/{RESULT_FILENAME}'
     s3_client.download_file(bucket, s3_key, local_file_path)
     datas = {
         'Wins': summary,
@@ -155,20 +170,22 @@ def upload_results(
     local_results_writer.write_xlsx(datas, local_file_path)
     upload_to_drive((local_file_path), SDGYM_FILE_ID)
     s3_client.upload_file(local_file_path, bucket, s3_key)
-    write_uploaded_marker(s3_client, bucket, prefix, folder_name)
+    write_uploaded_marker(s3_client, bucket, prefix, folder_name, modality=modality)
     if temp_dir:
         shutil.rmtree(temp_dir)
 
 
 def main():
     """Main function to upload benchmark results."""
+    args = _parse_args()
+    modality = args.modality
     aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
     aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY')
     folder_infos, s3_client, bucket, prefix = get_result_folder_name_and_s3_vars(
-        aws_access_key_id, aws_secret_access_key
+        aws_access_key_id, aws_secret_access_key, modality=modality
     )
     github_env = os.getenv('GITHUB_ENV')
-    if upload_already_done(s3_client, bucket, prefix, folder_infos['folder_name']):
+    if upload_already_done(s3_client, bucket, prefix, folder_infos['folder_name'], modality):
         LOGGER.warning('Benchmark results have already been uploaded. Exiting.')
         if github_env:
             with open(github_env, 'a') as env_file:
@@ -184,6 +201,7 @@ def main():
         bucket,
         prefix,
         github_env,
+        modality,
     )
 
 

sdgym/run_benchmark/utils.py

@@ -1,5 +1,6 @@
 """Utils file for the run_benchmark module."""
 
+import argparse
 import os
 from datetime import datetime
 from urllib.parse import quote_plus
@@ -12,7 +13,7 @@
 OUTPUT_DESTINATION_AWS = (
     's3://sdgym-benchmark/Debug/GCP_Github/'  # 's3://sdgym-benchmark/Benchmarks/'
 )
-UPLOAD_DESTINATION_AWS = 's3://sdgym-benchmark/Benchmarks/'
+UPLOAD_DESTINATION_AWS = 's3://sdgym-benchmark/Debug/GCP/'
 DEBUG_SLACK_CHANNEL = 'sdv-alerts-debug'
 SLACK_CHANNEL = 'sdv-alerts'
 KEY_DATE_FILE = '_BENCHMARK_DATES.json'
@@ -48,6 +49,10 @@
     'diamond-cross',
     'diamond-x',
 ]
+MODALITY_TO_GDRIVE_LINK = {
+    'single_table': 'https://docs.google.com/spreadsheets/d/1W3tsGOOtbtTw3g0EVE0irLgY_TN_cy2W4ONiZQ57OPo/edit?usp=sharing',
+    'multi_table': 'https://docs.google.com/spreadsheets/d/1R13RktVvKnxRecYIge07OBpbX1vbEkE2D1_2idNAKSY/edit?usp=sharing',
+}
 
 # The synthesizers inside the same list will be run by the same ec2 instance
 SYNTHESIZERS_SPLIT_SINGLE_TABLE = [
@@ -103,20 +108,22 @@ def post_benchmark_launch_message(date_str, compute_service='AWS', modality='sin
     folder_name = get_result_folder_name(date_str)
     bucket, prefix = parse_s3_path(OUTPUT_DESTINATION_AWS)
     url_link = get_s3_console_link(bucket, f'{prefix}{modality}/{folder_name}/')
-    body = f'🏃 SDGym benchmark has been launched on {compute_service}! '
+    modality_text = modality.replace('_', '-')
+    body = f'🏃 SDGym {modality_text} benchmark has been launched on {compute_service}!\n'
     body += f'Intermediate results can be found <{url_link}|here>.\n'
     post_slack_message(channel, body)
 
 
 def post_benchmark_uploaded_message(folder_name, commit_url=None, modality='single_table'):
     """Post benchmark uploaded message to sdv-alerts slack channel."""
-    channel = SLACK_CHANNEL
+    channel = DEBUG_SLACK_CHANNEL
     bucket, prefix = parse_s3_path(OUTPUT_DESTINATION_AWS)
+    modality_text = modality.replace('_', '-')
     url_link = get_s3_console_link(bucket, quote_plus(f'{prefix}{modality}/SDGym Monthly Run.xlsx'))
     body = (
-        f'🤸🏻‍♀️ SDGym benchmark results for *{folder_name}* are available! 🏋️‍♀️\n'
+        f'🤸🏻‍♀️ SDGym {modality_text} benchmark results for *{folder_name}* are available! 🏋️‍♀️\n'
         f'Check the results:\n'
-        f' - On GDrive: <{GDRIVE_LINK}|link>\n'
+        f' - On GDrive: <{MODALITY_TO_GDRIVE_LINK[modality]}|link>\n'
         f' - On S3: <{url_link}|link>\n'
     )
     if commit_url:
@@ -168,3 +175,14 @@ def get_df_to_plot(benchmark_result):
     df_to_plot = df_to_plot.rename(columns={'Adjusted_Quality_Score': 'Quality_Score'})
 
     return df_to_plot.drop(columns=['Cumulative Quality Score']).reset_index(drop=True)
+
+
+def _parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--modality',
+        choices=['single_table', 'multi_table'],
+        default='single_table',
+        help='Benchmark modality to run.',
+    )
+    return parser.parse_args()

tasks.py

@@ -208,13 +208,13 @@ def run_sdgym_benchmark(c, modality='single_table'):
     c.run(f'python sdgym/run_benchmark/run_benchmark.py --modality {modality}')
 
 @task
-def upload_benchmark_results(c):
+def upload_benchmark_results(c, modality='single_table'):
     """Upload the benchmark results to S3."""
-    c.run(f'python sdgym/run_benchmark/upload_benchmark_results.py')
+    c.run(f'python sdgym/run_benchmark/upload_benchmark_results.py --modality {modality}')
 
 @task
-def notify_sdgym_benchmark_uploaded(c, folder_name, commit_url=None):
+def notify_sdgym_benchmark_uploaded(c, folder_name, commit_url=None, modality='single_table'):
     """Notify Slack about the SDGym benchmark upload."""
     from sdgym.run_benchmark.utils import post_benchmark_uploaded_message
 
-    post_benchmark_uploaded_message(folder_name, commit_url)
+    post_benchmark_uploaded_message(folder_name, commit_url, modality)