update uploading workflow

Author: R-Palazzo

.github/workflows/upload_benchmark_results.yml

@@ -1,6 +1,9 @@
 name: Upload SDGym Benchmark results
 
 on:
+  push:
+    branches:
+      - issue-425-workflow-sdgym
   workflow_run:
     workflows: ["Run SDGym Benchmark"]
     types:
@@ -28,10 +31,37 @@ jobs:
           
     - name: Upload SDGym Benchmark
       env:
-        SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
-        
+        GITHUB_LOCAL_RESULTS_DIR: ${{ runner.temp }}/sdgym-leaderboard-files
+
       run: |
         invoke upload-benchmark-results
+    - name: Create pull request for SDGym benchmark result
+      id: create_pr
+      if: env.SKIP_UPLOAD != 'true'
+      uses: peter-evans/create-pull-request@v4
+      with:
+        token: ${{ secrets.GH_ACCESS_TOKEN }}
+        commit-message: "Upload SDGym Benchmark Results (${{ env.FOLDER_NAME }})"
+        author: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>"
+        committer: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>"
+        title: "Upload SDGym Benchmark Results - ${{ env.FOLDER_NAME }}"
+        body: |
+          This is an **auto-generated PR** for uploading SDGym benchmark latest results.
+
+        branch: sdgym-benchmark-upload
+        branch-suffix: short-commit-hash
+        base: gatsby-home
+
+    - name: Send Slack notification
+      if: env.SKIP_UPLOAD != 'true'
+      env:
+        SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
+        
+      run: |
+        invoke notify-sdgym-benchmark-uploaded \
+          --folder-name ${{ env.FOLDER_NAME }} \
+          --pr-url "${{ steps.create_pr.outputs.pull-request-url }}"
+

sdgym/result_writer.py

@@ -33,11 +33,11 @@ def write_yaml(self, data, file_path, append=False):
 class LocalResultsWriter(ResultsWriter):
     """Results writer for local file system."""
 
-    def write_dataframe(self, data, file_path, append=False):
+    def write_dataframe(self, data, file_path, append=False, index=False):
         """Write a DataFrame to a CSV file."""
         file_path = Path(file_path)
         if file_path.exists() and append:
-            data.to_csv(file_path, mode='a', index=False, header=False)
+            data.to_csv(file_path, mode='a', index=index, header=False)
         else:
             data.to_csv(file_path, mode='w', index=False)
 

sdgym/run_benchmark/upload_benchmark_results.py

@@ -8,7 +8,7 @@
 import boto3
 from botocore.exceptions import ClientError
 
-from sdgym.result_writer import S3ResultsWriter
+from sdgym.result_writer import LocalResultsWriter, S3ResultsWriter
 from sdgym.run_benchmark.utils import OUTPUT_DESTINATION_AWS
 from sdgym.s3 import S3_REGION, parse_s3_path
 from sdgym.sdgym_result_explorer.result_explorer import SDGymResultsExplorer
@@ -62,7 +62,7 @@ def get_result_folder_name_and_s3_vars(aws_access_key_id, aws_secret_access_key)
 
 
 def upload_results(
-    aws_access_key_id, aws_secret_access_key, folder_name, s3_client, bucket, prefix
+    aws_access_key_id, aws_secret_access_key, folder_name, s3_client, bucket, prefix, github_env
 ):
     """Upload benchmark results to S3."""
     result_explorer = SDGymResultsExplorer(
@@ -71,16 +71,31 @@ def upload_results(
         aws_secret_access_key=aws_secret_access_key,
     )
     result_writer = S3ResultsWriter(s3_client)
-
+    local_results_writer = LocalResultsWriter()
     if not result_explorer.all_runs_complete(folder_name):
         LOGGER.warning(f'Run {folder_name} is not complete yet. Exiting.')
+        if github_env:
+            with open(github_env, 'a') as env_file:
+                env_file.write('SKIP_UPLOAD=true\n')
+
         sys.exit(0)
+    else:
+        LOGGER.info(f'Run {folder_name} is complete! Proceeding with summarization...')
+        if github_env:
+            with open(github_env, 'a') as env_file:
+                env_file.write('SKIP_UPLOAD=false\n')
+                env_file.write(f'FOLDER_NAME={folder_name}\n')
 
-    LOGGER.info(f'Run {folder_name} is complete! Proceeding with summarization...')
     summary, _ = result_explorer.summarize(folder_name)
     result_writer.write_dataframe(
         summary, f'{OUTPUT_DESTINATION_AWS}{folder_name}/{folder_name}_summary.csv', index=True
     )
+    local_export_dir = os.environ.get('GITHUB_LOCAL_RESULTS_DIR')
+    if local_export_dir:
+        local_results_writer.write_dataframe(
+            summary, f'{local_export_dir}/{folder_name}_summary.csv', index=True
+        )
+
     write_uploaded_marker(s3_client, bucket, prefix, folder_name)
 
 
@@ -91,11 +106,18 @@ def main():
     folder_name, s3_client, bucket, prefix = get_result_folder_name_and_s3_vars(
         aws_access_key_id, aws_secret_access_key
     )
+    github_env = os.environ.get('GITHUB_ENV')
     if upload_already_done(s3_client, bucket, prefix, folder_name):
         LOGGER.warning('Benchmark results have already been uploaded. Exiting.')
+        if github_env:
+            with open(github_env, 'a') as env_file:
+                env_file.write('SKIP_UPLOAD=true\n')
+
         sys.exit(0)
 
-    upload_results(aws_access_key_id, aws_secret_access_key, folder_name, s3_client, bucket, prefix)
+    upload_results(
+        aws_access_key_id, aws_secret_access_key, folder_name, s3_client, bucket, prefix, github_env
+    )
 
 
 if __name__ == '__main__':

sdgym/run_benchmark/utils.py

@@ -68,17 +68,20 @@ def post_benchmark_launch_message(date_str):
     bucket, prefix = parse_s3_path(OUTPUT_DESTINATION_AWS)
     url_link = get_s3_console_link(bucket, f'{prefix}{folder_name}/')
     body = '🏃 SDGym benchmark has been launched! EC2 Instances are running. '
-    body += f'Intermediate results can be found <<{url_link} |here>.\n'
+    body += f'Intermediate results can be found <{url_link} |here>.\n'
     post_slack_message(channel, body)
 
 
-def post_run_summary(folder_name):
-    """Post run summary to sdv-alerts slack channel."""
+def post_benchmark_uploaded_message(folder_name, pr_url=None):
+    """Post benchmark uploaded message to sdv-alerts slack channel."""
     channel = DEBUG_SLACK_CHANNEL
     bucket, prefix = parse_s3_path(OUTPUT_DESTINATION_AWS)
     url_link = get_s3_console_link(bucket, f'{prefix}{folder_name}/{folder_name}_summary.csv')
-    body = ''
-    body += f'🤸🏻‍♀️ SDGym benchmark results for {folder_name} are available!🏋️‍♀️ \n'
-    body += f'Check the results <<{url_link} |here>.\n'
+    body = (
+        f'🤸🏻‍♀️ SDGym benchmark results for *{folder_name}* are available! 🏋️‍♀️\n'
+        f'Check the results <{url_link} |here>.\n'
+    )
+    if pr_url:
+        body += f'Waiting on merging this PR to update GitHub directory: <{pr_url}|PR Link>\n'
 
     post_slack_message(channel, body)

tasks.py

@@ -10,7 +10,7 @@
 from invoke import task
 from packaging.requirements import Requirement
 from packaging.version import Version
-
+from sdgym.run_benchmark.utils import post_benchmark_uploaded_message
 COMPARISONS = {'>=': operator.ge, '>': operator.gt, '<': operator.lt, '<=': operator.le}
 EGG_STRING = '#egg='
 
@@ -209,6 +209,11 @@ def run_sdgym_benchmark(c):
     c.run('python sdgym/run_benchmark/run_benchmark.py')
 
 @task
-def upload_benchmark_results(c, date=None):
+def upload_benchmark_results(c):
     """Upload the benchmark results to S3."""
-    c.run(f'python sdgym/run_benchmark/upload_benchmark_results.py {date}')
+    c.run(f'python sdgym/run_benchmark/upload_benchmark_results.py')
+
+@task
+def notify_sdgym_benchmark_uploaded(c, folder_name, pr_url=None):
+    """Notify Slack about the SDGym benchmark upload."""
+    post_benchmark_uploaded_message(folder_name, pr_url)

tests/unit/run_benchmark/test_upload_benchmark_result.py

@@ -101,7 +101,11 @@ def test_get_result_folder_name_and_s3_vars(
 @patch('sdgym.run_benchmark.upload_benchmark_results.write_uploaded_marker')
 @patch('sdgym.run_benchmark.upload_benchmark_results.LOGGER')
 @patch('sdgym.run_benchmark.upload_benchmark_results.OUTPUT_DESTINATION_AWS')
+@patch('sdgym.run_benchmark.upload_benchmark_results.LocalResultsWriter')
+@patch('sdgym.run_benchmark.upload_benchmark_results.os.environ.get')
 def test_upload_results(
+    mock_os_environ_get,
+    mock_local_results_writer,
     mock_output_destination_aws,
     mock_logger,
     mock_write_uploaded_marker,
@@ -119,9 +123,18 @@ def test_upload_results(
     result_explorer_instance = mock_sdgym_results_explorer.return_value
     result_explorer_instance.all_runs_complete.return_value = True
     result_explorer_instance.summarize.return_value = ('summary', 'results')
+    mock_os_environ_get.return_value = '/tmp/sdgym_results'
 
     # Run
-    upload_results(aws_access_key_id, aws_secret_access_key, run_name, s3_client, bucket, prefix)
+    upload_results(
+        aws_access_key_id,
+        aws_secret_access_key,
+        run_name,
+        s3_client,
+        bucket,
+        prefix,
+        github_env=None,
+    )
 
     # Assert
     mock_logger.info.assert_called_once_with(
@@ -136,6 +149,9 @@ def test_upload_results(
     result_explorer_instance.summarize.assert_called_once_with(run_name)
     mock_s3_results_writer.return_value.write_dataframe.assert_called_once()
     mock_write_uploaded_marker.assert_called_once_with(s3_client, bucket, prefix, run_name)
+    mock_local_results_writer.return_value.write_dataframe.assert_called_once_with(
+        'summary', '/tmp/sdgym_results/SDGym_results_10_01_2023_summary.csv', index=True
+    )
 
 
 @patch('sdgym.run_benchmark.upload_benchmark_results.SDGymResultsExplorer')
@@ -165,7 +181,13 @@ def test_upload_results_not_all_runs_complete(
     # Run
     with pytest.raises(SystemExit, match='0'):
         upload_results(
-            aws_access_key_id, aws_secret_access_key, run_name, s3_client, bucket, prefix
+            aws_access_key_id,
+            aws_secret_access_key,
+            run_name,
+            s3_client,
+            bucket,
+            prefix,
+            github_env=None,
         )
 
     # Assert
@@ -247,5 +269,5 @@ def test_main(
     )
     mock_upload_already_done.assert_called_once_with('s3_client', 'bucket', 'prefix', 'run_name')
     mock_upload_results.assert_called_once_with(
-        'my_access_key', 'my_secret_key', 'run_name', 's3_client', 'bucket', 'prefix'
+        'my_access_key', 'my_secret_key', 'run_name', 's3_client', 'bucket', 'prefix', None
     )