Benchmarking workflow fix (#8389)

* fix

* fixes

* add back the deadsnakes

* better messaging

* disable IP adapter tests for the moment.

* style

* up

* empty

Author: sayakpaul

.github/workflows/benchmark.yml

@@ -13,14 +13,16 @@ env:
 
 jobs:
   torch_pipelines_cuda_benchmark_tests:
+    env: 
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_BENCHMARK }}
     name: Torch Core Pipelines CUDA Benchmarking Tests
     strategy:
       fail-fast: false
       max-parallel: 1
     runs-on: [single-gpu, nvidia-gpu, a10, ci]
     container:
-      image: diffusers/diffusers-pytorch-cuda
-      options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/ --gpus 0
+      image: diffusers/diffusers-pytorch-compile-cuda
+      options: --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface/diffusers:/mnt/cache/ --gpus 0
     steps:
       - name: Checkout diffusers
         uses: actions/checkout@v3
@@ -50,4 +52,14 @@ jobs:
         uses: actions/upload-artifact@v2
         with:
           name: benchmark_test_reports
-          path: benchmarks/benchmark_outputs
+          path: benchmarks/benchmark_outputs
+
+      - name: Report success status
+        if: ${{ success() }}
+        run: |
+          pip install requests && python utils/notify_benchmarking_status.py --status=success
+        
+      - name: Report failure status
+        if: ${{ failure() }}
+        run: |
+          pip install requests && python utils/notify_benchmarking_status.py --status=failure

benchmarks/run_all.py

@@ -39,6 +39,9 @@ def main():
     for file in python_files:
         print(f"****** Running file: {file} ******")
 
+        if "ip_adapters" in file:
+            continue
+
         # Run with canonical settings.
         if file != "benchmark_text_to_image.py":
             command = f"python {file}"
@@ -49,6 +52,9 @@ def main():
 
     # Run variants.
     for file in python_files:
+        if "ip_adapters" in file:
+            continue
+
         if file == "benchmark_text_to_image.py":
             for ckpt in ALL_T2I_CKPTS:
                 command = f"python {file} --ckpt {ckpt}"

docker/diffusers-pytorch-compile-cuda/Dockerfile

@@ -16,23 +16,24 @@ RUN apt install -y bash \
     ca-certificates \
     libsndfile1-dev \
     libgl1 \
-    python3.10 \
+    python3.9 \
+    python3.9-dev \
     python3-pip \
-    python3.10-venv && \
+    python3.9-venv && \
     rm -rf /var/lib/apt/lists
 
 # make sure to use venv
-RUN python3.10 -m venv /opt/venv
+RUN python3.9 -m venv /opt/venv
 ENV PATH="/opt/venv/bin:$PATH"
 
 # pre-install the heavy dependencies (these can later be overridden by the deps from setup.py)
-RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \
-    python3.10 -m uv pip install --no-cache-dir \
+RUN python3.9 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \
+    python3.9 -m uv pip install --no-cache-dir \
     torch \
     torchvision \
     torchaudio \
     invisible_watermark && \
-    python3.10 -m pip install --no-cache-dir \
+    python3.9 -m pip install --no-cache-dir \
     accelerate \
     datasets \
     hf-doc-builder \

utils/notify_benchmarking_status.py

@@ -0,0 +1,56 @@
+# coding=utf-8
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+
+import requests
+
+
+# Configuration
+GITHUB_REPO = "huggingface/diffusers"
+GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID")
+SLACK_WEBHOOK_URL = os.getenv("SLACK_WEBHOOK_URL")
+
+
+def main(args):
+    action_url = f"https://github.com/{GITHUB_REPO}/actions/runs/{GITHUB_RUN_ID}"
+    if args.status == "success":
+        hub_path = "https://huggingface.co/datasets/diffusers/benchmarks/blob/main/collated_results.csv"
+        message = (
+            "✅ New benchmark workflow successfully run.\n"
+            f"🕸️ GitHub Action URL: {action_url}.\n"
+            f"🤗 Check out the benchmarks here: {hub_path}."
+        )
+    else:
+        message = (
+            "❌ Something wrong happened in the benchmarking workflow.\n"
+            f"Check out the GitHub Action to know more: {action_url}."
+        )
+
+    payload = {"text": message}
+    response = requests.post(SLACK_WEBHOOK_URL, json=payload)
+
+    if response.status_code == 200:
+        print("Notification sent to Slack successfully.")
+    else:
+        print("Failed to send notification to Slack.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--status", type=str, default="success", choices=["success", "failure"])
+    args = parser.parse_args()
+    main(args)