Skip to content

Commit c6fe46b

Browse files
sayakpaulsayakpaul
authored
Merge branch 'main' into fix-fast-tests-sayak
2 parents 490f0d1 + 924f880 commit c6fe46b

File tree

750 files changed

+30948
-5384
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

750 files changed

+30948
-5384
lines changed

‎.github/workflows/build_docker_images.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
id: file_changes
3535
uses: jitterbit/get-changed-files@v1
3636
with:
37-
format: 'space-delimited'
37+
format: "space-delimited"
3838
token: ${{ secrets.GITHUB_TOKEN }}
3939

4040
- name: Build Changed Docker Images
@@ -67,6 +67,7 @@ jobs:
6767
- diffusers-pytorch-cuda
6868
- diffusers-pytorch-compile-cuda
6969
- diffusers-pytorch-xformers-cuda
70+
- diffusers-pytorch-minimum-cuda
7071
- diffusers-flax-cpu
7172
- diffusers-flax-tpu
7273
- diffusers-onnxruntime-cpu

‎.github/workflows/nightly_tests.yml‎

Lines changed: 61 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,64 @@ jobs:
235235
run: |
236236
pip install slack_sdk tabulate
237237
python utils/log_reports.py >> $GITHUB_STEP_SUMMARY
238+
239+
torch_minimum_version_cuda_tests:
240+
name: Torch Minimum Version CUDA Tests
241+
runs-on:
242+
group: aws-g4dn-2xlarge
243+
container:
244+
image: diffusers/diffusers-pytorch-minimum-cuda
245+
options: --shm-size "16gb" --ipc host --gpus 0
246+
defaults:
247+
run:
248+
shell: bash
249+
steps:
250+
- name: Checkout diffusers
251+
uses: actions/checkout@v3
252+
with:
253+
fetch-depth: 2
254+
255+
- name: Install dependencies
256+
run: |
257+
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
258+
python -m uv pip install -e [quality,test]
259+
python -m uv pip install peft@git+https://github.com/huggingface/peft.git
260+
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
261+
262+
- name: Environment
263+
run: |
264+
python utils/print_env.py
265+
266+
- name: Run PyTorch CUDA tests
267+
env:
268+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
269+
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
270+
CUBLAS_WORKSPACE_CONFIG: :16:8
271+
run: |
272+
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
273+
-s -v -k "not Flax and not Onnx" \
274+
--make-reports=tests_torch_minimum_version_cuda \
275+
tests/models/test_modeling_common.py \
276+
tests/pipelines/test_pipelines_common.py \
277+
tests/pipelines/test_pipeline_utils.py \
278+
tests/pipelines/test_pipelines.py \
279+
tests/pipelines/test_pipelines_auto.py \
280+
tests/schedulers/test_schedulers.py \
281+
tests/others
282+
283+
- name: Failure short reports
284+
if: ${{ failure() }}
285+
run: |
286+
cat reports/tests_torch_minimum_version_cuda_stats.txt
287+
cat reports/tests_torch_minimum_version_cuda_failures_short.txt
238288
289+
- name: Test suite reports artifacts
290+
if: ${{ always() }}
291+
uses: actions/upload-artifact@v4
292+
with:
293+
name: torch_minimum_version_cuda_test_reports
294+
path: reports
295+
239296
run_flax_tpu_tests:
240297
name: Nightly Flax TPU Tests
241298
runs-on:
@@ -359,6 +416,8 @@ jobs:
359416
test_location: "bnb"
360417
- backend: "gguf"
361418
test_location: "gguf"
419+
- backend: "torchao"
420+
test_location: "torchao"
362421
runs-on:
363422
group: aws-g6e-xlarge-plus
364423
container:
@@ -446,7 +505,7 @@ jobs:
446505
# shell: arch -arch arm64 bash {0}
447506
# env:
448507
# HF_HOME: /System/Volumes/Data/mnt/cache
449-
# HF_TOKEN: ${{ secrets.HF_TOKEN }}
508+
# HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
450509
# run: |
451510
# ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \
452511
# --report-log=tests_torch_mps.log \
@@ -502,7 +561,7 @@ jobs:
502561
# shell: arch -arch arm64 bash {0}
503562
# env:
504563
# HF_HOME: /System/Volumes/Data/mnt/cache
505-
# HF_TOKEN: ${{ secrets.HF_TOKEN }}
564+
# HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
506565
# run: |
507566
# ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \
508567
# --report-log=tests_torch_mps.log \

‎.github/workflows/pr_tests.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,7 @@ jobs:
266266
# TODO (sayakpaul, DN6): revisit `--no-deps`
267267
python -m pip install -U peft@git+https://github.com/huggingface/peft.git --no-deps
268268
python -m uv pip install -U transformers@git+https://github.com/huggingface/transformers.git --no-deps
269+
python -m uv pip install -U tokenizers
269270
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
270271
271272
- name: Environment

‎.github/workflows/push_tests.yml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ jobs:
8383
python utils/print_env.py
8484
- name: PyTorch CUDA checkpoint tests on Ubuntu
8585
env:
86-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
86+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
8787
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
8888
CUBLAS_WORKSPACE_CONFIG: :16:8
8989
run: |
@@ -137,7 +137,7 @@ jobs:
137137
138138
- name: Run PyTorch CUDA tests
139139
env:
140-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
140+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
141141
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
142142
CUBLAS_WORKSPACE_CONFIG: :16:8
143143
run: |
@@ -195,7 +195,7 @@ jobs:
195195
196196
- name: Run Flax TPU tests
197197
env:
198-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
198+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
199199
run: |
200200
python -m pytest -n 0 \
201201
-s -v -k "Flax" \
@@ -243,7 +243,7 @@ jobs:
243243
244244
- name: Run ONNXRuntime CUDA tests
245245
env:
246-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
246+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
247247
run: |
248248
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
249249
-s -v -k "Onnx" \
@@ -291,7 +291,7 @@ jobs:
291291
python utils/print_env.py
292292
- name: Run example tests on GPU
293293
env:
294-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
294+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
295295
RUN_COMPILE: yes
296296
run: |
297297
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/
@@ -334,7 +334,7 @@ jobs:
334334
python utils/print_env.py
335335
- name: Run example tests on GPU
336336
env:
337-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
337+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
338338
run: |
339339
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "xformers" --make-reports=tests_torch_xformers_cuda tests/
340340
- name: Failure short reports
@@ -380,7 +380,7 @@ jobs:
380380
381381
- name: Run example tests on GPU
382382
env:
383-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
383+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
384384
run: |
385385
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
386386
python -m uv pip install timm

‎.github/workflows/pypi_publish.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ jobs:
6868
- name: Test installing diffusers and importing
6969
run: |
7070
pip install diffusers && pip uninstall diffusers -y
71-
pip install -i https://testpypi.python.org/pypi diffusers
71+
pip install -i https://test.pypi.org/simple/ diffusers
7272
python -c "from diffusers import __version__; print(__version__)"
7373
python -c "from diffusers import DiffusionPipeline; pipe = DiffusionPipeline.from_pretrained('fusing/unet-ldm-dummy-update'); pipe()"
7474
python -c "from diffusers import DiffusionPipeline; pipe = DiffusionPipeline.from_pretrained('hf-internal-testing/tiny-stable-diffusion-pipe', safety_checker=None); pipe('ah suh du')"

‎.github/workflows/release_tests_fast.yml‎

Lines changed: 64 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ jobs:
8181
python utils/print_env.py
8282
- name: Slow PyTorch CUDA checkpoint tests on Ubuntu
8383
env:
84-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
84+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
8585
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
8686
CUBLAS_WORKSPACE_CONFIG: :16:8
8787
run: |
@@ -135,7 +135,7 @@ jobs:
135135
136136
- name: Run PyTorch CUDA tests
137137
env:
138-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
138+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
139139
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
140140
CUBLAS_WORKSPACE_CONFIG: :16:8
141141
run: |
@@ -165,6 +165,63 @@ jobs:
165165
name: torch_cuda_${{ matrix.module }}_test_reports
166166
path: reports
167167

168+
torch_minimum_version_cuda_tests:
169+
name: Torch Minimum Version CUDA Tests
170+
runs-on:
171+
group: aws-g4dn-2xlarge
172+
container:
173+
image: diffusers/diffusers-pytorch-minimum-cuda
174+
options: --shm-size "16gb" --ipc host --gpus 0
175+
defaults:
176+
run:
177+
shell: bash
178+
steps:
179+
- name: Checkout diffusers
180+
uses: actions/checkout@v3
181+
with:
182+
fetch-depth: 2
183+
184+
- name: Install dependencies
185+
run: |
186+
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
187+
python -m uv pip install -e [quality,test]
188+
python -m uv pip install peft@git+https://github.com/huggingface/peft.git
189+
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
190+
191+
- name: Environment
192+
run: |
193+
python utils/print_env.py
194+
195+
- name: Run PyTorch CUDA tests
196+
env:
197+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
198+
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
199+
CUBLAS_WORKSPACE_CONFIG: :16:8
200+
run: |
201+
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
202+
-s -v -k "not Flax and not Onnx" \
203+
--make-reports=tests_torch_minimum_cuda \
204+
tests/models/test_modeling_common.py \
205+
tests/pipelines/test_pipelines_common.py \
206+
tests/pipelines/test_pipeline_utils.py \
207+
tests/pipelines/test_pipelines.py \
208+
tests/pipelines/test_pipelines_auto.py \
209+
tests/schedulers/test_schedulers.py \
210+
tests/others
211+
212+
- name: Failure short reports
213+
if: ${{ failure() }}
214+
run: |
215+
cat reports/tests_torch_minimum_version_cuda_stats.txt
216+
cat reports/tests_torch_minimum_version_cuda_failures_short.txt
217+
218+
- name: Test suite reports artifacts
219+
if: ${{ always() }}
220+
uses: actions/upload-artifact@v4
221+
with:
222+
name: torch_minimum_version_cuda_test_reports
223+
path: reports
224+
168225
flax_tpu_tests:
169226
name: Flax TPU Tests
170227
runs-on: docker-tpu
@@ -192,7 +249,7 @@ jobs:
192249
193250
- name: Run slow Flax TPU tests
194251
env:
195-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
252+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
196253
run: |
197254
python -m pytest -n 0 \
198255
-s -v -k "Flax" \
@@ -240,7 +297,7 @@ jobs:
240297
241298
- name: Run slow ONNXRuntime CUDA tests
242299
env:
243-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
300+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
244301
run: |
245302
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
246303
-s -v -k "Onnx" \
@@ -288,7 +345,7 @@ jobs:
288345
python utils/print_env.py
289346
- name: Run example tests on GPU
290347
env:
291-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
348+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
292349
RUN_COMPILE: yes
293350
run: |
294351
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/
@@ -331,7 +388,7 @@ jobs:
331388
python utils/print_env.py
332389
- name: Run example tests on GPU
333390
env:
334-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
391+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
335392
run: |
336393
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "xformers" --make-reports=tests_torch_xformers_cuda tests/
337394
- name: Failure short reports
@@ -377,7 +434,7 @@ jobs:
377434
378435
- name: Run example tests on GPU
379436
env:
380-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
437+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
381438
run: |
382439
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
383440
python -m uv pip install timm

‎.github/workflows/trufflehog.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,6 @@ jobs:
1313
fetch-depth: 0
1414
- name: Secret Scanning
1515
uses: trufflesecurity/trufflehog@main
16+
with:
17+
extra_args: --results=verified,unknown
18+

‎docker/diffusers-pytorch-minimum-cuda/Dockerfile‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
FROM nvidia/cuda:12.1.0-runtime-ubuntu20.04
2+
LABEL maintainer="Hugging Face"
3+
LABEL repository="diffusers"
4+
5+
ENV DEBIAN_FRONTEND=noninteractive
6+
ENV MINIMUM_SUPPORTED_TORCH_VERSION="2.1.0"
7+
ENV MINIMUM_SUPPORTED_TORCHVISION_VERSION="0.16.0"
8+
ENV MINIMUM_SUPPORTED_TORCHAUDIO_VERSION="2.1.0"
9+
10+
RUN apt-get -y update \
11+
&& apt-get install -y software-properties-common \
12+
&& add-apt-repository ppa:deadsnakes/ppa
13+
14+
RUN apt install -y bash \
15+
build-essential \
16+
git \
17+
git-lfs \
18+
curl \
19+
ca-certificates \
20+
libsndfile1-dev \
21+
libgl1 \
22+
python3.10 \
23+
python3.10-dev \
24+
python3-pip \
25+
python3.10-venv && \
26+
rm -rf /var/lib/apt/lists
27+
28+
# make sure to use venv
29+
RUN python3.10 -m venv /opt/venv
30+
ENV PATH="/opt/venv/bin:$PATH"
31+
32+
# pre-install the heavy dependencies (these can later be overridden by the deps from setup.py)
33+
RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \
34+
python3.10 -m uv pip install --no-cache-dir \
35+
torch==$MINIMUM_SUPPORTED_TORCH_VERSION \
36+
torchvision==$MINIMUM_SUPPORTED_TORCHVISION_VERSION \
37+
torchaudio==$MINIMUM_SUPPORTED_TORCHAUDIO_VERSION \
38+
invisible_watermark && \
39+
python3.10 -m pip install --no-cache-dir \
40+
accelerate \
41+
datasets \
42+
hf-doc-builder \
43+
huggingface-hub \
44+
hf_transfer \
45+
Jinja2 \
46+
librosa \
47+
numpy==1.26.4 \
48+
scipy \
49+
tensorboard \
50+
transformers \
51+
hf_transfer
52+
53+
CMD ["/bin/bash"]

0 commit comments

Comments
 (0)