Skip to content

Commit bd26f40

Browse files
sayakpaulsayakpaul
authored
Merge branch 'main' into use-tranaformers-main-pr
2 parents d733102 + 924f880 commit bd26f40

File tree

767 files changed

+32879
-5708
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

767 files changed

+32879
-5708
lines changed

‎.github/workflows/build_docker_images.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
id: file_changes
3535
uses: jitterbit/get-changed-files@v1
3636
with:
37-
format: 'space-delimited'
37+
format: "space-delimited"
3838
token: ${{ secrets.GITHUB_TOKEN }}
3939

4040
- name: Build Changed Docker Images
@@ -67,6 +67,7 @@ jobs:
6767
- diffusers-pytorch-cuda
6868
- diffusers-pytorch-compile-cuda
6969
- diffusers-pytorch-xformers-cuda
70+
- diffusers-pytorch-minimum-cuda
7071
- diffusers-flax-cpu
7172
- diffusers-flax-tpu
7273
- diffusers-onnxruntime-cpu

‎.github/workflows/nightly_tests.yml‎

Lines changed: 61 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,64 @@ jobs:
235235
run: |
236236
pip install slack_sdk tabulate
237237
python utils/log_reports.py >> $GITHUB_STEP_SUMMARY
238+
239+
torch_minimum_version_cuda_tests:
240+
name: Torch Minimum Version CUDA Tests
241+
runs-on:
242+
group: aws-g4dn-2xlarge
243+
container:
244+
image: diffusers/diffusers-pytorch-minimum-cuda
245+
options: --shm-size "16gb" --ipc host --gpus 0
246+
defaults:
247+
run:
248+
shell: bash
249+
steps:
250+
- name: Checkout diffusers
251+
uses: actions/checkout@v3
252+
with:
253+
fetch-depth: 2
254+
255+
- name: Install dependencies
256+
run: |
257+
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
258+
python -m uv pip install -e [quality,test]
259+
python -m uv pip install peft@git+https://github.com/huggingface/peft.git
260+
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
261+
262+
- name: Environment
263+
run: |
264+
python utils/print_env.py
265+
266+
- name: Run PyTorch CUDA tests
267+
env:
268+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
269+
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
270+
CUBLAS_WORKSPACE_CONFIG: :16:8
271+
run: |
272+
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
273+
-s -v -k "not Flax and not Onnx" \
274+
--make-reports=tests_torch_minimum_version_cuda \
275+
tests/models/test_modeling_common.py \
276+
tests/pipelines/test_pipelines_common.py \
277+
tests/pipelines/test_pipeline_utils.py \
278+
tests/pipelines/test_pipelines.py \
279+
tests/pipelines/test_pipelines_auto.py \
280+
tests/schedulers/test_schedulers.py \
281+
tests/others
282+
283+
- name: Failure short reports
284+
if: ${{ failure() }}
285+
run: |
286+
cat reports/tests_torch_minimum_version_cuda_stats.txt
287+
cat reports/tests_torch_minimum_version_cuda_failures_short.txt
238288
289+
- name: Test suite reports artifacts
290+
if: ${{ always() }}
291+
uses: actions/upload-artifact@v4
292+
with:
293+
name: torch_minimum_version_cuda_test_reports
294+
path: reports
295+
239296
run_flax_tpu_tests:
240297
name: Nightly Flax TPU Tests
241298
runs-on:
@@ -359,6 +416,8 @@ jobs:
359416
test_location: "bnb"
360417
- backend: "gguf"
361418
test_location: "gguf"
419+
- backend: "torchao"
420+
test_location: "torchao"
362421
runs-on:
363422
group: aws-g6e-xlarge-plus
364423
container:
@@ -446,7 +505,7 @@ jobs:
446505
# shell: arch -arch arm64 bash {0}
447506
# env:
448507
# HF_HOME: /System/Volumes/Data/mnt/cache
449-
# HF_TOKEN: ${{ secrets.HF_TOKEN }}
508+
# HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
450509
# run: |
451510
# ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \
452511
# --report-log=tests_torch_mps.log \
@@ -502,7 +561,7 @@ jobs:
502561
# shell: arch -arch arm64 bash {0}
503562
# env:
504563
# HF_HOME: /System/Volumes/Data/mnt/cache
505-
# HF_TOKEN: ${{ secrets.HF_TOKEN }}
564+
# HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
506565
# run: |
507566
# ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \
508567
# --report-log=tests_torch_mps.log \

‎.github/workflows/pr_tests.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,7 @@ jobs:
267267
# TODO (sayakpaul, DN6): revisit `--no-deps`
268268
python -m pip install -U peft@git+https://github.com/huggingface/peft.git --no-deps
269269
python -m uv pip install -U transformers@git+https://github.com/huggingface/transformers.git --no-deps
270+
python -m uv pip install -U tokenizers
270271
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
271272
272273
- name: Environment

‎.github/workflows/push_tests.yml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ jobs:
8383
python utils/print_env.py
8484
- name: PyTorch CUDA checkpoint tests on Ubuntu
8585
env:
86-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
86+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
8787
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
8888
CUBLAS_WORKSPACE_CONFIG: :16:8
8989
run: |
@@ -137,7 +137,7 @@ jobs:
137137
138138
- name: Run PyTorch CUDA tests
139139
env:
140-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
140+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
141141
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
142142
CUBLAS_WORKSPACE_CONFIG: :16:8
143143
run: |
@@ -187,7 +187,7 @@ jobs:
187187
188188
- name: Run Flax TPU tests
189189
env:
190-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
190+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
191191
run: |
192192
python -m pytest -n 0 \
193193
-s -v -k "Flax" \
@@ -235,7 +235,7 @@ jobs:
235235
236236
- name: Run ONNXRuntime CUDA tests
237237
env:
238-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
238+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
239239
run: |
240240
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
241241
-s -v -k "Onnx" \
@@ -283,7 +283,7 @@ jobs:
283283
python utils/print_env.py
284284
- name: Run example tests on GPU
285285
env:
286-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
286+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
287287
RUN_COMPILE: yes
288288
run: |
289289
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/
@@ -326,7 +326,7 @@ jobs:
326326
python utils/print_env.py
327327
- name: Run example tests on GPU
328328
env:
329-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
329+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
330330
run: |
331331
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "xformers" --make-reports=tests_torch_xformers_cuda tests/
332332
- name: Failure short reports
@@ -372,7 +372,7 @@ jobs:
372372
373373
- name: Run example tests on GPU
374374
env:
375-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
375+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
376376
run: |
377377
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
378378
python -m uv pip install timm

‎.github/workflows/push_tests_mps.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
shell: arch -arch arm64 bash {0}
4747
run: |
4848
${CONDA_RUN} python -m pip install --upgrade pip uv
49-
${CONDA_RUN} python -m uv pip install -e [quality,test]
49+
${CONDA_RUN} python -m uv pip install -e ".[quality,test]"
5050
${CONDA_RUN} python -m uv pip install torch torchvision torchaudio
5151
${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git
5252
${CONDA_RUN} python -m uv pip install transformers --upgrade

‎.github/workflows/pypi_publish.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ jobs:
6868
- name: Test installing diffusers and importing
6969
run: |
7070
pip install diffusers && pip uninstall diffusers -y
71-
pip install -i https://testpypi.python.org/pypi diffusers
71+
pip install -i https://test.pypi.org/simple/ diffusers
7272
python -c "from diffusers import __version__; print(__version__)"
7373
python -c "from diffusers import DiffusionPipeline; pipe = DiffusionPipeline.from_pretrained('fusing/unet-ldm-dummy-update'); pipe()"
7474
python -c "from diffusers import DiffusionPipeline; pipe = DiffusionPipeline.from_pretrained('hf-internal-testing/tiny-stable-diffusion-pipe', safety_checker=None); pipe('ah suh du')"

‎.github/workflows/release_tests_fast.yml‎

Lines changed: 64 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ jobs:
8181
python utils/print_env.py
8282
- name: Slow PyTorch CUDA checkpoint tests on Ubuntu
8383
env:
84-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
84+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
8585
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
8686
CUBLAS_WORKSPACE_CONFIG: :16:8
8787
run: |
@@ -135,7 +135,7 @@ jobs:
135135
136136
- name: Run PyTorch CUDA tests
137137
env:
138-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
138+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
139139
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
140140
CUBLAS_WORKSPACE_CONFIG: :16:8
141141
run: |
@@ -157,6 +157,63 @@ jobs:
157157
name: torch_cuda_${{ matrix.module }}_test_reports
158158
path: reports
159159

160+
torch_minimum_version_cuda_tests:
161+
name: Torch Minimum Version CUDA Tests
162+
runs-on:
163+
group: aws-g4dn-2xlarge
164+
container:
165+
image: diffusers/diffusers-pytorch-minimum-cuda
166+
options: --shm-size "16gb" --ipc host --gpus 0
167+
defaults:
168+
run:
169+
shell: bash
170+
steps:
171+
- name: Checkout diffusers
172+
uses: actions/checkout@v3
173+
with:
174+
fetch-depth: 2
175+
176+
- name: Install dependencies
177+
run: |
178+
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
179+
python -m uv pip install -e [quality,test]
180+
python -m uv pip install peft@git+https://github.com/huggingface/peft.git
181+
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
182+
183+
- name: Environment
184+
run: |
185+
python utils/print_env.py
186+
187+
- name: Run PyTorch CUDA tests
188+
env:
189+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
190+
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
191+
CUBLAS_WORKSPACE_CONFIG: :16:8
192+
run: |
193+
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
194+
-s -v -k "not Flax and not Onnx" \
195+
--make-reports=tests_torch_minimum_cuda \
196+
tests/models/test_modeling_common.py \
197+
tests/pipelines/test_pipelines_common.py \
198+
tests/pipelines/test_pipeline_utils.py \
199+
tests/pipelines/test_pipelines.py \
200+
tests/pipelines/test_pipelines_auto.py \
201+
tests/schedulers/test_schedulers.py \
202+
tests/others
203+
204+
- name: Failure short reports
205+
if: ${{ failure() }}
206+
run: |
207+
cat reports/tests_torch_minimum_version_cuda_stats.txt
208+
cat reports/tests_torch_minimum_version_cuda_failures_short.txt
209+
210+
- name: Test suite reports artifacts
211+
if: ${{ always() }}
212+
uses: actions/upload-artifact@v4
213+
with:
214+
name: torch_minimum_version_cuda_test_reports
215+
path: reports
216+
160217
flax_tpu_tests:
161218
name: Flax TPU Tests
162219
runs-on: docker-tpu
@@ -184,7 +241,7 @@ jobs:
184241
185242
- name: Run slow Flax TPU tests
186243
env:
187-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
244+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
188245
run: |
189246
python -m pytest -n 0 \
190247
-s -v -k "Flax" \
@@ -232,7 +289,7 @@ jobs:
232289
233290
- name: Run slow ONNXRuntime CUDA tests
234291
env:
235-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
292+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
236293
run: |
237294
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
238295
-s -v -k "Onnx" \
@@ -280,7 +337,7 @@ jobs:
280337
python utils/print_env.py
281338
- name: Run example tests on GPU
282339
env:
283-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
340+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
284341
RUN_COMPILE: yes
285342
run: |
286343
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/
@@ -323,7 +380,7 @@ jobs:
323380
python utils/print_env.py
324381
- name: Run example tests on GPU
325382
env:
326-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
383+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
327384
run: |
328385
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "xformers" --make-reports=tests_torch_xformers_cuda tests/
329386
- name: Failure short reports
@@ -369,7 +426,7 @@ jobs:
369426
370427
- name: Run example tests on GPU
371428
env:
372-
HF_TOKEN: ${{ secrets.HF_TOKEN }}
429+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
373430
run: |
374431
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
375432
python -m uv pip install timm

‎.github/workflows/trufflehog.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,6 @@ jobs:
1313
fetch-depth: 0
1414
- name: Secret Scanning
1515
uses: trufflesecurity/trufflehog@main
16+
with:
17+
extra_args: --results=verified,unknown
18+

‎docker/diffusers-pytorch-minimum-cuda/Dockerfile‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
FROM nvidia/cuda:12.1.0-runtime-ubuntu20.04
2+
LABEL maintainer="Hugging Face"
3+
LABEL repository="diffusers"
4+
5+
ENV DEBIAN_FRONTEND=noninteractive
6+
ENV MINIMUM_SUPPORTED_TORCH_VERSION="2.1.0"
7+
ENV MINIMUM_SUPPORTED_TORCHVISION_VERSION="0.16.0"
8+
ENV MINIMUM_SUPPORTED_TORCHAUDIO_VERSION="2.1.0"
9+
10+
RUN apt-get -y update \
11+
&& apt-get install -y software-properties-common \
12+
&& add-apt-repository ppa:deadsnakes/ppa
13+
14+
RUN apt install -y bash \
15+
build-essential \
16+
git \
17+
git-lfs \
18+
curl \
19+
ca-certificates \
20+
libsndfile1-dev \
21+
libgl1 \
22+
python3.10 \
23+
python3.10-dev \
24+
python3-pip \
25+
python3.10-venv && \
26+
rm -rf /var/lib/apt/lists
27+
28+
# make sure to use venv
29+
RUN python3.10 -m venv /opt/venv
30+
ENV PATH="/opt/venv/bin:$PATH"
31+
32+
# pre-install the heavy dependencies (these can later be overridden by the deps from setup.py)
33+
RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \
34+
python3.10 -m uv pip install --no-cache-dir \
35+
torch==$MINIMUM_SUPPORTED_TORCH_VERSION \
36+
torchvision==$MINIMUM_SUPPORTED_TORCHVISION_VERSION \
37+
torchaudio==$MINIMUM_SUPPORTED_TORCHAUDIO_VERSION \
38+
invisible_watermark && \
39+
python3.10 -m pip install --no-cache-dir \
40+
accelerate \
41+
datasets \
42+
hf-doc-builder \
43+
huggingface-hub \
44+
hf_transfer \
45+
Jinja2 \
46+
librosa \
47+
numpy==1.26.4 \
48+
scipy \
49+
tensorboard \
50+
transformers \
51+
hf_transfer
52+
53+
CMD ["/bin/bash"]

0 commit comments

Comments
 (0)