Add animate diff (#174)

Author: tastelikefeet

examples/pytorch/animatediff/animatediff_infer.py

@@ -0,0 +1,6 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from swift.llm.run import animatediff_infer_main
+
+if __name__ == '__main__':
+    animatediff_infer_main()

examples/pytorch/animatediff/animatediff_sft.py

@@ -0,0 +1,6 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from swift.llm.run import animatediff_main
+
+if __name__ == '__main__':
+    animatediff_main()

examples/pytorch/animatediff/scripts/full/infer.sh

@@ -0,0 +1,9 @@
+# Experimental environment: A100
+# 18GB GPU memory
+PYTHONPATH=../../.. \
+CUDA_VISIBLE_DEVICES=0 \
+python animatediff_infer.py \
+  --model_id_or_path wyj123456/Realistic_Vision_V5.1_noVAE \
+  --sft_type full \
+  --ckpt_dir /output/path/like/checkpoints/iter-xxx \
+  --eval_human true  \

examples/pytorch/animatediff/scripts/full/sft.sh

@@ -0,0 +1,14 @@
+# Experimental environment: A100 * 4
+# 200GB GPU memory totally
+PYTHONPATH=../../.. \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+torchrun --nproc_per_node=4 animatediff_sft.py \
+  --model_id_or_path wyj123456/Realistic_Vision_V5.1_noVAE \
+  --csv_path /mnt/workspace/yzhao/tastelikefeet/webvid/results_2M_train.csv \
+  --video_folder /mnt/workspace/yzhao/tastelikefeet/webvid/videos2 \
+  --sft_type full \
+  --lr_scheduler_type constant \
+  --trainable_modules .*motion_modules.* \
+  --batch_size 4 \
+  --eval_steps 100 \
+  --gradient_accumulation_steps 16 \

examples/pytorch/animatediff/scripts/lora/infer.sh

@@ -0,0 +1,10 @@
+# Experimental environment: A100
+# 18GB GPU memory
+PYTHONPATH=../../.. \
+CUDA_VISIBLE_DEVICES=0 \
+python animatediff_infer.py \
+  --model_id_or_path wyj123456/Realistic_Vision_V5.1_noVAE \
+  --motion_adapter_id_or_path Shanghai_AI_Laboratory/animatediff-motion-adapter-v1-5-2 \
+  --sft_type lora \
+  --ckpt_dir /output/path/like/checkpoints/iter-xxx \
+  --eval_human true  \

examples/pytorch/animatediff/scripts/lora/sft.sh

@@ -0,0 +1,16 @@
+# Experimental environment: A100
+# 20GB GPU memory
+PYTHONPATH=../../.. \
+CUDA_VISIBLE_DEVICES=0 \
+python animatediff_sft.py \
+  --model_id_or_path wyj123456/Realistic_Vision_V5.1_noVAE \
+  --csv_path /mnt/workspace/yzhao/tastelikefeet/webvid/results_2M_train.csv \
+  --video_folder /mnt/workspace/yzhao/tastelikefeet/webvid/videos2 \
+  --motion_adapter_id_or_path Shanghai_AI_Laboratory/animatediff-motion-adapter-v1-5-2 \
+  --sft_type lora \
+  --lr_scheduler_type constant \
+  --trainable_modules .*motion_modules.* \
+  --batch_size 1 \
+  --eval_steps 200 \
+  --dataset_sample_size 10000 \
+  --gradient_accumulation_steps 16 \

swift/llm/__init__.py

@@ -1,4 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+from .animatediff import animatediff_sft
+from .animatediff_infer import animatediff_infer
 from .infer import llm_infer, merge_lora, prepare_model_template
 from .rome import rome_infer
 from .sft import llm_sft