Add AniMemoryPipeline

docs/source/en/_toctree.yml

@@ -338,6 +338,8 @@
       title: aMUSEd
     - local: api/pipelines/animatediff
       title: AnimateDiff
+    - local: api/pipelines/animemory
+      title: AniMemory-alpha
     - local: api/pipelines/attend_and_excite
       title: Attend-and-Excite
     - local: api/pipelines/audioldm
@@ -523,6 +525,8 @@
       title: EDMEulerScheduler
     - local: api/schedulers/euler_ancestral
       title: EulerAncestralDiscreteScheduler
+    - local: api/schedulers/euler_ancestral_x_pred
+      title: EulerAncestralDiscreteXPredScheduler
     - local: api/schedulers/euler
       title: EulerDiscreteScheduler
     - local: api/schedulers/flow_match_euler_discrete

docs/source/en/api/pipelines/animemory.md

@@ -0,0 +1,58 @@
+<!--Copyright 2024 The HuggingFace Team and AniMemory Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+-->
+
+# AniMemory-alpha
+![img](https://github.com/animEEEmpire/AniMemory-alpha/raw/main/gallery_demo.png)
+
+
+
+[Animemory Alpha](https://huggingface.co/animEEEmpire/AniMemory-alpha) is a bilingual model primarily focused on anime-style image generation. It utilizes a SDXL-type Unet
+structure and a self-developed bilingual T5-XXL text encoder, achieving good alignment between Chinese and English. We
+first developed our general model using billion-level data and then tuned the anime model through a series of
+post-training strategies and curated data. By open-sourcing the Alpha version, we hope to contribute to the development
+of the anime community, and we greatly value any feedback.
+
+### Usage
+```
+from diffusers import AniMemoryPipeline
+import torch
+
+pipe = AniMemoryPipeline.from_pretrained("animEEEmpire/AniMemory-alpha", torch_dtype=torch.bfloat16)
+pipe.to("cuda")
+
+prompt = "一只凶恶的狼，猩红的眼神，在午夜咆哮，月光皎洁"
+negative_prompt = "nsfw, worst quality, low quality, normal quality, low resolution, monochrome, blurry, wrong, Mutated hands and fingers, text, ugly faces, twisted, jpeg artifacts, watermark, low contrast, realistic"
+
+images = pipe(prompt=prompt,
+              negative_prompt=negative_prompt,
+              num_inference_steps=40,
+              height=1024, width=1024,
+              guidance_scale=7.0
+              )[0]
+images.save("output.png")
+```
+
+Use pipe.enable_sequential_cpu_offload() to offload the model into CPU for less GPU memory cost (about 14.25 G, compared to 25.67 G if CPU offload is not enabled), but the inference time will increase significantly(5.18s v.s. 17.74s on A100 40G).
+
+<Tip>
+
+Make sure to check out the Schedulers [guide](../../using-diffusers/schedulers.md) to learn how to explore the tradeoff between scheduler speed and quality, and see the [reuse components across pipelines](../../using-diffusers/loading.md#reuse-a-pipeline) section to learn how to efficiently load the same components into multiple pipelines.
+
+</Tip>
+
+
+
+## AniMemoryPipeline
+
+[[autodoc]] AniMemoryPipeline
+	- all
+	- __call__

docs/source/en/api/schedulers/euler_ancestral_x_pred.md

@@ -0,0 +1,18 @@
+<!--Copyright 2024 The HuggingFace Team and AniMemory Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+-->
+
+# EulerAncestralDiscreteXPredScheduler
+
+An improved scheduler(SingDiffusion) that addresses the sampling challenge at the initial singular time step. To know more about SingDiffusion, check out the original [blog post](https://pangzecheung.github.io/SingDiffusion/). Our original paper can be found [here](https://arxiv.org/abs/2403.08381).
+
+## EulerAncestralDiscreteXPredScheduler
+[[autodoc]] EulerAncestralDiscreteXPredScheduler

src/diffusers/__init__.py

@@ -183,6 +183,7 @@
             "EDMDPMSolverMultistepScheduler",
             "EDMEulerScheduler",
             "EulerAncestralDiscreteScheduler",
+            "EulerAncestralDiscreteXPredScheduler",
             "EulerDiscreteScheduler",
             "FlowMatchEulerDiscreteScheduler",
             "FlowMatchHeunDiscreteScheduler",
@@ -257,6 +258,7 @@
             "AnimateDiffSparseControlNetPipeline",
             "AnimateDiffVideoToVideoControlNetPipeline",
             "AnimateDiffVideoToVideoPipeline",
+            "AniMemoryPipeline",
             "AudioLDM2Pipeline",
             "AudioLDM2ProjectionModel",
             "AudioLDM2UNet2DConditionModel",
@@ -736,6 +738,7 @@
             AnimateDiffSparseControlNetPipeline,
             AnimateDiffVideoToVideoControlNetPipeline,
             AnimateDiffVideoToVideoPipeline,
+            AniMemoryPipeline,
             AudioLDM2Pipeline,
             AudioLDM2ProjectionModel,
             AudioLDM2UNet2DConditionModel,

src/diffusers/pipelines/__init__.py

@@ -126,6 +126,7 @@
         "AnimateDiffVideoToVideoPipeline",
         "AnimateDiffVideoToVideoControlNetPipeline",
     ]
+    _import_structure["animemory"] = ["AniMemoryPipeline"]
     _import_structure["flux"] = [
         "FluxControlPipeline",
         "FluxControlImg2ImgPipeline",
@@ -476,6 +477,7 @@
             AnimateDiffVideoToVideoControlNetPipeline,
             AnimateDiffVideoToVideoPipeline,
         )
+        from .animemory import AniMemoryPipeline
         from .audioldm import AudioLDMPipeline
         from .audioldm2 import (
             AudioLDM2Pipeline,

src/diffusers/pipelines/animemory/__init__.py

@@ -0,0 +1,50 @@
+from typing import TYPE_CHECKING
+
+from ...utils import (
+    DIFFUSERS_SLOW_IMPORT,
+    OptionalDependencyNotAvailable,
+    _LazyModule,
+    get_objects_from_module,
+    is_torch_available,
+    is_transformers_available,
+)
+
+
+_dummy_objects = {}
+_import_structure = {"pipeline_output": ["AniMemoryPipelineOutput"]}
+
+
+try:
+    if not (is_transformers_available() and is_torch_available()):
+        raise OptionalDependencyNotAvailable()
+except OptionalDependencyNotAvailable:
+    from ...utils import dummy_torch_and_transformers_objects  # noqa F403
+
+    _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects))
+else:
+    _import_structure["modeling_movq"] = ["MoVQ"]
+    _import_structure["modeling_text_encoder"] = ["AniMemoryAltCLip", "AniMemoryT5"]
+    _import_structure["pipeline_animemory"] = ["AniMemoryPipeline"]
+
+if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
+    try:
+        if not (is_transformers_available() and is_torch_available()):
+            raise OptionalDependencyNotAvailable()
+
+    except OptionalDependencyNotAvailable:
+        from ...utils.dummy_torch_and_transformers_objects import *  # noqa F403
+    else:
+        from .pipeline_animemory import AniMemoryPipeline
+
+else:
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()["__file__"],
+        _import_structure,
+        module_spec=__spec__,
+    )
+
+    for name, value in _dummy_objects.items():
+        setattr(sys.modules[__name__], name, value)