init the folder directories for vLLM-omni

.gitignore

@@ -1,6 +1,6 @@
 # Byte-compiled / optimized / DLL files
 __pycache__/
-*.py[codz]
+*.py[cod]
 *$py.class
 
 # C extensions
@@ -46,7 +46,7 @@ htmlcov/
 nosetests.xml
 coverage.xml
 *.cover
-*.py.cover
+*.py,cover
 .hypothesis/
 .pytest_cache/
 cover/
@@ -94,35 +94,20 @@ ipython_config.py
 #   install all needed dependencies.
 #Pipfile.lock
 
-# UV
-#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#uv.lock
-
 # poetry
 #   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
 #   This is especially recommended for binary packages to ensure reproducibility, and is more
 #   commonly ignored for libraries.
 #   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
 #poetry.lock
-#poetry.toml
 
 # pdm
 #   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
-#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
 #pdm.lock
-#pdm.toml
-.pdm-python
-.pdm-build/
-
-# pixi
-#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
-#pixi.lock
-#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
-#   in the .venv directory. It is recommended not to include this directory in version control.
-.pixi
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
 
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
@@ -136,7 +121,6 @@ celerybeat.pid
 
 # Environments
 .env
-.envrc
 .venv
 env/
 venv/
@@ -170,38 +154,79 @@ cython_debug/
 
 # PyCharm
 #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-
-# Abstra
-# Abstra is an AI-powered process automation framework.
-# Ignore directories containing user credentials, local state, and settings.
-# Learn more at https://abstra.io/docs
-.abstra/
-
-# Visual Studio Code
-#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
-#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
-#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
-#  you could uncomment the following to ignore the entire vscode folder
-# .vscode/
-
-# Ruff stuff:
-.ruff_cache/
-
-# PyPI configuration file
-.pypirc
-
-# Cursor
-#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
-#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
-#  refer to https://docs.cursor.com/context/ignore-files
-.cursorignore
-.cursorindexingignore
-
-# Marimo
-marimo/_static/
-marimo/_lsp/
-__marimo__/
+#  be added to the global gitignore or merged into this project gitignore.  For a PyCharm
+#  project, it is recommended to include the following files:
+#  .idea/
+#  *.iml
+#  *.ipr
+#  *.iws
+#  .idea/
+
+# vLLM-omni specific
+# Model files and checkpoints
+models/
+checkpoints/
+*.bin
+*.safetensors
+*.pt
+*.pth
+
+# Cache directories
+cache/
+.cache/
+diffusion_cache/
+kv_cache/
+
+# Logs
+logs/
+*.log
+
+# Temporary files
+tmp/
+temp/
+.tmp/
+
+# IDE files
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Data files
+data/
+datasets/
+*.csv
+*.json
+*.jsonl
+*.parquet
+
+# Output files
+outputs/
+results/
+generated/
+
+# Configuration overrides
+configs/local.yaml
+configs/production.yaml
+configs/development.yaml
+
+# Docker
+.dockerignore
+Dockerfile.dev
+
+# Kubernetes
+k8s/
+*.yaml
+*.yml
+!configs/*.yaml
+!configs/*.yml

README.md

@@ -1,2 +1,3 @@
-# vllm-omni
-vLLM-omni supports multi-modality models inference and serving with non-autoregressive structures and non-textual outputs, extending vLLM beyond traditional text-based, autoregressive generation.
+# vLLM-omni: Multi-modal Extension for vLLM
+
+vLLM-omni is designed to extend vLLM capabilities to support multi-modality model inference and serving, particularly focusing on non-autoregressive structures and non-textual outputs.

docs/README.md

@@ -0,0 +1,5 @@
+# vLLM-omni Documentation
+
+Welcome to the vLLM-omni documentation! This documentation provides comprehensive information about using and developing with vLLM-omni.
+
+# TODO (add examples)

examples/README.md

@@ -0,0 +1,62 @@
+# vLLM-omni Examples
+
+This directory contains examples demonstrating how to use vLLM-omni for various tasks.
+
+## Basic Examples
+
+- [Text Generation](basic/text_generation.py) - Basic text generation using AR models
+- [Image Generation](basic/image_generation.py) - Image generation using diffusion models
+- [Multimodal Processing](basic/multimodal_processing.py) - Processing text and images together
+
+## Advanced Examples
+
+- [Custom Model Integration](advanced/custom_model.py) - Integrating custom models
+- [Batch Processing](advanced/batch_processing.py) - Efficient batch processing
+- [Streaming Output](advanced/streaming.py) - Real-time streaming output
+
+## Multimodal Examples
+
+- [Image-to-Text](multimodal/image_to_text.py) - Image captioning and description
+- [Text-to-Image](multimodal/text_to_image.py) - Text-to-image generation
+- [Audio Processing](multimodal/audio_processing.py) - Audio generation and processing
+- [Video Generation](multimodal/video_generation.py) - Video generation workflows
+
+## API Examples
+
+- [REST API](api/rest_api.py) - Using the REST API interface
+- [Gradio Interface](api/gradio_interface.py) - Creating Gradio web interfaces
+- [ComfyUI Integration](api/comfyui_integration.py) - ComfyUI workflow integration
+
+## Configuration Examples
+
+- [Custom Configuration](config/custom_config.py) - Custom configuration setup
+- [Multi-GPU Setup](config/multi_gpu.py) - Multi-GPU configuration
+- [Distributed Processing](config/distributed.py) - Distributed processing setup
+
+## Getting Started
+
+1. Install vLLM-omni:
+   ```bash
+   pip install vllm-omni
+   ```
+
+2. Run a basic example:
+   ```bash
+   python examples/basic/text_generation.py
+   ```
+
+3. Explore the examples in each subdirectory for more advanced usage.
+
+## Requirements
+
+Most examples require additional dependencies. Install them with:
+
+```bash
+pip install -r requirements.txt
+```
+
+For development examples, install the development dependencies:
+
+```bash
+pip install -r requirements-dev.txt
+```

pyproject.toml

@@ -0,0 +1,137 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "vllm-omni"
+version = "0.1.0"
+description = "vLLM-omni: Multi-modality models inference and serving with non-autoregressive structures"
+readme = "README.md"
+requires-python = ">=3.8"
+license = {text = "Apache-2.0"}
+authors = [
+    {name = "vLLM-omni Team", email = "hsliuustc@gmail.com"}
+]
+keywords = ["vllm", "multimodal", "diffusion", "transformer", "inference", "serving"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+dependencies = [
+    "vllm>=0.2.0",
+    "torch>=2.0.0",
+    "transformers>=4.30.0",
+    "numpy>=1.21.0",
+    "pillow>=8.0.0",
+    "opencv-python>=4.5.0",
+    "gradio>=3.0.0",
+    "fastapi>=0.100.0",
+    "uvicorn>=0.20.0",
+    "pydantic>=2.0.0",
+    "asyncio-mqtt>=0.11.0",
+    "ray>=2.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-asyncio>=0.21.0",
+    "pytest-cov>=4.0.0",
+    "black>=23.0.0",
+    "isort>=5.12.0",
+    "flake8>=6.0.0",
+    "mypy>=1.0.0",
+    "pre-commit>=3.0.0",
+]
+docs = [
+    "sphinx>=6.0.0",
+    "sphinx-rtd-theme>=1.2.0",
+    "myst-parser>=1.0.0",
+]
+examples = [
+    "jupyter>=1.0.0",
+    "matplotlib>=3.5.0",
+    "seaborn>=0.11.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/hsliuustc0106/vllm-omni"
+Repository = "https://github.com/hsliuustc0106/vllm-omni"
+Documentation = "https://vllm-omni.readthedocs.io"
+"Bug Tracker" = "https://github.com/hsliuustc0106/vllm-omni/issues"
+
+[project.scripts]
+vllm-omni = "vllm_omni.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["vllm_omni*"]
+
+[tool.black]
+line-length = 88
+target-version = ['py38', 'py39', 'py310', 'py311']
+include = '\.pyi?$'
+extend-exclude = '''
+/(
+  # directories
+  \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | build
+  | dist
+)/
+'''
+
+[tool.isort]
+profile = "black"
+multi_line_output = 3
+line_length = 88
+known_first_party = ["vllm_omni"]
+
+[tool.mypy]
+python_version = "3.8"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+disallow_untyped_decorators = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+warn_unreachable = true
+strict_equality = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py", "*_test.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "--strict-markers",
+    "--strict-config",
+    "--cov=vllm_omni",
+    "--cov-report=term-missing",
+    "--cov-report=html",
+    "--cov-report=xml",
+]
+markers = [
+    "unit: Unit tests",
+    "integration: Integration tests",
+    "benchmark: Benchmark tests",
+    "slow: Slow tests",
+]