Merge branch 'main' into feature/intermediates-cache-prefetch

Author: GOavi101

.MAINTAINERS

@@ -0,0 +1,9 @@
+# Code Quality Check
+
+After completing a batch of code changes:
+
+1. Run `make style` to auto-format the code
+2. Run `make quality` to validate code quality
+3. Fix any issues reported by `make quality`
+
+When writing code, keep lines under 88 characters and avoid unnecessary indentation.

.claude/skills/style.md

@@ -0,0 +1,20 @@
+# Running Tests
+
+Nearly all tests in this project require a GPU.
+
+When running tests:
+
+1. First check if `canhazgpu` is available: `which canhazgpu`
+2. If available, MUST run tests using `canhazgpu` with appropriate GPU allocation
+3. Use the format: `canhazgpu --gpus 1 -- python3 -m pytest tests/...`
+
+Example:
+```bash
+# Check if canhazgpu is available
+which canhazgpu
+
+# Run tests with canhazgpu (required if available)
+canhazgpu --gpus 1 -- python3 -m pytest tests/test_example.py
+```
+
+Note: Adjust the `--gpus` count based on test requirements (typically 1 GPU is sufficient for most tests).

.claude/skills/test.md

@@ -1,4 +1,43 @@
+queue_rules:
+  - name: default
+    merge_method: merge
+    commit_message_template: |
+      {{ title }} (#{{ number }})
+
+      {{ body }}
+
+      Signed-off-by: Mergify <noreply@mergify.com>
+    queue_conditions:
+      - check-success=DCO
+      - check-success=quality-check
+      - check-success=transformers-tests
+      - check-success=base-tests (3.10)
+      - check-success=base-tests (3.13)
+      - check-success=pytorch-tests (3.10)
+      - check-success=pytorch-tests (3.13)
+      - check-success=markdown-link-check
+
 pull_request_rules:
+  - name: Automatically merge when ready
+    conditions:
+      - base=main
+      - label=ready
+      - "#approved-reviews-by>=2"
+      - check-success=DCO
+      - check-success=quality-check
+      - check-success=transformers-tests
+      - check-success=base-tests (3.10)
+      - check-success=base-tests (3.13)
+      - check-success=pytorch-tests (3.10)
+      - check-success=pytorch-tests (3.13)
+      - check-success=markdown-link-check
+      - check-success=ready-label-check
+      - -conflict
+      - -draft
+    actions:
+      queue:
+        name: default
+
   - name: label-documentation
     description: Automatically apply documentation label
     conditions:

.github/mergify.yml

@@ -128,6 +128,13 @@ venv.bak/
 /site
 docs/.cache/*
 
+# zensical docs build (generated by pre-build scripts)
+docs/api/llmcompressor/
+docs/examples/
+docs/experimental/
+docs/developer/code-of-conduct.md
+docs/developer/contributing.md
+
 # mypy
 .mypy_cache/
 ### Example user template template
@@ -811,5 +818,3 @@ env_log.json
 # uv artifacts
 uv.lock
 .venv/
-
-.claude/

.gitignore

@@ -1,22 +1,16 @@
-# Read the Docs configuration file
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-# Required
 version: 2
 
-# Set the OS, Python version, and other tools you might need
 build:
   os: ubuntu-24.04
   tools:
     python: "3.12"
-
-# Build documentation with Mkdocs
-mkdocs:
-   configuration: mkdocs.yml
-
-python:
-  install:
-    - method: pip
-      path: .
-      extra_requirements:
-        - dev
+  jobs:
+    install:
+      - pip install -e ".[dev]"
+    build:
+      html:
+        - python docs/scripts/zensical_gen_files.py
+        - zensical build
+    post_build:
+      - mkdir -p $READTHEDOCS_OUTPUT/html/
+      - cp --recursive site/* $READTHEDOCS_OUTPUT/html/

.readthedocs.yaml

@@ -37,13 +37,12 @@ Big updates have landed in LLM Compressor! To get a more in-depth look, check ou
 
 Some of the exciting new features include:
 
-* **Batched Calibration Support**: LLM Compressor now supports calibration with batch sizes > 1. A new [`batch_size`](src/llmcompressor/args/dataset_arguments.py#L70) argument has been added to the `dataset_arguments` enabling the option to improve quantization speed. Default `batch_size` is currently set to 1
+* **Updated offloading and model loading support**: Loading transformers models that are offloaded to disk and/or offloaded across distributed process ranks is now supported. Disk offloading allows users to load and compress very large models which normally would not fit in CPU memory. Offloading functionality is no longer supported through accelerate but through model loading utilities added to compressed-tensors. For a full summary of updated loading and offloading functionality, for both single-process and distributed flows, see the [Big Models and Distributed Support guide](docs/guides/big_models_and_distributed/model_loading.md).
+* **Distributed GPTQ Support**: GPTQ now supports Distributed Data Parallel (DDP) functionality to significantly improve calibration runtime. An example using DDP with GPTQ can be found [here](examples/quantization_w4a16/llama3_ddp_example.py).
+* **Updated FP4 Microscale Support**: GPTQ now supports FP4 quantization schemes, including both [MXFP4](examples/quantization_w4a16_fp4/mxfp4/llama3_example.py) and [NVFP4](examples/quantization_w4a4_fp4/llama3_gptq_example.py). MXFP4 support has also been improved with updated weight scale generation. Models with weight-only quantization in the MXFP4 format can now run in vLLM as of vLLM v0.14.0. MXFP4 models with activation quantization are not yet supported in vLLM for compressed-tensors models
 * **New Model-Free PTQ Pathway**: A new model-free PTQ pathway has been added to LLM Compressor, called [`model_free_ptq`](src/llmcompressor/entrypoints/model_free/__init__.py#L36). This pathway allows you to quantize your model without the requirement of Hugging Face model definition and is especially useful in cases where `oneshot` may fail. This pathway is currently supported for data-free pathways only i.e FP8 quantization and was leveraged to quantize the [Mistral Large 3 model](https://huggingface.co/mistralai/Mistral-Large-3-675B-Instruct-2512). Additional [examples](examples/model_free_ptq) have been added illustrating how LLM Compressor can be used for Kimi K2
 * **Extended KV Cache and Attention Quantization Support**: LLM Compressor now supports attention quantization. KV Cache quantization, which previously only supported per-tensor scales, has been extended to support any quantization scheme including a new `per-head` quantization scheme. Support for these checkpoints is on-going in vLLM and scripts to get started have been added to the [experimental folder](experimental/attention)
-* **Generalized AWQ Support**: The AWQModifier has been updated to support quantization schemes beyond W4A16 (e.g W4AFp8). In particular, AWQ no longer constrains that the quantization config needs to have the same settings for `group_size`, `symmetric`, and `num_bits` for each config_group
-* **AutoRound Quantization Support**: Added [`AutoRoundModifier`](examples/autoround) for quantization using [AutoRound](https://aclanthology.org/2024.findings-emnlp.662.pdf), an advanced post-training algorithm that optimizes rounding and clipping ranges through sign-gradient descent. This approach combines the efficiency of post-training quantization with the adaptability of parameter tuning, delivering robust compression for large language models while maintaining strong performance
-* **Experimental MXFP4 Support**: Models can now be quantized using an [`MXFP4`](https://github.com/vllm-project/compressed-tensors/blob/main/src/compressed_tensors/quantization/quant_scheme.py#L208) pre-set scheme. Examples can be found under the [experimental folder](experimental/mxfp4/llama3_mxfp4.py). This pathway is still experimental as support and validation with vLLM is still a WIP. 
-* **R3 Transform Support**: LLM Compressor now supports applying transforms to attention in the style of SpinQuant's R3 rotation. Note: this feature is currently not yet supported in vLLM. An example applying R3 can be found in the [experimental folder](experimental/attention/llama3_attention_r3_nvfp4.py)
+
 
 ### Supported Formats
 * Activation Quantization: W8A8 (int8 and fp8)

README.md

@@ -1,14 +1,16 @@
 nav:
   - Home: index.md
-  - Why use LLM Compressor?: getting-started/why-llmcompressor.md
-  - Choosing the right compression scheme: getting-started/choosing-scheme.md
-  - Choosing the right compression algorithm: getting-started/choosing-algo.md
+  - Why use LLM Compressor?: steps/why-llmcompressor.md
+  - Compressing your model, step-by-step:
+    - Choosing your model: steps/choosing-model.md
+    - Choosing the right compression scheme: steps/choosing-scheme.md
+    - Choosing the right compression algorithm: steps/choosing-algo.md
+    - Choosing a dataset: steps/choosing-dataset.md
+    - Compressing your model: steps/compress.md
+  - Deploying with vLLM: steps/deploy.md
   - Getting started:
     - getting-started/index.md
     - Installing LLM Compressor: getting-started/install.md
-    - Compressing your Model: getting-started/compress.md
-    - Deploying with vLLM: getting-started/deploy.md
-    - FAQ: getting-started/faq.md
   - Key Models:
     - key-models/index.md
     - Llama 4:
@@ -24,14 +26,25 @@ nav:
       - key-models/mistral-large-3/index.md
       - FP8 Example: key-models/mistral-large-3/fp8-example.md
   - Guides:
+    - Big Models and Distributed Support:
+      - Model Loading: guides/big_models_and_distributed/model_loading.md
+      - Sequential Onloading: guides/big_models_and_distributed/sequential_onloading.md
+      - Distributed Oneshot: guides/big_models_and_distributed/distributed_oneshot.md
     - Compression Schemes: guides/compression_schemes.md
     - Saving a Model: guides/saving_a_model.md
-    - Observers: observers.md
+    - Observers: guides/observers.md
+    - Memory Requirements: guides/memory.md
+    - Runtime Performance: guides/runtime.md
   - Examples:
-    - examples/index.md
+    - examples/README.md
     - examples/*
+  - Experimental:
+    - experimental/README.md
+    - experimental/*
   - Developer:
     - developer/index.md
     - developer/*
   - API Reference:
     - api/*
+  - FAQ: 
+    - faq/faq.md

docs/.nav.yml

@@ -0,0 +1,43 @@
+# Getting started with LLM Compressor docs
+
+```bash
+cd docs
+```
+
+- Install the dependencies:
+
+```bash
+make install
+```
+
+- Clean the previous build (optional but recommended):
+
+```bash
+make clean
+```
+
+- Generate docs content (files, API references, and navigation):
+
+```bash
+make gen
+```
+
+- Serve the docs locally (runs `gen` automatically):
+
+```bash
+make serve
+```
+
+This will start a local server. You can now open your browser and view the documentation.
+
+- Build the static site (runs `gen` automatically):
+
+```bash
+make build
+```
+
+- List all available targets:
+
+```bash
+make help
+```

docs/DEVELOPMENT.md

@@ -1,26 +1,32 @@
-# Minimal mkdocs makefile
+# Minimal zensical makefile
 
-PYTHON      := python3
-MKDOCS_CMD  := mkdocs
-MKDOCS_CONF := ../mkdocs.yml
+ZENSICAL_CMD := zensical
+ZENSICAL_CONF := ../zensical.toml
 
-.PHONY: help install serve build clean
+.PHONY: help install gen serve build clean
 
 help:
 	@echo "Available targets:"
 	@echo "  install  Install dependencies globally"
+	@echo "  gen      Generate docs content (files + API + nav)"
 	@echo "  serve    Serve docs locally"
 	@echo "  build    Build static site"
 	@echo "  clean    Remove build artifacts"
 
 install:
 	pip install -e "../[dev]"
 
-serve:
-	$(MKDOCS_CMD) serve --livereload -f $(MKDOCS_CONF)
+gen:
+	cd .. && python docs/scripts/zensical_gen_files.py
 
-build:
-	$(MKDOCS_CMD) build -f $(MKDOCS_CONF)
+serve: gen
+	cd .. && $(ZENSICAL_CMD) serve
+
+build: gen
+	cd .. && $(ZENSICAL_CMD) build
 
 clean:
-	rm -rf site/ .cache/ 
+	rm -rf site/ .cache/ api/llmcompressor/
+	rm -rf examples/ experimental/
+	rm -f developer/code-of-conduct.md developer/contributing.md
+	cd .. && python3 docs/scripts/zensical_gen_files.py --clean