Use export_llm in CI

ghstack-source-id: bd75cc2
ghstack-comment-id: 2993075817
Pull-Request: #11836

Author: jackzhxng

.ci/configs/README.md

@@ -0,0 +1,42 @@
+# CI Configuration Files for LLM Export
+
+This directory contains YAML configuration files used by CI tests for exporting LLM models with the new `extension.llm.export.export_llm` command.
+
+## Usage
+
+These config files can be used with the export command like this:
+
+```bash
+python -m extension.llm.export.export_llm --config path/to/config.yaml
+```
+
+Or you can override specific parameters:
+
+```bash
+python -m extension.llm.export.export_llm --config ci_stories110m_xnnpack_quantized.yaml base.checkpoint=my_checkpoint.pt
+```
+
+## Configuration Files
+
+### CI Test Configurations
+- `ci_stories110m_xnnpack_quantized.yaml` - Stories110M with XNNPACK quantization (used in test_llama.sh)
+- `ci_stories110m_mps.yaml` - Stories110M with MPS backend
+- `ci_stories110m_coreml.yaml` - Stories110M with CoreML backend  
+- `ci_stories110m_qnn.yaml` - Stories110M with QNN backend
+
+### Performance Test Configurations
+- `llama3_spinquant.yaml` - Llama3 with SpinQuant (used in apple-perf.yml, android-perf.yml)
+- `llama3_qlora.yaml` - Llama3 with QLoRA (QAT + LoRA)
+- `llama3_coreml_ane.yaml` - Llama3 with CoreML ANE
+- `xnnpack_8da4w_basic.yaml` - Basic XNNPACK 8da4w quantization
+- `qwen3_xnnpack_8da4w.yaml` - Qwen3 with XNNPACK 8da4w quantization
+
+### Specialized Configurations
+- `stories110m_torchao_lowbit.yaml` - Stories110M with TorchAO lowbit quantization
+- `xnnpack_custom_quantized.yaml` - XNNPACK with custom ops and quantization
+
+## Background
+
+These configuration files were created as part of migrating CI tests from the old `examples.models.llama.export_llama` command to the new `extension.llm.export.export_llm` command with hydra configuration support.
+
+The config files help reduce duplication in CI scripts and make it easier to maintain consistent export settings across different test scenarios.

.ci/configs/ci_stories110m_coreml.yaml

@@ -0,0 +1,20 @@
+# Configuration for CI test_llama.sh - stories110M with CoreML backend
+
+base:
+  model_class: "stories110m"
+
+model:
+  dtype_override: "fp32"
+  use_kv_cache: true
+  enable_dynamic_shape: false
+
+export:
+  max_seq_length: 128
+  max_context_length: 128
+
+backend:
+  coreml:
+    enabled: true
+
+debug:
+  verbose: true

.ci/configs/ci_stories110m_mps.yaml

@@ -0,0 +1,20 @@
+# Configuration for CI test_llama.sh - stories110M with MPS backend
+
+base:
+  model_class: "stories110m"
+
+model:
+  dtype_override: "fp32"
+  use_kv_cache: true
+  enable_dynamic_shape: false
+
+export:
+  max_seq_length: 128
+  max_context_length: 128
+
+backend:
+  mps:
+    enabled: true
+
+debug:
+  verbose: true

.ci/configs/ci_stories110m_qnn.yaml

@@ -0,0 +1,28 @@
+# Configuration for CI test_llama.sh - stories110M with QNN backend
+
+base:
+  model_class: "stories110m"
+  tokenizer_path: "tokenizer.model"
+
+model:
+  dtype_override: "fp32"
+  use_kv_cache: true
+  enable_dynamic_shape: false
+
+export:
+  max_seq_length: 128
+  max_context_length: 128
+
+quantization:
+  pt2e_quantize: "qnn_16a16w"
+  calibration_tasks: ["wikitext"]
+  calibration_limit: 1
+  calibration_seq_length: 128
+  calibration_data: "Once"
+
+backend:
+  qnn:
+    enabled: true
+
+debug:
+  verbose: true

.ci/configs/ci_stories110m_xnnpack_quantized.yaml

@@ -0,0 +1,27 @@
+# Configuration for CI test_llama.sh - stories110M with XNNPACK quantization
+# Used when XNNPACK=ON, CUSTOM=ON, QE=ON modes are enabled
+
+base:
+  model_class: "stories110m"
+
+model:
+  dtype_override: "fp32"
+  use_kv_cache: true
+  use_sdpa_with_kv_cache: true
+
+export:
+  max_seq_length: 128
+  max_context_length: 128
+
+quantization:
+  qmode: "8da4w"
+  group_size: 128
+  embedding_quantize: "8,1024"
+
+backend:
+  xnnpack:
+    enabled: true
+    extended_ops: true
+
+debug:
+  verbose: false

.ci/configs/llama3_coreml_ane.yaml

@@ -0,0 +1,27 @@
+# Configuration for Llama3 with CoreML ANE
+# Used in apple-perf.yml
+
+base:
+  model_class: "llama3_2"
+
+model:
+  dtype_override: "fp32"
+  use_kv_cache: true
+  enable_dynamic_shape: false
+
+export:
+  max_seq_length: 128
+  max_context_length: 128
+
+quantization:
+  embedding_quantize: "4,32"
+
+backend:
+  coreml:
+    enabled: true
+    ios: 18
+    quantize: "c4w"
+    compute_units: "cpu_and_ne"
+
+debug:
+  verbose: false

.ci/configs/llama3_qlora.yaml

@@ -0,0 +1,30 @@
+# Configuration for Llama3 with QLoRA (QAT + LoRA)
+# Used in apple-perf.yml and android-perf.yml
+
+base:
+  model_class: "llama3_2"
+  use_lora: 16
+  preq_mode: "8da4w_output_8da8w"
+  preq_group_size: 32
+  preq_embedding_quantize: "8,0"
+
+model:
+  dtype_override: "fp32"
+  use_kv_cache: true
+  use_sdpa_with_kv_cache: true
+  enable_dynamic_shape: false
+
+export:
+  max_seq_length: 2048
+  max_context_length: 2048
+
+quantization:
+  use_qat: true
+
+backend:
+  xnnpack:
+    enabled: true
+    extended_ops: true
+
+debug:
+  verbose: false

.ci/configs/llama3_spinquant.yaml

@@ -0,0 +1,29 @@
+# Configuration for Llama3 with SpinQuant
+# Used in apple-perf.yml and android-perf.yml
+
+base:
+  model_class: "llama3_2"
+  preq_mode: "8da4w_output_8da8w"
+  preq_group_size: 32
+  preq_embedding_quantize: "8,0"
+
+model:
+  dtype_override: "fp32"
+  use_kv_cache: true
+  use_sdpa_with_kv_cache: true
+  enable_dynamic_shape: false
+
+export:
+  max_seq_length: 2048
+  max_context_length: 2048
+
+quantization:
+  use_spin_quant: "native"
+
+backend:
+  xnnpack:
+    enabled: true
+    extended_ops: true
+
+debug:
+  verbose: false

.ci/configs/qwen3_xnnpack_8da4w.yaml

@@ -0,0 +1,28 @@
+# Configuration for Qwen3-0.6B with XNNPACK 8da4w quantization
+# Used in apple-perf.yml and android-perf.yml
+
+base:
+  model_class: "qwen3-0_6b"
+  params: "examples/models/qwen3/0_6b_config.json"
+
+model:
+  dtype_override: "fp32"
+  use_kv_cache: true
+  use_sdpa_with_kv_cache: true
+
+export:
+  max_seq_length: 128
+  max_context_length: 128
+
+quantization:
+  qmode: "8da4w"
+  group_size: 32
+  embedding_quantize: "8,0"
+
+backend:
+  xnnpack:
+    enabled: true
+    extended_ops: true
+
+debug:
+  verbose: false

.ci/configs/stories110m_torchao_lowbit.yaml

@@ -0,0 +1,26 @@
+# Configuration for stories110M with TorchAO lowbit quantization
+# Used in CI test_llama_torchao_lowbit.sh
+
+base:
+  model_class: "stories110m"
+
+model:
+  dtype_override: "fp32"
+  use_kv_cache: true
+  use_sdpa_with_kv_cache: true
+
+export:
+  max_seq_length: 128
+  max_context_length: 128
+
+quantization:
+  qmode: "torchao:8da3w"  # QLINEAR_BITWIDTH=3
+  group_size: 128        # QLINEAR_GROUP_SIZE=128
+  embedding_quantize: "4,32"  # QEMBEDDING_BITWIDTH=4, QEMBEDDING_GROUP_SIZE=32
+
+backend:
+  xnnpack:
+    enabled: false
+
+debug:
+  verbose: false