|
| 1 | +from datasets import load_dataset |
| 2 | +from transformers import AutoModelForCausalLM, AutoTokenizer |
| 3 | +import torch |
| 4 | +from compressed_tensors.utils import update_parameter_data |
| 5 | +from llmcompressor import oneshot |
| 6 | +from llmcompressor.modifiers.quantization import GPTQModifier, QuantizationModifier |
| 7 | +from llmcompressor.modifiers.transform import TransformModifier |
| 8 | +from llmcompressor.utils import dispatch_for_generation |
| 9 | +from transformers.models.llama.modeling_llama import ( |
| 10 | + LlamaRMSNorm, |
| 11 | +) |
| 12 | + |
| 13 | +hidden_dim = intermediate_dim = 64 |
| 14 | +up_dim = 128 |
| 15 | +num_embeddings = 12 |
| 16 | + |
| 17 | + |
| 18 | +# TODO remove file before merging |
| 19 | + |
| 20 | + |
| 21 | +class DummySelfAttn(torch.nn.Module): |
| 22 | + def __init__(self, hidden_dim, intermediate_dim): |
| 23 | + super().__init__() |
| 24 | + self.q_proj = torch.nn.Linear(hidden_dim, hidden_dim, bias=None) |
| 25 | + self.k_proj = torch.nn.Linear(hidden_dim, intermediate_dim, bias=None) |
| 26 | + self.v_proj = torch.nn.Linear(hidden_dim, intermediate_dim, bias=None) |
| 27 | + self.o_proj = torch.nn.Linear(hidden_dim, hidden_dim, bias=None) |
| 28 | + self.num_heads = 1 |
| 29 | + self.num_key_value_groups = 1 |
| 30 | + |
| 31 | + def forward(self, hidden_states): |
| 32 | + q = self.q_proj(hidden_states) |
| 33 | + k = self.k_proj(hidden_states) |
| 34 | + v = self.v_proj(hidden_states) |
| 35 | + |
| 36 | + ### EAGER ATTENTION |
| 37 | + attn_weights = torch.matmul(q.T, k) |
| 38 | + |
| 39 | + attn_weights = torch.nn.functional.softmax( |
| 40 | + attn_weights, dim=-1, dtype=torch.float32 |
| 41 | + ).to(q.dtype) |
| 42 | + attn_output = torch.matmul(attn_weights, v.T) |
| 43 | + attn_output = attn_output.T.contiguous() |
| 44 | + |
| 45 | + return self.o_proj(attn_output) |
| 46 | + |
| 47 | + |
| 48 | +class DummyMLP(torch.nn.Module): |
| 49 | + def __init__(self, hidden_dim, up_dim): |
| 50 | + super().__init__() |
| 51 | + self.up_proj = torch.nn.Linear(hidden_dim, up_dim, bias=None) |
| 52 | + self.gate_proj = torch.nn.Linear(hidden_dim, up_dim, bias=None) |
| 53 | + self.down_proj = torch.nn.Linear(up_dim, hidden_dim, bias=None) |
| 54 | + self.act_fn = torch.nn.SiLU() |
| 55 | + |
| 56 | + def forward(self, x): |
| 57 | + return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) |
| 58 | + |
| 59 | + |
| 60 | +class DummyModel(torch.nn.Module): |
| 61 | + def __init__(self, num_embeddings, hidden_dim, intermediate_dim, up_dim): |
| 62 | + super().__init__() |
| 63 | + self.embed_tokens = torch.nn.Embedding(num_embeddings, hidden_dim) |
| 64 | + self.input_layernorm = LlamaRMSNorm(hidden_dim) |
| 65 | + self.post_attention_layernorm = LlamaRMSNorm(hidden_dim) |
| 66 | + self.self_attn = DummySelfAttn(hidden_dim, intermediate_dim) |
| 67 | + self.mlp = DummyMLP(hidden_dim, up_dim) |
| 68 | + self.lm_head = torch.nn.Linear(hidden_dim, num_embeddings, bias=None) |
| 69 | + |
| 70 | + def forward(self, input_ids): |
| 71 | + x = self.embed_tokens(input_ids) |
| 72 | + x = self.input_layernorm(x) |
| 73 | + x = self.self_attn(x) |
| 74 | + x = self.post_attention_layernorm(x) |
| 75 | + x = self.mlp(x) |
| 76 | + return self.lm_head(x) |
| 77 | + |
| 78 | + |
| 79 | +model = DummyModel(num_embeddings, hidden_dim, intermediate_dim, up_dim) |
| 80 | + |
| 81 | +# TODO Uncomment this to see norm diff > 1e-6 |
| 82 | +# This is due to issue Kyle spotted in https://arxiv.org/pdf/2405.16406 Page 5 Footnote 2 |
| 83 | +# Will have to fuse layernorms with subsequent layers so that input_layernorm.weight is equal to torch.ones() (this apparently makes it rotation invariant) |
| 84 | +# https://github.com/facebookresearch/SpinQuant/blob/8f47aa3f00e8662caf1a484153920a07e5281c3a/utils/fuse_norm_utils.py#L39 |
| 85 | +# update_parameter_data( |
| 86 | +# model.input_layernorm, |
| 87 | +# torch.rand(model.input_layernorm.weight.shape), |
| 88 | +# "weight", |
| 89 | +# ) |
| 90 | + |
| 91 | +input_ids = torch.IntTensor([1, 2, 3, 4, 5]) |
| 92 | +orig_output = model(input_ids) |
| 93 | + |
| 94 | +recipe = [ |
| 95 | + # NOTE: preset_config="QUIP" output sensible, but cannot load saved |
| 96 | + # checkpoint or run evals (~4hrs to run) |
| 97 | + TransformModifier(preset_config="LLAMA_SPINQUANT_R1R2"), |
| 98 | + # QuantizationModifier(targets="Linear", scheme="W4A16", ignore=["lm_head"]), |
| 99 | +] |
| 100 | + |
| 101 | +oneshot( |
| 102 | + model=model, |
| 103 | + recipe=recipe, |
| 104 | + pipeline="datafree", |
| 105 | + log_dir=None, |
| 106 | +) |
| 107 | + |
| 108 | +# # Confirm generations of the quantized model look the same |
| 109 | +transformed_output = model(input_ids) |
| 110 | + |
| 111 | +print(f"Norm Diff {(orig_output-transformed_output).norm()}") |
| 112 | +print(f"Norm {orig_output.norm()}, {transformed_output.norm()}") |
0 commit comments