Skip to content

Commit 020d557

Browse files
Add Phi-3 support (#299)
Adds initial Phi-3 support. Tested with [Phi-3-Mini-128K-Instruct](https://huggingface.co/microsoft/Phi-3-mini-128k-instruct).
1 parent 8ee638e commit 020d557

File tree

1 file changed

+50
-0
lines changed

1 file changed

+50
-0
lines changed
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
{
2+
"model_type": "phi",
3+
"architectures": [
4+
"Phi3ForCausalLM"
5+
],
6+
"pre_weights": [
7+
{
8+
"name": "model.embed_tokens.weight",
9+
"is_embed": true
10+
}
11+
],
12+
"post_weights": [
13+
{
14+
"name": "lm_head.weight",
15+
"is_embed": true
16+
},
17+
{
18+
"name": "model.norm.weight"
19+
}
20+
],
21+
"num_layers_config_key": "num_hidden_layers",
22+
"layer_templates": {
23+
"weights": [
24+
{
25+
"name": "model.layers.${layer_index}.input_layernorm.weight",
26+
"is_embed": false
27+
},
28+
{
29+
"name": "model.layers.${layer_index}.post_attention_layernorm.weight",
30+
"is_embed": false
31+
},
32+
{
33+
"name": "model.layers.${layer_index}.self_attn.o_proj.weight",
34+
"is_embed": false
35+
},
36+
{
37+
"name": "model.layers.${layer_index}.self_attn.qkv_proj.weight",
38+
"is_embed": false
39+
},
40+
{
41+
"name": "model.layers.${layer_index}.mlp.gate_up_proj.weight",
42+
"is_embed": false
43+
},
44+
{
45+
"name": "model.layers.${layer_index}.mlp.down_proj.weight",
46+
"is_embed": false
47+
}
48+
]
49+
}
50+
}

0 commit comments

Comments
 (0)