Skip to content

Commit 42158dc

Browse files
committed
Update model maps
1 parent aa48c35 commit 42158dc

File tree

1 file changed

+9
-1
lines changed

1 file changed

+9
-1
lines changed

configs/nsa_340M.json

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
11
{
22
"attention_bias": false,
3+
"architectures": [
4+
"NSAForCausalLM"
5+
],
6+
"auto_map": {
7+
"AutoConfig": "native_sparse_attention.NSAConfig",
8+
"AutoModel": "native_sparse_attention.NSAModel",
9+
"AutoModelForCausalLM": "native_sparse_attention.NSAForCausalLM"
10+
},
311
"bos_token_id": 1,
412
"eos_token_id": 2,
513
"fuse_cross_entropy": true,
@@ -8,7 +16,7 @@
816
"hidden_size": 1024,
917
"initializer_range": 0.006,
1018
"max_position_embeddings": 8192,
11-
"model_type": "transformer",
19+
"model_type": "nsa",
1220
"num_heads": 64,
1321
"num_kv_heads": 4,
1422
"block_size": 64,

0 commit comments

Comments
 (0)