Merge pull request #181 from VikParuchuri/main

yanboliang · web-flow · commit 9b908fb74da8 · 2024-06-14T10:14:11.000-07:00
Fix rope base issue with llama 3
diff --git a/model.py b/model.py
@@ -65,7 +65,7 @@ def from_name(cls, name: str):
     "Mistral-7B": dict(n_layer=32, n_head=32, n_local_heads=8, dim=4096, intermediate_size=14336, vocab_size=32000),
     "stories15M": dict(n_layer=6, n_head=6, dim=288),
     "stories110M": dict(n_layer=12, n_head=12, dim=768),
-    "Llama-3-8B": dict(block_size=8192, n_layer=32, n_head=32, n_local_heads=8, dim=4096, intermediate_size=14336, vocab_size=128256),
+    "Llama-3-8B": dict(block_size=8192, n_layer=32, n_head=32, n_local_heads=8, dim=4096, intermediate_size=14336, vocab_size=128256, rope_base=500000),
 }
 
 class KVCache(nn.Module):

Original file line number	Diff line number	Diff line change
`@@ -65,7 +65,7 @@ def from_name(cls, name: str):`
`65`	`65`	`"Mistral-7B": dict(n_layer=32, n_head=32, n_local_heads=8, dim=4096, intermediate_size=14336, vocab_size=32000),`
`66`	`66`	`"stories15M": dict(n_layer=6, n_head=6, dim=288),`
`67`	`67`	`"stories110M": dict(n_layer=12, n_head=12, dim=768),`
`68`		`- "Llama-3-8B": dict(block_size=8192, n_layer=32, n_head=32, n_local_heads=8, dim=4096, intermediate_size=14336, vocab_size=128256),`
	`68`	`+ "Llama-3-8B": dict(block_size=8192, n_layer=32, n_head=32, n_local_heads=8, dim=4096, intermediate_size=14336, vocab_size=128256, rope_base=500000),`
`69`	`69`	`}`
`70`	`70`
`71`	`71`	`class KVCache(nn.Module):`