Changes to support codellama

Don't load inv_freq tensor, add rope_theta config parameter

Based on huggingface/transformers#24998 and huggingface/transformers#25740

Author: njhill

server/text_generation_server/models/custom_modeling/flash_llama_modeling.py

@@ -62,6 +62,7 @@ def __init__(
         pretraining_tp=1,
         tie_word_embeddings=False,
         rope_scaling=None,
+        rope_theta=10000.0,
         **kwargs,
     ):
         self.vocab_size = vocab_size
@@ -82,6 +83,7 @@ def __init__(
         self.pretraining_tp = pretraining_tp
         self.use_cache = use_cache
         self.rope_scaling = rope_scaling
+        self.rope_theta = rope_theta
 
         super().__init__(
             pad_token_id=pad_token_id,
@@ -178,8 +180,11 @@ def __init__(
         self.hidden_size = config.hidden_size
         self.head_size = self.hidden_size // self.num_heads
 
-        self.rotary_emb = PositionRotaryEmbedding.load(
-            prefix=f"{prefix}.rotary_emb", weights=weights
+        # self.rotary_emb = PositionRotaryEmbedding.load(
+        #     prefix=f"{prefix}.rotary_emb", weights=weights
+        # )
+        self.rotary_emb = PositionRotaryEmbedding.static(
+            dim=self.head_size, base=config.rope_theta, device=weights.device
         )
 
         self.softmax_scale = self.head_size**-0.5