Fix/loading gemma 3 1b (#2004)

Author: pquadri

litgpt/config.py

@@ -1069,7 +1069,7 @@ def norm_class(self) -> Type:
         sliding_window_size=512,
         # 5 local layers for every global layer
         sliding_window_indices=[0 if (i + 1) % 6 == 0 else 1 for i in range(26)],
-        intermediate_size=21504,
+        intermediate_size=6912,
         n_embd=1152,
         n_layer=26,
         n_head=4,

litgpt/scripts/convert_hf_checkpoint.py

@@ -602,6 +602,9 @@ def convert_hf_checkpoint(
     elif model_name.lower().startswith("gemma-2"):
         qkv_weights = {}
         copy_fn = partial(copy_weights_gemma_2, qkv_weights)
+    elif model_name.lower().startswith("gemma-3"):
+        qkv_weights = {}
+        copy_fn = partial(copy_weights_gemma_3, qkv_weights)
     elif model_name.lower().startswith("phi"):
         # holder to reconstitute the split q, k, v
         qkv_weights = {}