LLM-VEDHA/config.json at master · Nandhan-Golla/LLM-VEDHA · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
{
  "model_configs": {
    "10M": {
      "n_layers": 6,
      "n_head": 8,
      "n_embd": 512,
      "vocab_size": 128,
      "max_seq_len": 256,
      "dropout": 0.1
    },
    "25M": {
      "n_layers": 12,
      "n_head": 12,
      "n_embd": 768,
      "vocab_size": 128,
      "max_seq_len": 256,
      "dropout": 0.1
    },
    "50M": {
      "n_layers": 24,
      "n_head": 16,
      "n_embd": 1024,
      "vocab_size": 128,
      "max_seq_len": 256,
      "dropout": 0.1
    }
  },
  "training_config": {
    "batch_size": 64,
    "learning_rate": 3e-4,
    "max_epochs": 100,
    "warmup_steps": 1000,
    "gradient_clip": 1.0,
    "beta1": 0.9,
    "beta2": 0.95,
    "weight_decay": 0.01
  },
  "data_config": {
    "train_split": 0.9,
    "val_split": 0.05,
    "test_split": 0.05,
    "max_seq_len": 256
  }
}