Sheared-Llama.json

{
    "model_name_or_path": "princeton-nlp/Sheared-LLaMA-1.3B",
    "dataset_name": "wikimedia/wikipedia",
    "dataset_config_name": "20231101.en",
    "per_device_train_batch_size": 32,
    "per_device_eval_batch_size": 32,
    "gradient_accumulation_steps": 1,
    "do_train": true,
    "do_eval": true,
    "max_seq_length": 512,
    "mask_token_type": "blank",
    "data_collator_type": "default",
    "mlm_probability": 0.3,
    "overwrite_output_dir": true,
    "output_dir": "output/mntp/Sheared-LLaMA-1.3B",
    "evaluation_strategy": "steps",
    "eval_steps": 50000,
    "save_steps": 30000,
    "stop_after_n_steps": 300000,
    "learning_rate":5e-6,
    "lora_r": 16,
    "enable_peft":false,
    "gradient_checkpointing": true,
    "torch_dtype": "bfloat16",
    "attn_implementation": "flash_attention_2"
}