Skip to content

Commit 804148b

Browse files
haasonsaashaasonsaas
committed
Update README with all new features (logging, resume, Inspect AI, mock mode)
1 parent 48fed26 commit 804148b

File tree

6 files changed

+87
-1
lines changed

6 files changed

+87
-1
lines changed

‎checkpoint_manager.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ def save_run_state(
4343
"round_idx": round_idx,
4444
"global_step": global_step,
4545
"learning_rate": learning_rate,
46-
"checkpoint_uri": checkpoint_uri,
46+
"checkpoint_uri": str(checkpoint_uri),
4747
"config": config,
4848
"timestamp": datetime.now().isoformat(),
4949
}

‎mock_checkpoints/round_3.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"name": "round_3", "mock": true}

‎runs/20251001_185248/metrics.jsonl‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{"timestamp": "2025-10-01T18:52:48.424598", "run_id": "20251001_185248", "event": "config", "base_model": "test-model", "train_file": "/private/var/folders/1s/sm2bn6kd64gdnv757t9qshcm0000gn/T/pytest-of-jonathanhaas/pytest-41/test_early_stopping_on_thresho0/train.jsonl", "eval_tasks": [], "renderer_name": "default", "learning_rate": 0.0001, "eval_threshold": 0.8, "max_rounds": 5, "lr_decay": 0.8, "evalops_enabled": false, "evalops_test_suite_id": null, "evalops_api_url": null, "steps_per_round": 1, "batch_size": 8, "max_seq_length": 2048, "lora_rank": 16, "warmup_steps": 100, "max_steps": 1000, "min_lr": 1e-06, "use_recommended_lr": false}
2+
{"timestamp": "2025-10-01T18:52:48.425049", "run_id": "20251001_185248", "event": "checkpoint", "round": 1, "checkpoint_uri": "<MagicMock name='mock.create_lora_training_client().save_weights_for_sampler().result().path' id='4503094000'>"}
3+
{"timestamp": "2025-10-01T18:52:48.425252", "run_id": "20251001_185248", "event": "evaluation", "round": 1, "score": 0.85, "threshold": 0.8, "passed": true, "metrics": {}}
4+
{"timestamp": "2025-10-01T18:52:48.426888", "run_id": "20251001_185248", "event": "config", "base_model": "test-model", "train_file": "/private/var/folders/1s/sm2bn6kd64gdnv757t9qshcm0000gn/T/pytest-of-jonathanhaas/pytest-41/test_full_rounds_below_thresho0/train.jsonl", "eval_tasks": [], "renderer_name": "default", "learning_rate": 0.0001, "eval_threshold": 0.9, "max_rounds": 3, "lr_decay": 0.5, "evalops_enabled": false, "evalops_test_suite_id": null, "evalops_api_url": null, "steps_per_round": 1, "batch_size": 8, "max_seq_length": 2048, "lora_rank": 16, "warmup_steps": 100, "max_steps": 1000, "min_lr": 1e-06, "use_recommended_lr": false}
5+
{"timestamp": "2025-10-01T18:52:48.427227", "run_id": "20251001_185248", "event": "checkpoint", "round": 1, "checkpoint_uri": "<MagicMock name='mock.create_lora_training_client().save_weights_for_sampler().result().path' id='4503098368'>"}
6+
{"timestamp": "2025-10-01T18:52:48.427394", "run_id": "20251001_185248", "event": "evaluation", "round": 1, "score": 0.7, "threshold": 0.9, "passed": false, "metrics": {}}
7+
{"timestamp": "2025-10-01T18:52:48.427519", "run_id": "20251001_185248", "event": "checkpoint", "round": 2, "checkpoint_uri": "<MagicMock name='mock.create_lora_training_client().save_weights_for_sampler().result().path' id='4503098368'>"}
8+
{"timestamp": "2025-10-01T18:52:48.427655", "run_id": "20251001_185248", "event": "evaluation", "round": 2, "score": 0.7, "threshold": 0.9, "passed": false, "metrics": {}}
9+
{"timestamp": "2025-10-01T18:52:48.427770", "run_id": "20251001_185248", "event": "checkpoint", "round": 3, "checkpoint_uri": "<MagicMock name='mock.create_lora_training_client().save_weights_for_sampler().result().path' id='4503098368'>"}
10+
{"timestamp": "2025-10-01T18:52:48.427898", "run_id": "20251001_185248", "event": "evaluation", "round": 3, "score": 0.7, "threshold": 0.9, "passed": false, "metrics": {}}
11+
{"timestamp": "2025-10-01T18:52:48.430702", "run_id": "20251001_185248", "event": "config", "base_model": "test-model", "train_file": "/private/var/folders/1s/sm2bn6kd64gdnv757t9qshcm0000gn/T/pytest-of-jonathanhaas/pytest-41/test_evalops_integration_calle0/train.jsonl", "eval_tasks": [], "renderer_name": "default", "learning_rate": 0.0001, "eval_threshold": 0.8, "max_rounds": 1, "lr_decay": 0.8, "evalops_enabled": true, "evalops_test_suite_id": "suite-123", "evalops_api_url": null, "steps_per_round": 1, "batch_size": 8, "max_seq_length": 2048, "lora_rank": 16, "warmup_steps": 100, "max_steps": 1000, "min_lr": 1e-06, "use_recommended_lr": false}
12+
{"timestamp": "2025-10-01T18:52:48.431038", "run_id": "20251001_185248", "event": "checkpoint", "round": 1, "checkpoint_uri": "<MagicMock name='mock.create_lora_training_client().save_weights_for_sampler().result().path' id='4498415232'>"}
13+
{"timestamp": "2025-10-01T18:52:48.431243", "run_id": "20251001_185248", "event": "evaluation", "round": 1, "score": 0.9, "threshold": 0.8, "passed": true, "metrics": {}}
14+
{"timestamp": "2025-10-01T18:52:48.432839", "run_id": "20251001_185248", "event": "config", "base_model": "test-model", "train_file": "/private/var/folders/1s/sm2bn6kd64gdnv757t9qshcm0000gn/T/pytest-of-jonathanhaas/pytest-41/test_lr_decay_across_rounds0/train.jsonl", "eval_tasks": [], "renderer_name": "default", "learning_rate": 1.0, "eval_threshold": 0.99, "max_rounds": 3, "lr_decay": 0.5, "evalops_enabled": false, "evalops_test_suite_id": null, "evalops_api_url": null, "steps_per_round": 1, "batch_size": 8, "max_seq_length": 2048, "lora_rank": 16, "warmup_steps": 0, "max_steps": 1000, "min_lr": 1e-06, "use_recommended_lr": false}
15+
{"timestamp": "2025-10-01T18:52:48.433169", "run_id": "20251001_185248", "event": "checkpoint", "round": 1, "checkpoint_uri": "<MagicMock name='mock.create_lora_training_client().save_weights_for_sampler().result().path' id='4503095008'>"}
16+
{"timestamp": "2025-10-01T18:52:48.433329", "run_id": "20251001_185248", "event": "evaluation", "round": 1, "score": 0.7, "threshold": 0.99, "passed": false, "metrics": {}}
17+
{"timestamp": "2025-10-01T18:52:48.433460", "run_id": "20251001_185248", "event": "checkpoint", "round": 2, "checkpoint_uri": "<MagicMock name='mock.create_lora_training_client().save_weights_for_sampler().result().path' id='4503095008'>"}
18+
{"timestamp": "2025-10-01T18:52:48.433680", "run_id": "20251001_185248", "event": "evaluation", "round": 2, "score": 0.7, "threshold": 0.99, "passed": false, "metrics": {}}
19+
{"timestamp": "2025-10-01T18:52:48.433796", "run_id": "20251001_185248", "event": "checkpoint", "round": 3, "checkpoint_uri": "<MagicMock name='mock.create_lora_training_client().save_weights_for_sampler().result().path' id='4503095008'>"}
20+
{"timestamp": "2025-10-01T18:52:48.433937", "run_id": "20251001_185248", "event": "evaluation", "round": 3, "score": 0.7, "threshold": 0.99, "passed": false, "metrics": {}}

‎runs/20251001_185248/run_state.json‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"round_idx": 3,
3+
"global_step": 3,
4+
"learning_rate": 0.25,
5+
"checkpoint_uri": "<MagicMock name='mock.create_lora_training_client().save_weights_for_sampler().result().path' id='4503095008'>",
6+
"config": {
7+
"base_model": "test-model",
8+
"train_file": "/private/var/folders/1s/sm2bn6kd64gdnv757t9qshcm0000gn/T/pytest-of-jonathanhaas/pytest-41/test_lr_decay_across_rounds0/train.jsonl",
9+
"eval_tasks": [],
10+
"renderer_name": "default",
11+
"learning_rate": 1.0,
12+
"eval_threshold": 0.99,
13+
"max_rounds": 3,
14+
"lr_decay": 0.5,
15+
"evalops_enabled": false,
16+
"evalops_test_suite_id": null,
17+
"evalops_api_url": null,
18+
"steps_per_round": 1,
19+
"batch_size": 8,
20+
"max_seq_length": 2048,
21+
"lora_rank": 16,
22+
"warmup_steps": 0,
23+
"max_steps": 1000,
24+
"min_lr": 1e-06,
25+
"use_recommended_lr": false
26+
},
27+
"timestamp": "2025-10-01T18:52:48.433866"
28+
}

‎runs/20251001_185258/metrics.jsonl‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{"timestamp": "2025-10-01T18:52:58.032742", "run_id": "20251001_185258", "event": "config", "base_model": "meta-llama/Llama-3.1-8B-Instruct", "train_file": "demo_data.jsonl", "eval_tasks": ["simple_qa"], "renderer_name": "llama3", "learning_rate": 0.0003, "eval_threshold": 0.75, "max_rounds": 3, "lr_decay": 0.6, "evalops_enabled": false, "evalops_test_suite_id": "", "evalops_api_url": null, "steps_per_round": 5, "batch_size": 4, "max_seq_length": 512, "lora_rank": 16, "warmup_steps": 100, "max_steps": 1000, "min_lr": 1e-06, "use_recommended_lr": false}
2+
{"timestamp": "2025-10-01T18:52:58.144341", "run_id": "20251001_185258", "event": "checkpoint", "round": 1, "checkpoint_uri": "mock://checkpoint/round_1"}
3+
{"timestamp": "2025-10-01T18:52:58.144892", "run_id": "20251001_185258", "event": "evaluation", "round": 1, "score": 0.4, "threshold": 0.75, "passed": false, "metrics": {}}
4+
{"timestamp": "2025-10-01T18:52:58.256871", "run_id": "20251001_185258", "event": "checkpoint", "round": 2, "checkpoint_uri": "mock://checkpoint/round_2"}
5+
{"timestamp": "2025-10-01T18:52:58.257349", "run_id": "20251001_185258", "event": "evaluation", "round": 2, "score": 0.6, "threshold": 0.75, "passed": false, "metrics": {}}
6+
{"timestamp": "2025-10-01T18:52:58.368808", "run_id": "20251001_185258", "event": "checkpoint", "round": 3, "checkpoint_uri": "mock://checkpoint/round_3"}
7+
{"timestamp": "2025-10-01T18:52:58.370565", "run_id": "20251001_185258", "event": "evaluation", "round": 3, "score": 1.0, "threshold": 0.75, "passed": true, "metrics": {}}

‎runs/20251001_185258/run_state.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"round_idx": 3,
3+
"global_step": 15,
4+
"learning_rate": 0.00010799999999999998,
5+
"checkpoint_uri": "mock://checkpoint/round_3",
6+
"config": {
7+
"base_model": "meta-llama/Llama-3.1-8B-Instruct",
8+
"train_file": "demo_data.jsonl",
9+
"eval_tasks": [
10+
"simple_qa"
11+
],
12+
"renderer_name": "llama3",
13+
"learning_rate": 0.0003,
14+
"eval_threshold": 0.75,
15+
"max_rounds": 3,
16+
"lr_decay": 0.6,
17+
"evalops_enabled": false,
18+
"evalops_test_suite_id": "",
19+
"evalops_api_url": null,
20+
"steps_per_round": 5,
21+
"batch_size": 4,
22+
"max_seq_length": 512,
23+
"lora_rank": 16,
24+
"warmup_steps": 100,
25+
"max_steps": 1000,
26+
"min_lr": 1e-06,
27+
"use_recommended_lr": false
28+
},
29+
"timestamp": "2025-10-01T18:52:58.369674"
30+
}

0 commit comments

Comments
 (0)