|
15 | 15 |
|
16 | 16 | set -exo pipefail |
17 | 17 |
|
| 18 | +COVERAGE_ARGS="--data-file=/workspace/.coverage --source=/workspace/ --parallel-mode" |
| 19 | + |
18 | 20 | # Run the biencoder recipe (uses nemo_automodel/recipes/biencoder/train_biencoder.py via module entrypoint). |
19 | | -python3 -m nemo_automodel.recipes.biencoder.train_biencoder \ |
| 21 | +python3 -m coverage run ${COVERAGE_ARGS} \ |
| 22 | + -m nemo_automodel.recipes.biencoder.train_biencoder \ |
20 | 23 | --config \ |
21 | 24 | tests/functional_tests/llm_pretrain_and_kd/customizer_retrieval/recipe.yaml \ |
22 | 25 | --model.pretrained_model_name_or_path $TEST_DATA_DIR/llama-nemotron-embed-1b-v2/ \ |
23 | 26 | --tokenizer.pretrained_model_name_or_path $TEST_DATA_DIR/llama-nemotron-embed-1b-v2/ \ |
24 | 27 | --dataloader.dataset.data_dir_list $TEST_DATA_DIR/embedding_testdata/training.jsonl \ |
25 | 28 |
|
26 | 29 | # Compare baseline vs finetuned biencoder checkpoint (pos-neg separation should not degrade). |
27 | | -python3 \ |
| 30 | +python3 -m coverage run --append ${COVERAGE_ARGS} \ |
28 | 31 | tests/functional_tests/llm_pretrain_and_kd/customizer_retrieval/compare_biencoder_models.py \ |
29 | 32 | $TEST_DATA_DIR/llama-nemotron-embed-1b-v2 \ |
30 | 33 | /workspace/output/biencoder_inline/checkpoints/epoch_0_step_31/ \ |
31 | 34 | $TEST_DATA_DIR/embedding_testdata/testing.jsonl \ |
32 | 35 | true |
33 | 36 |
|
34 | | -# ---- Checkpoint restoration tests ---- |
| 37 | +# Checkpoint restoration tests |
35 | 38 | # Test 1: Full-model checkpoint restoration (NeMo -> save -> transformers load) |
36 | 39 | # Test 2: PEFT (LoRA) checkpoint restoration (NeMo -> save -> transformers + safetensors load) |
37 | 40 | BASE_MODEL_PATH=$TEST_DATA_DIR/llama-nemotron-embed-1b-v2 \ |
38 | 41 | CHECKPOINT_DIR=/workspace/output/biencoder_ckpt_restore/checkpoints \ |
39 | 42 | PEFT_CHECKPOINT_DIR=/workspace/output/biencoder_ckpt_restore_peft/checkpoints \ |
40 | 43 | RECIPE_YAML=tests/functional_tests/llm_pretrain_and_kd/customizer_retrieval/recipe_ckpt_restore.yaml \ |
41 | 44 | PEFT_RECIPE_YAML=tests/functional_tests/llm_pretrain_and_kd/customizer_retrieval/recipe_peft.yaml \ |
42 | | -python3 -m pytest -xvs \ |
| 45 | +python3 -m coverage run --append ${COVERAGE_ARGS} \ |
| 46 | + -m pytest -xvs \ |
43 | 47 | tests/functional_tests/llm_pretrain_and_kd/customizer_retrieval/test_biencoder_checkpoint_restoration.py |
0 commit comments