Skip to content

Commit cf0a5a0

Browse files
YuanTingHsiehYuanTingHsieh
committed
[2.7] Fix cifar10 integration tests (NVIDIA#4150)
Fix cifar10 integration tests ### Description Fix cifar10 integration tests ### Types of changes <!--- Put an `x` in all the boxes that apply, and remove the not applicable items --> - [x] Non-breaking change (fix or new feature that would not break existing functionality). - [ ] Breaking change (fix or new feature that would cause existing functionality to change). - [ ] New tests added to cover the changes. - [ ] Quick tests passed locally by running `./runtest.sh`. - [ ] In-line docstrings updated. - [ ] Documentation updated.
1 parent 902bf98 commit cf0a5a0

File tree

10 files changed

+375
-119
lines changed

10 files changed

+375
-119
lines changed

‎ci/run_integration.sh‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,6 @@ integration_test() {
8383
add_dns_entries
8484
testFolder="tests/integration_test"
8585
pushd ${testFolder}
86-
pipenv run ./generate_test_configs_for_examples.sh
8786
pipenv run ./run_integration_tests.sh -m "$1"
8887
popd
8988
clean_up_snapshot_and_job

‎tests/integration_test/data/test_configs/standalone_job/cifar_examples.yml‎

Lines changed: 121 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,94 @@
1-
jobs_root_dir: ../../examples/advanced/cifar10/cifar10-real-world/jobs
1+
# CIFAR10 integration tests using recipe-based examples
2+
# These tests export recipe-based jobs to traditional job folders for testing
3+
#
4+
# All CIFAR10 PT examples share the same data preparation and model structure.
5+
# Each test exports the actual job.py from examples to ensure consistency.
6+
7+
jobs_root_dir: ./data/jobs
28
cleanup: true
3-
project_yaml: ./data/projects/2_clients_with_he.yml
4-
additional_python_paths:
5-
- ../../examples/advanced/cifar10
9+
project_yaml: ./data/projects/dummy.yml
10+
# No additional_python_paths needed - export_recipe_job.py copies src/ contents directly to app/custom/
611

712
tests:
8-
- test_name: Test a simplified copy of job cifar10_fedavg_he for example cifar10-real-world.
13+
# =============================================================================
14+
# CIFAR10 Simulation Examples (cifar10-sim)
15+
# =============================================================================
16+
17+
- test_name: Test CIFAR10 FedAvg (recipe-based)
18+
event_sequence:
19+
- trigger:
20+
type: server_log
21+
data: Server started
22+
actions: [ "submit_job cifar10_fedavg_test" ]
23+
result:
24+
type: job_submit_success
25+
- trigger:
26+
type: run_state
27+
data: { "run_finished": True }
28+
actions: [ "ensure_current_job_done" ]
29+
result:
30+
type: run_state
31+
data: { "run_finished": True }
32+
validators:
33+
- path: tests.integration_test.src.validators.CIFAR10ResultValidator
34+
setup:
35+
# Install requirements (strip nvflare since we use dev version)
36+
- REQ=$(mktemp) && grep -v "^nvflare" ../../examples/advanced/cifar10/pt/cifar10-sim/requirements.txt > "$REQ" && pip install -r "$REQ" --quiet && rm -f "$REQ"
37+
# Export runs the actual job.py which also prepares data
38+
- python export_recipe_job.py --recipe_dir ../../examples/advanced/cifar10/pt/cifar10-sim/cifar10_fedavg --output_dir ./data/jobs --recipe_args "--n_clients 2 --num_rounds 2 --name cifar10_fedavg_test"
39+
teardown:
40+
- rm -rf ./data/jobs/cifar10_fedavg_test
41+
42+
- test_name: Test CIFAR10 FedOpt (recipe-based)
43+
event_sequence:
44+
- trigger:
45+
type: server_log
46+
data: Server started
47+
actions: [ "submit_job cifar10_fedopt_test" ]
48+
result:
49+
type: job_submit_success
50+
- trigger:
51+
type: run_state
52+
data: { "run_finished": True }
53+
actions: [ "ensure_current_job_done" ]
54+
result:
55+
type: run_state
56+
data: { "run_finished": True }
57+
validators:
58+
- path: tests.integration_test.src.validators.CIFAR10ResultValidator
59+
setup:
60+
- python export_recipe_job.py --recipe_dir ../../examples/advanced/cifar10/pt/cifar10-sim/cifar10_fedopt --output_dir ./data/jobs --recipe_args "--n_clients 2 --num_rounds 2 --name cifar10_fedopt_test"
61+
teardown:
62+
- rm -rf ./data/jobs/cifar10_fedopt_test
63+
64+
- test_name: Test CIFAR10 FedProx (recipe-based)
65+
event_sequence:
66+
- trigger:
67+
type: server_log
68+
data: Server started
69+
actions: [ "submit_job cifar10_fedprox_test" ]
70+
result:
71+
type: job_submit_success
72+
- trigger:
73+
type: run_state
74+
data: { "run_finished": True }
75+
actions: [ "ensure_current_job_done" ]
76+
result:
77+
type: run_state
78+
data: { "run_finished": True }
79+
validators:
80+
- path: tests.integration_test.src.validators.CIFAR10ResultValidator
81+
setup:
82+
- python export_recipe_job.py --recipe_dir ../../examples/advanced/cifar10/pt/cifar10-sim/cifar10_fedprox --output_dir ./data/jobs --recipe_args "--n_clients 2 --num_rounds 2 --name cifar10_fedprox_test"
83+
teardown:
84+
- rm -rf ./data/jobs/cifar10_fedprox_test
85+
86+
- test_name: Test CIFAR10 SCAFFOLD (recipe-based)
987
event_sequence:
1088
- trigger:
1189
type: server_log
1290
data: Server started
13-
actions: [ "submit_job cifar10_fedavg_he_copy" ]
91+
actions: [ "submit_job cifar10_scaffold_test" ]
1492
result:
1593
type: job_submit_success
1694
- trigger:
@@ -23,20 +101,16 @@ tests:
23101
validators:
24102
- path: tests.integration_test.src.validators.CIFAR10ResultValidator
25103
setup:
26-
- cp ../../examples/advanced/cifar10/cifar10-real-world/requirements.txt ../../examples/advanced/cifar10/cifar10-real-world/temp_requirements.txt
27-
- sed -i '/nvflare\|jupyter\|notebook/d' ../../examples/advanced/cifar10/cifar10-real-world/temp_requirements.txt
28-
- python -m pip install -r ../../examples/advanced/cifar10/cifar10-real-world/temp_requirements.txt
29-
- rm -f ../../examples/advanced/cifar10/cifar10-real-world/temp_requirements.txt
30-
- bash ../../examples/advanced/cifar10/cifar10-real-world/prepare_data.sh
31-
- python convert_to_test_job.py --job ../../examples/advanced/cifar10/cifar10-real-world/jobs/cifar10_fedavg_he --post _copy
104+
- python export_recipe_job.py --recipe_dir ../../examples/advanced/cifar10/pt/cifar10-sim/cifar10_scaffold --output_dir ./data/jobs --recipe_args "--n_clients 2 --num_rounds 2 --name cifar10_scaffold_test"
32105
teardown:
33-
- rm -rf ../../examples/advanced/cifar10/cifar10-real-world/jobs/cifar10_fedavg_he_copy
34-
- test_name: Test a simplified copy of job cifar10_fedavg_stream_tb for example cifar10-real-world.
106+
- rm -rf ./data/jobs/cifar10_scaffold_test
107+
108+
- test_name: Test CIFAR10 Custom Aggregators (recipe-based)
35109
event_sequence:
36110
- trigger:
37111
type: server_log
38112
data: Server started
39-
actions: [ "submit_job cifar10_fedavg_stream_tb_copy" ]
113+
actions: [ "submit_job cifar10_custom_aggr_test" ]
40114
result:
41115
type: job_submit_success
42116
- trigger:
@@ -49,10 +123,36 @@ tests:
49123
validators:
50124
- path: tests.integration_test.src.validators.CIFAR10ResultValidator
51125
setup:
52-
- cp ../../examples/advanced/cifar10/cifar10-real-world/requirements.txt ../../examples/advanced/cifar10/cifar10-real-world/temp_requirements.txt
53-
- sed -i '/nvflare\|jupyter\|notebook/d' ../../examples/advanced/cifar10/cifar10-real-world/temp_requirements.txt
54-
- python -m pip install -r ../../examples/advanced/cifar10/cifar10-real-world/temp_requirements.txt
55-
- rm -f ../../examples/advanced/cifar10/cifar10-real-world/temp_requirements.txt
56-
- python convert_to_test_job.py --job ../../examples/advanced/cifar10/cifar10-real-world/jobs/cifar10_fedavg_stream_tb --post _copy
126+
- python export_recipe_job.py --recipe_dir ../../examples/advanced/cifar10/pt/cifar10-sim/cifar10_custom_aggr --output_dir ./data/jobs --recipe_args "--n_clients 2 --num_rounds 2 --name cifar10_custom_aggr_test --aggregator weighted"
57127
teardown:
58-
- rm -rf ../../examples/advanced/cifar10/cifar10-real-world/jobs/cifar10_fedavg_stream_tb_copy
128+
- rm -rf ./data/jobs/cifar10_custom_aggr_test
129+
130+
# =============================================================================
131+
# CIFAR10 Real-World Examples (cifar10-real-world) - Requires HE project
132+
# =============================================================================
133+
134+
# Note: These tests require HE-enabled project configuration
135+
# Uncomment and use project_yaml: ./data/projects/2_clients_with_he.yml
136+
137+
# - test_name: Test CIFAR10 FedAvg with Homomorphic Encryption (recipe-based)
138+
# event_sequence:
139+
# - trigger:
140+
# type: server_log
141+
# data: Server started
142+
# actions: [ "submit_job cifar10_fedavg_he_test" ]
143+
# result:
144+
# type: job_submit_success
145+
# - trigger:
146+
# type: run_state
147+
# data: { "run_finished": True }
148+
# actions: [ "ensure_current_job_done" ]
149+
# result:
150+
# type: run_state
151+
# data: { "run_finished": True }
152+
# validators:
153+
# - path: tests.integration_test.src.validators.CIFAR10ResultValidator
154+
# setup:
155+
# - mkdir -p ./data/jobs
156+
# - python export_recipe_job.py --recipe_dir ../../examples/advanced/cifar10/pt/cifar10-real-world/cifar10_fedavg_he --output_dir ./data/jobs --recipe_args "--n_clients 2 --num_rounds 2 --name cifar10_fedavg_he_test"
157+
# teardown:
158+
# - rm -rf ./data/jobs/cifar10_fedavg_he_test

‎tests/integration_test/example_registry.yml‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

0 commit comments

Comments
 (0)