Skip to content

Commit 281f6cd

Browse files
nv-brafnv-braf
authored
Add remote mode config sweep to brute search (#751)
* Brute run in remote mode changes * Fixing L0 test * Fixes based on review comments * Fix L0
1 parent 6ed033c commit 281f6cd

File tree

7 files changed

+42
-59
lines changed

7 files changed

+42
-59
lines changed

‎model_analyzer/config/generate/brute_run_config_generator.py‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -102,11 +102,7 @@ def _get_next_config(self) -> Generator[RunConfig, None, None]:
102102
if not self._skip_default_config:
103103
yield from self._generate_subset(0, default_only=True)
104104

105-
if self._should_generate_non_default_configs():
106-
yield from self._generate_subset(0, default_only=False)
107-
108-
def _should_generate_non_default_configs(self) -> bool:
109-
return self._config.triton_launch_mode != "remote"
105+
yield from self._generate_subset(0, default_only=False)
110106

111107
def _generate_subset(
112108
self, index: int, default_only: bool

‎model_analyzer/config/generate/manual_model_config_generator.py‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -130,23 +130,7 @@ def _get_next_model_config_variant(self) -> ModelConfigVariant:
130130

131131
def _generate_model_config_variants(self) -> List[List[ModelConfigVariant]]:
132132
"""Generate all model config combinations"""
133-
if self._remote_mode:
134-
configs = self._generate_remote_mode_model_config_variants()
135-
else:
136-
configs = self._generate_direct_modes_model_config_variants()
137-
138-
return configs
139-
140-
def _generate_remote_mode_model_config_variants(
141-
self,
142-
) -> List[List[ModelConfigVariant]]:
143-
"""Generate model configs for remote mode"""
144-
return [[self._make_remote_model_config_variant()]]
145133

146-
def _generate_direct_modes_model_config_variants(
147-
self,
148-
) -> List[List[ModelConfigVariant]]:
149-
"""Generate model configs for direct (non-remote) modes"""
150134
model_config_variants = []
151135
for param_combo in self._non_max_batch_size_param_combos:
152136
configs_with_max_batch_size = []
@@ -171,9 +155,6 @@ def _determine_max_batch_sizes_and_param_combos(self) -> None:
171155
"""
172156
Determine self._max_batch_sizes and self._non_max_batch_size_param_combos
173157
"""
174-
if self._remote_mode:
175-
return
176-
177158
if self._default_only:
178159
self._non_max_batch_size_param_combos = [DEFAULT_CONFIG_PARAMS]
179160
else:

‎model_analyzer/config/generate/model_config_generator_factory.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,10 @@ def create_model_config_generator(
6767
A generator that implements ConfigGeneratorInterface and creates ModelConfigs
6868
"""
6969

70-
remote_mode = config.triton_launch_mode == "remote"
7170
search_disabled = config.run_config_search_disable
7271
model_config_params = model.model_config_parameters()
7372

74-
if remote_mode or search_disabled or model_config_params:
73+
if search_disabled or model_config_params:
7574
return ManualModelConfigGenerator(
7675
config,
7776
gpus,

‎qa/L0_config_search/test.sh‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,6 @@ for launch_mode in $TRITON_LAUNCH_MODES; do
7070
MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS $MODEL_ANALYZER_PORTS"
7171

7272
if [ $launch_mode == 'remote' ]; then
73-
NUM_ROW_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-param-$launch_mode.txt
74-
NUM_MODELS_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-models-$launch_mode.txt
75-
7673
# For remote launch, set server args and start server
7774
SERVER=`which tritonserver`
7875
SERVER_ARGS="--model-repository=$MODEL_REPOSITORY --model-control-mode=explicit --http-port $http_port --grpc-port $grpc_port --metrics-port $metrics_port"
@@ -84,11 +81,10 @@ for launch_mode in $TRITON_LAUNCH_MODES; do
8481
cat $SERVER_LOG
8582
RET=1
8683
fi
87-
else
88-
NUM_MODELS_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-models.txt
89-
NUM_ROW_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-param.txt
9084
fi
9185

86+
NUM_MODELS_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-models.txt
87+
NUM_ROW_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-param.txt
9288
TEST_OUTPUT_NUM_ROWS=`cat $NUM_ROW_OUTPUT_FILE`
9389
TEST_MODELS_NUM=`cat $NUM_MODELS_OUTPUT_FILE`
9490
MODEL_ANALYZER_SUBCOMMAND="profile"

‎qa/L0_config_search/test_config_generator.py‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ def generate_search_disable(self):
7070
total_param_count = self._calculate_total_params(
7171
concurrency_count, instance_count
7272
)
73-
self._write_file(total_param_count, 1, 2, 1, model_config)
73+
self._write_file(total_param_count, 2, model_config)
7474

7575
def generate_max_limit_with_model_config(self):
7676
concurrency_count = 2
@@ -91,7 +91,7 @@ def generate_max_limit_with_model_config(self):
9191
total_param_count = self._calculate_total_params(
9292
concurrency_count, instance_count
9393
)
94-
self._write_file(total_param_count, 2, 2, 1, model_config)
94+
self._write_file(total_param_count, 2, model_config)
9595

9696
def generate_max_limit(self):
9797
concurrency_count = 2
@@ -105,7 +105,7 @@ def generate_max_limit(self):
105105
total_param_count = self._calculate_total_params(
106106
concurrency_count, instance_count
107107
)
108-
self._write_file(total_param_count, 2, 8, 1, model_config)
108+
self._write_file(total_param_count, 8, model_config)
109109

110110
def generate_max_limit_with_param(self):
111111
concurrency_count = 1 # 1 because concurrency parameter is 1 entry below
@@ -124,7 +124,7 @@ def generate_max_limit_with_param(self):
124124
total_param_count = self._calculate_total_params(
125125
concurrency_count, instance_count
126126
)
127-
self._write_file(total_param_count, 1, 6, 1, model_config)
127+
self._write_file(total_param_count, 6, model_config)
128128

129129
def generate_max_limit_with_param_and_model_config(self):
130130
concurrency_count = 1 # 1 because concurrency parameter is 1 entry below
@@ -146,7 +146,7 @@ def generate_max_limit_with_param_and_model_config(self):
146146
total_param_count = self._calculate_total_params(
147147
concurrency_count, instance_count
148148
)
149-
self._write_file(total_param_count, 1, 2, 1, model_config)
149+
self._write_file(total_param_count, 2, model_config)
150150

151151
def generate_max_limit_with_dynamic_batch_disable(self):
152152
concurrency_count = 2
@@ -160,7 +160,7 @@ def generate_max_limit_with_dynamic_batch_disable(self):
160160
total_param_count = self._calculate_total_params(
161161
concurrency_count, instance_count
162162
)
163-
self._write_file(total_param_count, 2, 4, 1, model_config)
163+
self._write_file(total_param_count, 4, model_config)
164164

165165
def _calculate_total_params(
166166
self, concurrency_count, instance_count, default_config_count=1
@@ -178,19 +178,13 @@ def _calculate_total_params(
178178
def _write_file(
179179
self,
180180
total_param,
181-
total_param_remote,
182181
total_models,
183-
total_models_remote,
184182
model_config,
185183
):
186184
with open(f"./config-{self.test_id}-param.txt", "w") as file:
187185
file.write(str(total_param))
188-
with open(f"./config-{self.test_id}-param-remote.txt", "w") as file:
189-
file.write(str(total_param_remote))
190186
with open(f"./config-{self.test_id}-models.txt", "w") as file:
191187
file.write(str(total_models))
192-
with open(f"./config-{self.test_id}-models-remote.txt", "w") as file:
193-
file.write(str(total_models_remote))
194188
with open(f"./config-{self.test_id}.yml", "w") as file:
195189
yaml.dump(model_config, file)
196190

‎tests/test_model_config_generator.py‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -455,8 +455,6 @@ def test_direct_nonempty_default_config(self):
455455
def test_remote_yes_params_specified(self):
456456
"""
457457
Test remote mode with model_config_parameters specified
458-
459-
It should always return a single empty config in remote mode
460458
"""
461459

462460
# yapf: disable
@@ -473,27 +471,39 @@ def test_remote_yes_params_specified(self):
473471
count: [1,2]
474472
""")
475473

476-
expected_configs = [{}]
474+
expected_configs = [
475+
{'instance_group': [{'count': 1, 'kind': 'KIND_GPU'}], 'max_batch_size': 1},
476+
{'instance_group': [{'count': 1, 'kind': 'KIND_GPU'}], 'max_batch_size': 4},
477+
{'instance_group': [{'count': 1, 'kind': 'KIND_GPU'}], 'max_batch_size': 16},
478+
{'instance_group': [{'count': 2, 'kind': 'KIND_GPU'}], 'max_batch_size': 1},
479+
{'instance_group': [{'count': 2, 'kind': 'KIND_GPU'}], 'max_batch_size': 4},
480+
{'instance_group': [{'count': 2, 'kind': 'KIND_GPU'}], 'max_batch_size': 16}
481+
]
477482
# yapf: enable
478483

479484
self._run_and_test_model_config_generator(yaml_str, expected_configs)
480485

481486
def test_remote_no_params_specified(self):
482487
"""
483488
Test remote mode with no model_config_parameters specified
484-
485-
It should always return a single empty config in remote mode
486489
"""
487490

488491
# yapf: disable
489492
yaml_str = ("""
490493
triton_launch_mode: remote
491-
run_config_search_max_instance_count: 16
494+
run_config_search_max_instance_count: 4
495+
run_config_search_min_model_batch_size: 8
496+
run_config_search_max_model_batch_size: 8
492497
profile_models:
493498
- my-model
494499
""")
495500

496-
expected_configs = [{}]
501+
expected_configs = [
502+
{'max_batch_size': 8, 'instance_group': [{'count': 1, 'kind': 'KIND_GPU'}],'dynamic_batching': {}},
503+
{'max_batch_size': 8, 'instance_group': [{'count': 2, 'kind': 'KIND_GPU'}],'dynamic_batching': {}},
504+
{'max_batch_size': 8, 'instance_group': [{'count': 3, 'kind': 'KIND_GPU'}],'dynamic_batching': {}},
505+
{'max_batch_size': 8, 'instance_group': [{'count': 4, 'kind': 'KIND_GPU'}],'dynamic_batching': {}}
506+
]
497507
# yapf: enable
498508

499509
self._run_and_test_model_config_generator(yaml_str, expected_configs)

‎tests/test_model_manager.py‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -257,24 +257,31 @@ def test_manual_concurrency(self):
257257
def test_remote_mode(self):
258258
"""
259259
Test remote mode
260-
261-
In remote mode all model_config_parameters (ie. instance count) are ignored
262260
"""
263261

264262
expected_ranges = [
265263
{
266-
"instances": [None],
267-
"kind": [None],
264+
"instances": [1, 2],
265+
"kind": ["KIND_GPU"],
266+
"batching": [0],
267+
"batch_sizes": [1],
268+
"max_batch_size": [8],
269+
"concurrency": [1, 2, 4, 8, 16],
270+
},
271+
{
272+
"instances": [1],
273+
"kind": ["KIND_CPU"],
268274
"batching": [None],
269275
"batch_sizes": [1],
270-
"concurrency": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
271-
}
276+
"max_batch_size": [8],
277+
"concurrency": [1, 2, 4, 8, 16],
278+
},
272279
]
273280

274281
yaml_str = """
275282
profile_models: test_model
276-
run_config_search_max_concurrency: 512
277-
run_config_search_max_instance_count: 7
283+
run_config_search_max_concurrency: 16
284+
run_config_search_max_instance_count: 2
278285
run_config_search_min_model_batch_size: 8
279286
run_config_search_max_model_batch_size: 8
280287
run_config_search_disable: False

0 commit comments

Comments
 (0)