Skip to content

Commit c157d15

Browse files
nv-brafmc-nv
nv-braf
authored and
mc-nv
committed
Added L0 quick search test (#521)
* Added L0 quick search test * Fixing copyrights
1 parent 25b27d6 commit c157d15

File tree

3 files changed

+244
-0
lines changed

3 files changed

+244
-0
lines changed

‎qa/L0_quick_search/check_results.py‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from collections import defaultdict
16+
import argparse
17+
import sys
18+
import yaml
19+
import os
20+
21+
22+
class TestOutputValidator:
23+
"""
24+
Functions that validate the output
25+
of the test
26+
"""
27+
28+
def __init__(self, config, test_name, analyzer_log):
29+
self._config = config
30+
self._models = config['profile_models']
31+
self._analyzer_log = analyzer_log
32+
33+
check_function = self.__getattribute__(f'check_{test_name}')
34+
35+
if check_function():
36+
sys.exit(0)
37+
else:
38+
sys.exit(1)
39+
40+
def check_profile_logs(self):
41+
"""
42+
Check that each model was profiled the number of times
43+
corresponding with batch size and concurrency combinations
44+
45+
(No model config parameter combos expected here!)
46+
"""
47+
48+
with open(self._analyzer_log, 'r') as f:
49+
log_contents = f.read()
50+
51+
expected_min_num_measurements = 20
52+
expected_max_num_measurements = 80
53+
for model in self._models:
54+
token = f"Profiling {model}_config"
55+
token_idx = 0
56+
found_count = 0
57+
while True:
58+
token_idx = log_contents.find(token, token_idx + 1)
59+
if token_idx == -1:
60+
break
61+
found_count += 1
62+
if found_count < expected_min_num_measurements or found_count > expected_max_num_measurements:
63+
print(
64+
f"\n***\n*** Expected range of measurements for {model} : {expected_min_num_measurements} to {expected_max_num_measurements}. "
65+
f"Found {found_count}. \n***")
66+
return False
67+
return True
68+
69+
70+
if __name__ == '__main__':
71+
parser = argparse.ArgumentParser()
72+
parser.add_argument('-f',
73+
'--config-file',
74+
type=str,
75+
required=True,
76+
help='The path to the config yaml file.')
77+
parser.add_argument('-l',
78+
'--analyzer-log-file',
79+
type=str,
80+
required=True,
81+
help='The full path to the analyzer log.')
82+
parser.add_argument('-t',
83+
'--test-name',
84+
type=str,
85+
required=True,
86+
help='The name of the test to be run.')
87+
args = parser.parse_args()
88+
89+
with open(args.config_file, 'r') as f:
90+
config = yaml.safe_load(f)
91+
92+
TestOutputValidator(config, args.test_name, args.analyzer_log_file)

‎qa/L0_quick_search/test.sh‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
ANALYZER_LOG="test.log"
16+
source ../common/util.sh
17+
source ../common/check_analyzer_results.sh
18+
19+
rm -f *.log
20+
rm -rf results && mkdir -p results
21+
22+
# Set test parameters
23+
MODEL_ANALYZER="`which model-analyzer`"
24+
REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION}
25+
MODEL_REPOSITORY=${MODEL_REPOSITORY:="/mnt/nvdl/datasets/inferenceserver/$REPO_VERSION/libtorch_model_store"}
26+
QA_MODELS="resnet50_libtorch"
27+
MODEL_NAMES="$(echo $QA_MODELS | sed 's/ /,/g')"
28+
TRITON_LAUNCH_MODE=${TRITON_LAUNCH_MODE:="local"}
29+
CLIENT_PROTOCOL="grpc"
30+
PORTS=(`find_available_ports 3`)
31+
GPUS=(`get_all_gpus_uuids`)
32+
OUTPUT_MODEL_REPOSITORY=${OUTPUT_MODEL_REPOSITORY:=`get_output_directory`}
33+
CONFIG_FILE="config.yml"
34+
EXPORT_PATH="`pwd`/results"
35+
FILENAME_SERVER_ONLY="server-metrics.csv"
36+
FILENAME_INFERENCE_MODEL="model-metrics-inference.csv"
37+
FILENAME_GPU_MODEL="model-metrics-gpu.csv"
38+
39+
rm -rf $OUTPUT_MODEL_REPOSITORY
40+
41+
python3 test_config_generator.py --profile-models $MODEL_NAMES
42+
43+
# Run the analyzer and check the results
44+
RET=0
45+
46+
set +e
47+
48+
MODEL_ANALYZER_ARGS="-m $MODEL_REPOSITORY -f $CONFIG_FILE"
49+
MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --client-protocol=$CLIENT_PROTOCOL --triton-launch-mode=$TRITON_LAUNCH_MODE --run-config-search-mode=quick"
50+
MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-http-endpoint localhost:${PORTS[0]} --triton-grpc-endpoint localhost:${PORTS[1]}"
51+
MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-metrics-url http://localhost:${PORTS[2]}/metrics"
52+
MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --output-model-repository-path $OUTPUT_MODEL_REPOSITORY --override-output-model-repository"
53+
MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS -e $EXPORT_PATH --filename-server-only=$FILENAME_SERVER_ONLY"
54+
MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --filename-model-inference=$FILENAME_INFERENCE_MODEL --filename-model-gpu=$FILENAME_GPU_MODEL"
55+
MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --skip-summary-reports"
56+
MODEL_ANALYZER_SUBCOMMAND="profile"
57+
run_analyzer
58+
if [ $? -ne 0 ]; then
59+
echo -e "\n***\n*** Test Failed. model-analyzer $MODEL_ANALYZER_SUBCOMMAND exited with non-zero exit code. \n***"
60+
cat $ANALYZER_LOG
61+
RET=1
62+
else
63+
# Check the Analyzer log for correct output
64+
TEST_NAME='profile_logs'
65+
python3 check_results.py -f $CONFIG_FILE -t $TEST_NAME -l $ANALYZER_LOG
66+
if [ $? -ne 0 ]; then
67+
echo -e "\n***\n*** Test Output Verification Failed for $TEST_NAME test.\n***"
68+
cat $ANALYZER_LOG
69+
RET=1
70+
fi
71+
72+
SERVER_METRICS_FILE=${EXPORT_PATH}/results/${FILENAME_SERVER_ONLY}
73+
MODEL_METRICS_GPU_FILE=${EXPORT_PATH}/results/${FILENAME_GPU_MODEL}
74+
MODEL_METRICS_INFERENCE_FILE=${EXPORT_PATH}/results/${FILENAME_INFERENCE_MODEL}
75+
76+
for file in SERVER_METRICS_FILE, MODEL_METRICS_GPU_FILE, MODEL_METRICS_INFERENCE_FILE; do
77+
check_no_csv_exists $file
78+
if [ $? -ne 0 ]; then
79+
echo -e "\n***\n*** Test Output Verification Failed.\n***"
80+
cat $ANALYZER_LOG
81+
RET=1
82+
fi
83+
done
84+
fi
85+
set -e
86+
87+
if [ $RET -eq 0 ]; then
88+
echo -e "\n***\n*** Test PASSED\n***"
89+
else
90+
echo -e "\n***\n*** Test FAILED\n***"
91+
fi
92+
93+
exit $RET

‎qa/L0_quick_search/test_config_generator.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import argparse
16+
import yaml
17+
18+
19+
class TestConfigGenerator:
20+
"""
21+
This class contains functions that
22+
create configs for various test scenarios.
23+
24+
The `setup` function does the work common to all tests
25+
26+
TO ADD A TEST: Simply add a member function whose name starts
27+
with 'generate'.
28+
"""
29+
30+
def __init__(self):
31+
test_functions = [
32+
self.__getattribute__(name)
33+
for name in dir(self)
34+
if name.startswith('generate')
35+
]
36+
37+
for test_function in test_functions:
38+
self.setup()
39+
test_function()
40+
41+
def setup(self):
42+
parser = argparse.ArgumentParser()
43+
parser.add_argument('-m',
44+
'--profile-models',
45+
type=str,
46+
required=True,
47+
help='The config file for this test')
48+
49+
args = parser.parse_args()
50+
self.config = {}
51+
self.config['profile_models'] = sorted(args.profile_models.split(','))
52+
53+
def generate_config(self):
54+
with open('config.yml', 'w+') as f:
55+
yaml.dump(self.config, f)
56+
57+
58+
if __name__ == '__main__':
59+
TestConfigGenerator()

0 commit comments

Comments
 (0)